diff options
| author | Yaowu Xu <yaowu@google.com> | 2014-04-09 17:32:48 -0700 |
|---|---|---|
| committer | Yaowu Xu <yaowu@google.com> | 2014-04-09 17:56:06 -0700 |
| commit | 1dcc1aa9423baf3bf453f9b2c0bcaed201920d81 (patch) | |
| tree | c1861e03e4362238b2f1d871e8b7b07084b0d076 /vp9/encoder | |
| parent | 2dc92482d060ba57926301d9e74dff962e412d7e (diff) | |
| download | libvpx-1dcc1aa9423baf3bf453f9b2c0bcaed201920d81.tar libvpx-1dcc1aa9423baf3bf453f9b2c0bcaed201920d81.tar.gz libvpx-1dcc1aa9423baf3bf453f9b2c0bcaed201920d81.tar.bz2 libvpx-1dcc1aa9423baf3bf453f9b2c0bcaed201920d81.zip | |
Prevent the usage of invalid best_mode_index
This commit changed the initialization of best_mode_index to -1 to make
sure it is not mistakenly used for mode masking.
Change-Id: I75b05db51466070dd23c4ee57a4d4b40764dc019
Diffstat (limited to 'vp9/encoder')
| -rw-r--r-- | vp9/encoder/vp9_rdopt.c | 10 |
1 files changed, 6 insertions, 4 deletions
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index f4ea7cd55..698088d52 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -3147,7 +3147,7 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, int64_t best_filter_rd[SWITCHABLE_FILTER_CONTEXTS]; int64_t best_filter_diff[SWITCHABLE_FILTER_CONTEXTS]; MB_MODE_INFO best_mbmode = { 0 }; - int mode_index, best_mode_index = 0; + int mode_index, best_mode_index = -1; unsigned int ref_costs_single[MAX_REF_FRAMES], ref_costs_comp[MAX_REF_FRAMES]; vp9_prob comp_mode_p; int64_t best_intra_rd = INT64_MAX; @@ -3304,7 +3304,7 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, // Look at the reference frame of the best mode so far and set the // skip mask to look at a subset of the remaining modes. - if (mode_index == mode_skip_start && best_rd < INT64_MAX) { + if (mode_index == mode_skip_start && best_mode_index >= 0) { switch (vp9_mode_order[best_mode_index].ref_frame[0]) { case INTRA_FRAME: break; @@ -3341,6 +3341,7 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, comp_pred = second_ref_frame > INTRA_FRAME; if (comp_pred) { if ((mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA) && + best_mode_index >=0 && vp9_mode_order[best_mode_index].ref_frame[0] == INTRA_FRAME) continue; if ((mode_search_skip_flags & FLAG_SKIP_COMP_REFMISMATCH) && @@ -3368,7 +3369,8 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, // one of the neighboring directional modes if ((mode_search_skip_flags & FLAG_SKIP_INTRA_BESTINTER) && (this_mode >= D45_PRED && this_mode <= TM_PRED)) { - if (vp9_mode_order[best_mode_index].ref_frame[0] > INTRA_FRAME) + if (best_mode_index >= 0 && + vp9_mode_order[best_mode_index].ref_frame[0] > INTRA_FRAME) continue; } if (mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) { @@ -3656,7 +3658,7 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, break; } - if (best_rd >= best_rd_so_far) + if (best_mode_index < 0 || best_rd >= best_rd_so_far) return INT64_MAX; // If we used an estimate for the uv intra rd in the loop above... |