diff options
| author | Jingning Han <jingning@google.com> | 2015-06-16 12:00:50 -0700 |
|---|---|---|
| committer | Jingning Han <jingning@google.com> | 2015-06-16 19:31:24 -0700 |
| commit | bc7074508a3d0d877fadee02fb2027347f4a9c04 (patch) | |
| tree | a08899d918e5ba9dd0e548c2fa5ad338c949e494 /vp9/encoder/vp9_pickmode.c | |
| parent | e820ca6973f53815608d44854eebf380cc17f2f6 (diff) | |
| download | libvpx-bc7074508a3d0d877fadee02fb2027347f4a9c04.tar libvpx-bc7074508a3d0d877fadee02fb2027347f4a9c04.tar.gz libvpx-bc7074508a3d0d877fadee02fb2027347f4a9c04.tar.bz2 libvpx-bc7074508a3d0d877fadee02fb2027347f4a9c04.zip | |
Fix integer overflow issue in rtc coding flow intra mode search
The overflow issue affects a variable that is only used in inter
mode. This commit fixes the ioc warning triggered in the intra
mode. It does not affect the compression performance.
Change-Id: I593d1b5650599de07f3e68176dd1442c6cb7bdbc
Diffstat (limited to 'vp9/encoder/vp9_pickmode.c')
| -rw-r--r-- | vp9/encoder/vp9_pickmode.c | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c index 2479b6e73..944a34e3c 100644 --- a/vp9/encoder/vp9_pickmode.c +++ b/vp9/encoder/vp9_pickmode.c @@ -658,7 +658,8 @@ static void block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate, int64_t *dist, block = 0; *rate = 0; *dist = 0; - *sse = (*sse << 6) >> shift; + if (*sse < INT64_MAX) + *sse = (*sse << 6) >> shift; for (r = 0; r < max_blocks_high; r += block_step) { for (c = 0; c < num_4x4_w; c += block_step) { if (c < max_blocks_wide) { @@ -912,6 +913,8 @@ static void estimate_block_intra(int plane, int block, BLOCK_SIZE plane_bsize, // TODO(jingning): This needs further refactoring. block_yrd(cpi, x, &rate, &dist, &is_skippable, &this_sse, 0, bsize_tx, MIN(tx_size, TX_16X16)); + // this_sse is a dummy variable here. Its value should remain INT64_MAX. + assert(this_sse == INT64_MAX); x->skip_txfm[0] = is_skippable; rate += vp9_cost_bit(vp9_get_skip_prob(&cpi->common, xd), is_skippable); |