diff options
| author | Yaowu Xu <yaowu@google.com> | 2013-08-29 10:26:52 -0700 |
|---|---|---|
| committer | Yaowu Xu <yaowu@google.com> | 2013-08-29 10:26:52 -0700 |
| commit | aaa7b444607225a0655be6b97ace8128a176028c (patch) | |
| tree | d45622e6aaff72b41dd66ed9f738995572444512 /vp9 | |
| parent | b71807082c10f44c5b80b7c81eaa90a93bd500e3 (diff) | |
| download | libvpx-aaa7b444607225a0655be6b97ace8128a176028c.tar libvpx-aaa7b444607225a0655be6b97ace8128a176028c.tar.gz libvpx-aaa7b444607225a0655be6b97ace8128a176028c.tar.bz2 libvpx-aaa7b444607225a0655be6b97ace8128a176028c.zip | |
Fixed potential overflows
The two arrays are typically initialized to INT64_MAX, if they are not
filled with valid values before the addition, the values can overflow
and lead to wrong results.
Change-Id: I515de22cf3e8f55af4b74bdb2c8eb821a02d3059
Diffstat (limited to 'vp9')
| -rw-r--r-- | vp9/encoder/vp9_rdopt.c | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index a51ce6e05..c8b7e1415 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -1247,7 +1247,7 @@ static int64_t rd_pick_intra_sby_mode(VP9_COMP *cpi, MACROBLOCK *x, } if (cpi->sf.tx_size_search_method == USE_FULL_RD && this_rd < INT64_MAX) { - for (i = 0; i < TX_MODES; i++) { + for (i = 0; i < TX_MODES && local_tx_cache[i] < INT64_MAX; i++) { const int64_t adj_rd = this_rd + local_tx_cache[i] - local_tx_cache[cpi->common.tx_mode]; if (adj_rd < tx_cache[i]) { @@ -3816,7 +3816,7 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, tx_cache[ALLOW_32X32] = tx_cache[ALLOW_16X16]; } if (!mode_excluded && this_rd != INT64_MAX) { - for (i = 0; i < TX_MODES; i++) { + for (i = 0; i < TX_MODES && tx_cache[i] < INT64_MAX; i++) { int64_t adj_rd = INT64_MAX; if (this_mode != I4X4_PRED) { adj_rd = this_rd + tx_cache[i] - tx_cache[cm->tx_mode]; |