diff options
| author | Yaowu Xu <yaowu@google.com> | 2013-10-10 13:38:16 -0700 |
|---|---|---|
| committer | Gerrit Code Review <gerrit@gerrit.golo.chromium.org> | 2013-10-10 13:38:16 -0700 |
| commit | e2d6e37a545e817322cf82c5f3b2264140977fb8 (patch) | |
| tree | 183edb554a1eac2ede170053ae701eea8e10e37a /vp9 | |
| parent | 09aca3089f7d35febb2454ce49c7c95cc22be92f (diff) | |
| parent | b47cef056e51751d5d7b1cb50f91a71110294a9c (diff) | |
| download | libvpx-e2d6e37a545e817322cf82c5f3b2264140977fb8.tar libvpx-e2d6e37a545e817322cf82c5f3b2264140977fb8.tar.gz libvpx-e2d6e37a545e817322cf82c5f3b2264140977fb8.tar.bz2 libvpx-e2d6e37a545e817322cf82c5f3b2264140977fb8.zip | |
Merge "change to avoid out-of-range computation"
Diffstat (limited to 'vp9')
| -rw-r--r-- | vp9/encoder/vp9_rdopt.c | 13 |
1 files changed, 10 insertions, 3 deletions
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index df960655d..5ba3ec8ad 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -3633,10 +3633,17 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, // values, which actually are bigger than this_rd itself. This can // cause negative best_filter_rd[] values, which is obviously silly. // Therefore, if filter_cache < ref, we do an adjusted calculation. - if (cpi->rd_filter_cache[i] >= ref) + if (cpi->rd_filter_cache[i] >= ref) { adj_rd = this_rd + cpi->rd_filter_cache[i] - ref; - else // FIXME(rbultje) do this for comppred also - adj_rd = this_rd - (ref - cpi->rd_filter_cache[i]) * this_rd / ref; + } else { + // FIXME(rbultje) do this for comppsred also + // + // To prevent out-of-range computation in + // adj_rd = cpi->rd_filter_cache[i] * this_rd / ref + // cpi->rd_filter_cache[i] / ref is converted to a 256 based ratio. + int tmp = cpi->rd_filter_cache[i] * 256 / ref; + adj_rd = (this_rd * tmp) >> 8; + } best_filter_rd[i] = MIN(best_filter_rd[i], adj_rd); } } |