diff options
author | James Zern <jzern@google.com> | 2017-07-22 13:01:49 -0700 |
---|---|---|
committer | James Zern <jzern@google.com> | 2017-07-22 13:27:32 -0700 |
commit | 8836e46ffd84f3e272ea9e5efefa143c2437829c (patch) | |
tree | 44d190689f6ffd3ebc3038854a038471e8250705 /vp9 | |
parent | 8c7a60e04d3b7e9f5189685cfa9701445aeac3fc (diff) | |
download | libvpx-8836e46ffd84f3e272ea9e5efefa143c2437829c.tar libvpx-8836e46ffd84f3e272ea9e5efefa143c2437829c.tar.gz libvpx-8836e46ffd84f3e272ea9e5efefa143c2437829c.tar.bz2 libvpx-8836e46ffd84f3e272ea9e5efefa143c2437829c.zip |
set_var_thresh_from_histogram: prevent negative variance
For 8-bit the subtrahend is small enough to fit into uint32_t.
For 10/12-bit apply:
63a37d16f Prevent negative variance
previously:
47b9a0912 Resolve -Wshorten-64-to-32 in highbd variance.
c0241664a Resolve -Wshorten-64-to-32 in variance.
Change-Id: I181c85f0b9a03da37c2e8b89482d48aa3dbc0aee
Diffstat (limited to 'vp9')
-rw-r--r-- | vp9/encoder/vp9_encodeframe.c | 17 |
1 files changed, 16 insertions, 1 deletions
diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c index e69b337b6..d8115bc25 100644 --- a/vp9/encoder/vp9_encodeframe.c +++ b/vp9/encoder/vp9_encodeframe.c @@ -4231,6 +4231,17 @@ static void encode_nonrd_sb_row(VP9_COMP *cpi, ThreadData *td, } // end RTC play code +static INLINE uint32_t variance(const diff *const d) { + return d->sse - (uint32_t)(((int64_t)d->sum * d->sum) >> 8); +} + +#if CONFIG_VP9_HIGHBITDEPTH +static INLINE uint32_t variance_highbd(diff *const d) { + const int64_t var = (int64_t)d->sse - (((int64_t)d->sum * d->sum) >> 8); + return (var >= 0) ? (uint32_t)var : 0; +} +#endif // CONFIG_VP9_HIGHBITDEPTH + static int set_var_thresh_from_histogram(VP9_COMP *cpi) { const SPEED_FEATURES *const sf = &cpi->sf; const VP9_COMMON *const cm = &cpi->common; @@ -4260,14 +4271,17 @@ static int set_var_thresh_from_histogram(VP9_COMP *cpi) { case VPX_BITS_8: vpx_highbd_8_get16x16var(src, src_stride, last_src, last_stride, &var16->sse, &var16->sum); + var16->var = variance(var16); break; case VPX_BITS_10: vpx_highbd_10_get16x16var(src, src_stride, last_src, last_stride, &var16->sse, &var16->sum); + var16->var = variance_highbd(var16); break; case VPX_BITS_12: vpx_highbd_12_get16x16var(src, src_stride, last_src, last_stride, &var16->sse, &var16->sum); + var16->var = variance_highbd(var16); break; default: assert(0 && @@ -4278,12 +4292,13 @@ static int set_var_thresh_from_histogram(VP9_COMP *cpi) { } else { vpx_get16x16var(src, src_stride, last_src, last_stride, &var16->sse, &var16->sum); + var16->var = variance(var16); } #else vpx_get16x16var(src, src_stride, last_src, last_stride, &var16->sse, &var16->sum); + var16->var = variance(var16); #endif // CONFIG_VP9_HIGHBITDEPTH - var16->var = var16->sse - (((uint32_t)var16->sum * var16->sum) >> 8); if (var16->var >= VAR_HIST_MAX_BG_VAR) hist[VAR_HIST_BINS - 1]++; |