diff options
| author | James Zern <jzern@google.com> | 2017-06-27 23:30:16 +0000 |
|---|---|---|
| committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2017-06-27 23:30:17 +0000 |
| commit | 515fed8f387a0332d39de2f7fcbd4eefc1150b0d (patch) | |
| tree | 0388f0c9f724c4549f6e96ee43ef9edd6abe4c80 /vp9/encoder | |
| parent | 0bb31a46a4f3a490a4cb483fe4db79d7bb20a0d3 (diff) | |
| parent | 9e1d2de67c1e74334cb452523343faee2ef15341 (diff) | |
| download | libvpx-515fed8f387a0332d39de2f7fcbd4eefc1150b0d.tar libvpx-515fed8f387a0332d39de2f7fcbd4eefc1150b0d.tar.gz libvpx-515fed8f387a0332d39de2f7fcbd4eefc1150b0d.tar.bz2 libvpx-515fed8f387a0332d39de2f7fcbd4eefc1150b0d.zip | |
Merge "highbd_quantize_fp_32x32: normalize abs_qcoeff type"
Diffstat (limited to 'vp9/encoder')
| -rw-r--r-- | vp9/encoder/vp9_quantize.c | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/vp9/encoder/vp9_quantize.c b/vp9/encoder/vp9_quantize.c index f2a59a4af..da8ca4c08 100644 --- a/vp9/encoder/vp9_quantize.c +++ b/vp9/encoder/vp9_quantize.c @@ -138,7 +138,7 @@ void vp9_highbd_quantize_fp_32x32_c( if (!skip_block) { for (i = 0; i < n_coeffs; i++) { - uint32_t abs_qcoeff = 0; + int abs_qcoeff = 0; const int rc = scan[i]; const int coeff = coeff_ptr[rc]; const int coeff_sign = (coeff >> 31); @@ -147,7 +147,7 @@ void vp9_highbd_quantize_fp_32x32_c( if (abs_coeff >= (dequant_ptr[rc != 0] >> 2)) { const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1); - abs_qcoeff = (uint32_t)((tmp * quant_ptr[rc != 0]) >> 15); + abs_qcoeff = (int)((tmp * quant_ptr[rc != 0]) >> 15); qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2; } |