diff options
| author | Johann Koenig <johannkoenig@google.com> | 2017-08-18 16:00:28 +0000 |
|---|---|---|
| committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2017-08-18 16:00:28 +0000 |
| commit | 1426f04e91fa15c3443d279928e8af3d4e71e4bc (patch) | |
| tree | 3694800b634a4685154d1c3f69a91c13a9cf3383 /vpx_dsp | |
| parent | bb15fd51be0c699d9c2ef088d807762d25311d6e (diff) | |
| parent | 7f602d6114ed0544a2f8526f8660488e92b4038e (diff) | |
| download | libvpx-1426f04e91fa15c3443d279928e8af3d4e71e4bc.tar libvpx-1426f04e91fa15c3443d279928e8af3d4e71e4bc.tar.gz libvpx-1426f04e91fa15c3443d279928e8af3d4e71e4bc.tar.bz2 libvpx-1426f04e91fa15c3443d279928e8af3d4e71e4bc.zip | |
Merge "quantize: normalize intermediate types"
Diffstat (limited to 'vpx_dsp')
| -rw-r--r-- | vpx_dsp/quantize.c | 12 |
1 files changed, 4 insertions, 8 deletions
diff --git a/vpx_dsp/quantize.c b/vpx_dsp/quantize.c index 00fa4dc82..bef27048b 100644 --- a/vpx_dsp/quantize.c +++ b/vpx_dsp/quantize.c @@ -201,11 +201,8 @@ void vpx_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, if (abs_coeff >= zbins[rc != 0]) { const int64_t tmp1 = abs_coeff + round_ptr[rc != 0]; const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; - const uint32_t abs_qcoeff = - (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16); - // Restoring the sign triggers unsigned overflow warnings with negative - // values because the result of the xor operation is unsigned. - qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign; + const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 16); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; if (abs_qcoeff) eob = i; } @@ -310,9 +307,8 @@ void vpx_highbd_quantize_b_32x32_c( const int64_t tmp1 = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1); const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; - const uint32_t abs_qcoeff = - (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); - qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign; + const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2; if (abs_qcoeff) eob = idx_arr[i]; } |