From 7f602d6114ed0544a2f8526f8660488e92b4038e Mon Sep 17 00:00:00 2001 From: Johann Date: Wed, 16 Aug 2017 10:22:48 -0700 Subject: quantize: normalize intermediate types Despite abs_coeff being a positive value, all the other implementations treat it as signed which simplifies restoring the sign. HBD builds cast qcoeff to avoid a visual studio warning. Match vp9_quantize.c style of casting the entire expression. Change-Id: I62b539b8df05364df3d7644311e325288da7c5b5 --- vp9/encoder/vp9_quantize.c | 2 +- vpx_dsp/quantize.c | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/vp9/encoder/vp9_quantize.c b/vp9/encoder/vp9_quantize.c index da8ca4c08..56bcd60f7 100644 --- a/vp9/encoder/vp9_quantize.c +++ b/vp9/encoder/vp9_quantize.c @@ -78,7 +78,7 @@ void vp9_highbd_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t count, const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int64_t tmp = abs_coeff + round_ptr[rc != 0]; const int abs_qcoeff = (int)((tmp * quant_ptr[rc != 0]) >> 16); - qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign; dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; if (abs_qcoeff) eob = i; } diff --git a/vpx_dsp/quantize.c b/vpx_dsp/quantize.c index 00fa4dc82..bef27048b 100644 --- a/vpx_dsp/quantize.c +++ b/vpx_dsp/quantize.c @@ -201,11 +201,8 @@ void vpx_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, if (abs_coeff >= zbins[rc != 0]) { const int64_t tmp1 = abs_coeff + round_ptr[rc != 0]; const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; - const uint32_t abs_qcoeff = - (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16); - // Restoring the sign triggers unsigned overflow warnings with negative - // values because the result of the xor operation is unsigned. - qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign; + const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 16); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; if (abs_qcoeff) eob = i; } @@ -310,9 +307,8 @@ void vpx_highbd_quantize_b_32x32_c( const int64_t tmp1 = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1); const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; - const uint32_t abs_qcoeff = - (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); - qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign; + const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2; if (abs_qcoeff) eob = idx_arr[i]; } -- cgit v1.2.3