Simplify rdmult computation

Recognizing that max dc_quant used in rdmult computation is 21387 and
21387 * 21387 * 88 / 24 is still within the range of int32_t, this
commit simplifies the computation with minor cleanups.

Change-Id: I2ac7e8315d103c0bb39b70c312c87c0fda47b4f9

1 files changed, 9 insertions, 14 deletions

diff --git a/vp9/encoder/vp9_rd.c b/vp9/encoder/vp9_rd.c
index d6ec96b82..e65cf42d8 100644
--- a/vp9/encoder/vp9_rd.c
+++ b/vp9/encoder/vp9_rd.c
@@ -173,27 +173,23 @@ static const int rd_boost_factor[16] = { 64, 32, 32, 32, 24, 16, 12, 12,
 static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = { 128, 144, 128,
                                                               128, 144, 144 };
 
-int64_t vp9_compute_rd_mult_based_on_qindex(const VP9_COMP *cpi, int qindex) {
-  const int64_t q = vp9_dc_quant(qindex, 0, cpi->common.bit_depth);
+int vp9_compute_rd_mult_based_on_qindex(const VP9_COMP *cpi, int qindex) {
+  // largest dc_quant is 21387, therefore rdmult should always fit in int32_t
+  const int q = vp9_dc_quant(qindex, 0, cpi->common.bit_depth);
+  int rdmult = q * q;
+  rdmult = rdmult * 3 + (rdmult * 2 / 3);
 #if CONFIG_VP9_HIGHBITDEPTH
-  int64_t rdmult = 0;
   switch (cpi->common.bit_depth) {
-    case VPX_BITS_8: rdmult = 88 * q * q / 24; break;
-    case VPX_BITS_10: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 4); break;
-    default:
-      assert(cpi->common.bit_depth == VPX_BITS_12);
-      rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 8);
-      break;
+    case VPX_BITS_10: rdmult = ROUND_POWER_OF_TWO(rdmult, 4); break;
+    case VPX_BITS_12: rdmult = ROUND_POWER_OF_TWO(rdmult, 8); break;
+    default: break;
   }
-#else
-  int64_t rdmult = 88 * q * q / 24;
 #endif  // CONFIG_VP9_HIGHBITDEPTH
-  return rdmult;
+  return rdmult > 0 ? rdmult : 1;
 }
 
 int vp9_compute_rd_mult(const VP9_COMP *cpi, int qindex) {
   int64_t rdmult = vp9_compute_rd_mult_based_on_qindex(cpi, qindex);
-
   if (cpi->oxcf.pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
     const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
     const FRAME_UPDATE_TYPE frame_type = gf_group->update_type[gf_group->index];
@@ -205,7 +201,6 @@ int vp9_compute_rd_mult(const VP9_COMP *cpi, int qindex) {
     rdmult = (rdmult * rd_frame_type_factor[frame_type]) >> 7;
     rdmult += ((rdmult * rd_boost_factor[boost_index]) >> 7);
   }
-  if (rdmult < 1) rdmult = 1;
   return (int)rdmult;
 }