Remove conversion warnings from hbd shifts

ROUND_POWER_OF_TWO has some poor side effects when used
with [u]int64_t such as doing the shifting in 32bits.

Change-Id: Ic85a19765cd316fb43657cb21c86f35ceb772773

1 files changed, 22 insertions, 12 deletions

diff --git a/test/variance_test.cc b/test/variance_test.cc
index 23d4ae725..075254208 100644
--- a/test/variance_test.cc
+++ b/test/variance_test.cc
@@ -41,6 +41,25 @@ using ::std::tr1::make_tuple;
 using ::std::tr1::tuple;
 using libvpx_test::ACMRandom;
 
+// Truncate high bit depth results by downshifting (with rounding) by:
+// 2 * (bit_depth - 8) for sse
+// (bit_depth - 8) for se
+static void RoundHighBitDepth(int bit_depth, int64_t *se, uint64_t *sse) {
+  switch (bit_depth) {
+    case VPX_BITS_12:
+      *sse = (*sse + 128) >> 8;
+      *se = (*se + 8) >> 4;
+      break;
+    case VPX_BITS_10:
+      *sse = (*sse + 8) >> 4;
+      *se = (*se + 2) >> 2;
+      break;
+    case VPX_BITS_8:
+    default:
+      break;
+  }
+}
+
 static unsigned int mb_ss_ref(const int16_t *src) {
   unsigned int res = 0;
   for (int i = 0; i < 256; ++i) {
@@ -76,10 +95,7 @@ static unsigned int variance_ref(const uint8_t *src, const uint8_t *ref,
       }
     }
   }
-  if (bit_depth > VPX_BITS_8) {
-    sse = ROUND_POWER_OF_TWO(sse, 2 * (bit_depth - 8));
-    se = ROUND_POWER_OF_TWO(se, bit_depth - 8);
-  }
+  RoundHighBitDepth(bit_depth, &se, &sse);
   *sse_ptr = sse;
   return sse - (((int64_t) se * se) >> (l2w + l2h));
 }
@@ -125,10 +141,7 @@ static unsigned int subpel_variance_ref(const uint8_t *ref, const uint8_t *src,
       }
     }
   }
-  if (bit_depth > VPX_BITS_8) {
-    sse = ROUND_POWER_OF_TWO(sse, 2 * (bit_depth - 8));
-    se = ROUND_POWER_OF_TWO(se, bit_depth - 8);
-  }
+  RoundHighBitDepth(bit_depth, &se, &sse);
   *sse_ptr = sse;
   return sse - (((int64_t) se * se) >> (l2w + l2h));
 }
@@ -496,10 +509,7 @@ unsigned int subpel_avg_variance_ref(const uint8_t *ref,
       }
     }
   }
-  if (bit_depth > 8) {
-    sse = ROUND_POWER_OF_TWO(sse, 2*(bit_depth-8));
-    se = ROUND_POWER_OF_TWO(se, bit_depth-8);
-  }
+  RoundHighBitDepth(bit_depth, &se, &sse);
   *sse_ptr = sse;
   return sse - (((int64_t) se * se) >> (l2w + l2h));
 }