Change highbd variance rounding to prevent negative variance.

Always round sum error and sum square error toward zero in variance
calculations. This prevents variance from becoming negative.
Avoiding rounding variance at all might be better but would be far
more invasive.

Change-Id: Icf24e0e75ff94952fc026ba6a4d26adf8d373f1c

1 files changed, 11 insertions, 10 deletions

diff --git a/vpx_dsp/x86/highbd_variance_sse2.c b/vpx_dsp/x86/highbd_variance_sse2.c
index b45331caa..c45e7dbb9 100644
--- a/vpx_dsp/x86/highbd_variance_sse2.c
+++ b/vpx_dsp/x86/highbd_variance_sse2.c
@@ -9,6 +9,7 @@
  */
 #include "./vpx_config.h"
 
+#include "vpx_dsp/variance.h"
 #include "vpx_ports/mem.h"
 
 typedef uint32_t (*high_variance_fn_t) (const uint16_t *src, int src_stride,
@@ -62,8 +63,8 @@ static void highbd_10_variance_sse2(const uint16_t *src, int src_stride,
       sum_long += sum0;
     }
   }
-  *sum = ROUND_POWER_OF_TWO(sum_long, 2);
-  *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4);
+  *sum = ROUND_ZERO_POWER_OF_TWO(sum_long, 2);
+  *sse = (uint32_t)ROUND_ZERO_POWER_OF_TWO(sse_long, 4);
 }
 
 static void highbd_12_variance_sse2(const uint16_t *src, int src_stride,
@@ -84,8 +85,8 @@ static void highbd_12_variance_sse2(const uint16_t *src, int src_stride,
       sum_long += sum0;
     }
   }
-  *sum = ROUND_POWER_OF_TWO(sum_long, 4);
-  *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 8);
+  *sum = ROUND_ZERO_POWER_OF_TWO(sum_long, 4);
+  *sse = (uint32_t)ROUND_ZERO_POWER_OF_TWO(sse_long, 8);
 }
 
 
@@ -106,7 +107,7 @@ void vpx_highbd_10_get##S##x##S##var_sse2(const uint8_t *src8, int src_stride, \
   uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \
   vpx_highbd_calc##S##x##S##var_sse2(src, src_stride, ref, ref_stride, \
                                      sse, sum); \
-  *sum = ROUND_POWER_OF_TWO(*sum, 2); \
+  *sum = ROUND_ZERO_POWER_OF_TWO(*sum, 2); \
   *sse = ROUND_POWER_OF_TWO(*sse, 4); \
 } \
 \
@@ -117,7 +118,7 @@ void vpx_highbd_12_get##S##x##S##var_sse2(const uint8_t *src8, int src_stride, \
   uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \
   vpx_highbd_calc##S##x##S##var_sse2(src, src_stride, ref, ref_stride, \
                                      sse, sum); \
-  *sum = ROUND_POWER_OF_TWO(*sum, 4); \
+  *sum = ROUND_ZERO_POWER_OF_TWO(*sum, 4); \
   *sse = ROUND_POWER_OF_TWO(*sse, 8); \
 }
 
@@ -338,7 +339,7 @@ uint32_t vpx_highbd_10_sub_pixel_variance##w##x##h##_##opt( \
       sse += sse2; \
     } \
   } \
-  se = ROUND_POWER_OF_TWO(se, 2); \
+  se = ROUND_ZERO_POWER_OF_TWO(se, 2); \
   sse = ROUND_POWER_OF_TWO(sse, 4); \
   *sse_ptr = sse; \
   return sse - ((cast se * se) >> (wlog2 + hlog2)); \
@@ -385,7 +386,7 @@ uint32_t vpx_highbd_12_sub_pixel_variance##w##x##h##_##opt( \
       }\
     } \
   } \
-  se = ROUND_POWER_OF_TWO(se, 4); \
+  se = ROUND_ZERO_POWER_OF_TWO(se, 4); \
   sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8); \
   *sse_ptr = sse; \
   return sse - ((cast se * se) >> (wlog2 + hlog2)); \
@@ -502,7 +503,7 @@ uint32_t vpx_highbd_10_sub_pixel_avg_variance##w##x##h##_##opt( \
       sse += sse2; \
     } \
   } \
-  se = ROUND_POWER_OF_TWO(se, 2); \
+  se = ROUND_ZERO_POWER_OF_TWO(se, 2); \
   sse = ROUND_POWER_OF_TWO(sse, 4); \
   *sse_ptr = sse; \
   return sse - ((cast se * se) >> (wlog2 + hlog2)); \
@@ -554,7 +555,7 @@ uint32_t vpx_highbd_12_sub_pixel_avg_variance##w##x##h##_##opt( \
       } \
     } \
   } \
-  se = ROUND_POWER_OF_TWO(se, 4); \
+  se = ROUND_ZERO_POWER_OF_TWO(se, 4); \
   sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8); \
   *sse_ptr = sse; \
   return sse - ((cast se * se) >> (wlog2 + hlog2)); \