1 files changed, 33 insertions, 4 deletions
diff --git a/vp9/common/vp9_idct.c b/vp9/common/vp9_idct.c
index ff7c1dd3f..dc2e03946 100644
--- a/vp9/common/vp9_idct.c
+++ b/vp9/common/vp9_idct.c
@@ -139,8 +139,6 @@ void vp9_idct8x8_add(const tran_low_t *input, uint8_t *dest, int stride,
 
   // The calculation can be simplified if there are not many non-zero dct
   // coefficients. Use eobs to decide what to do.
-  // TODO(yunqingwang): "eobs = 1" case is also handled in vp9_short_idct8x8_c.
-  // Combine that with code here.
   if (eob == 1)
     // DC only DCT coefficient
     vpx_idct8x8_1_add(input, dest, stride);
@@ -204,6 +202,18 @@ void vp9_iht16x16_add(TX_TYPE tx_type, const tran_low_t *input, uint8_t *dest,
 }
 
 #if CONFIG_VP9_HIGHBITDEPTH
+
+// 12 signal input bits + 7 forward transform amplify bits + 1 bit
+// for contingency in rounding and quantizing
+#define VALID_IHT_MAGNITUDE_RANGE (1 << 20)
+
+static INLINE int detect_invalid_iht_input(const tran_low_t *input, int size) {
+  int i;
+  for (i = 0; i < size; ++i)
+    if (abs(input[i]) >= VALID_IHT_MAGNITUDE_RANGE) return 1;
+  return 0;
+}
+
 void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,
                                 int stride, int tx_type, int bd) {
   const highbd_transform_2d IHT_4[] = {
@@ -219,6 +229,13 @@ void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,
   tran_low_t *outptr = out;
   tran_low_t temp_in[4], temp_out[4];
 
+  if (detect_invalid_iht_input(input, 16)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd iht input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    return;
+  }
+
   // Inverse transform row vectors.
   for (i = 0; i < 4; ++i) {
     IHT_4[tx_type].rows(input, outptr, bd);
@@ -253,6 +270,13 @@ void vp9_highbd_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest8,
   const highbd_transform_2d ht = HIGH_IHT_8[tx_type];
   uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);
 
+  if (detect_invalid_iht_input(input, 64)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd iht input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    return;
+  }
+
   // Inverse transform row vectors.
   for (i = 0; i < 8; ++i) {
     ht.rows(input, outptr, bd);
@@ -287,6 +311,13 @@ void vp9_highbd_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest8,
   const highbd_transform_2d ht = HIGH_IHT_16[tx_type];
   uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);
 
+  if (detect_invalid_iht_input(input, 256)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd iht input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    return;
+  }
+
   // Rows
   for (i = 0; i < 16; ++i) {
     ht.rows(input, outptr, bd);
@@ -329,8 +360,6 @@ void vp9_highbd_idct8x8_add(const tran_low_t *input, uint8_t *dest, int stride,
 
   // The calculation can be simplified if there are not many non-zero dct
   // coefficients. Use eobs to decide what to do.
-  // TODO(yunqingwang): "eobs = 1" case is also handled in vp9_short_idct8x8_c.
-  // Combine that with code here.
   // DC only DCT coefficient
   if (eob == 1) {
     vpx_highbd_idct8x8_1_add(input, dest, stride, bd);