fixed inconsistency of transform size decision

for MBs using i8x8 prediciton mode when hybridtransform8x8 is not
enabled in configure.

Change-Id: I3e3c8bb1c0261108c1c71213ca9557b0ce558036

3 files changed, 22 insertions, 32 deletions

diff --git a/vp8/common/blockd.h b/vp8/common/blockd.h
index 57dacf841..96345c2bd 100644
--- a/vp8/common/blockd.h
+++ b/vp8/common/blockd.h
@@ -479,34 +479,6 @@ static void txfm_map(BLOCKD *b, B_PREDICTION_MODE bmode) {
       break;
   }
 }
-
-static TX_TYPE get_tx_type(MACROBLOCKD *xd, BLOCKD *b) {
-  TX_TYPE tx_type = DCT_DCT;
-#if CONFIG_HYBRIDTRANSFORM16X16
-  if (xd->mode_info_context->mbmi.txfm_size == TX_16X16) {
-    if (xd->mode_info_context->mbmi.mode < I8X8_PRED &&
-        xd->q_index < ACTIVE_HT16)
-      tx_type = b->bmi.as_mode.tx_type;
-    return tx_type;
-  }
-#endif
-#if CONFIG_HYBRIDTRANSFORM8X8
-  if (xd->mode_info_context->mbmi.txfm_size  == TX_8X8) {
-    if (xd->mode_info_context->mbmi.mode == I8X8_PRED)
-      tx_type = b->bmi.as_mode.tx_type;
-    return tx_type;
-  }
-#endif
-#if CONFIG_HYBRIDTRANSFORM
-  if (xd->mode_info_context->mbmi.txfm_size  == TX_4X4) {
-    if (xd->mode_info_context->mbmi.mode == B_PRED &&
-        xd->q_index < ACTIVE_HT)
-      tx_type = b->bmi.as_mode.tx_type;
-    return tx_type;
-  }
-#endif
-  return tx_type;
-}
 #endif
 
 extern void vp8_build_block_doffsets(MACROBLOCKD *xd);
@@ -516,8 +488,8 @@ static void update_blockd_bmi(MACROBLOCKD *xd) {
   int i;
   int is_4x4;
   is_4x4 = (xd->mode_info_context->mbmi.mode == SPLITMV) ||
-      (xd->mode_info_context->mbmi.mode == I8X8_PRED) ||
-      (xd->mode_info_context->mbmi.mode == B_PRED);
+           (xd->mode_info_context->mbmi.mode == I8X8_PRED) ||
+           (xd->mode_info_context->mbmi.mode == B_PRED);
 
   if (is_4x4) {
     for (i = 0; i < 16; i++) {
diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c
index 80efa5802..e2d022388 100644
--- a/vp8/decoder/decodframe.c
+++ b/vp8/decoder/decodframe.c
@@ -243,6 +243,9 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
     else
 #endif
       if (pbi->common.txfm_mode == ALLOW_8X8 &&
+#if !CONFIG_HYBRIDTRANSFORM8X8
+          xd->mode_info_context->mbmi.mode != I8X8_PRED &&
+#endif
         xd->mode_info_context->mbmi.mode != B_PRED)
       xd->mode_info_context->mbmi.txfm_size = TX_8X8;
     else
@@ -258,6 +261,9 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
     else
 #endif
       if (pbi->common.txfm_mode == ALLOW_8X8 &&
+#if !CONFIG_HYBRIDTRANSFORM8X8
+        xd->mode_info_context->mbmi.mode != I8X8_PRED &&
+#endif
         xd->mode_info_context->mbmi.mode != B_PRED &&
         xd->mode_info_context->mbmi.mode != SPLITMV)
       xd->mode_info_context->mbmi.txfm_size = TX_8X8;
@@ -616,8 +622,11 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
 #if CONFIG_SUPERBLOCKS
     if (!xd->mode_info_context->mbmi.encoded_as_sb) {
 #endif
-      if ((tx_type == TX_8X8 &&
-           xd->mode_info_context->mbmi.mode != I8X8_PRED)
+      if ((tx_type == TX_8X8
+#if CONFIG_HYBRIDTRANSFORM8X8
+           && xd->mode_info_context->mbmi.mode != I8X8_PRED
+#endif
+           )
 #if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
       || tx_type == TX_16X16
 #endif
diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c
index e25b64305..0d125ec94 100644
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1833,6 +1833,9 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
   else
 #endif
   if (cpi->common.txfm_mode == ALLOW_8X8
+#if !CONFIG_HYBRIDTRANSFORM8X8
+      && mbmi->mode != I8X8_PRED
+#endif
       && mbmi->mode != B_PRED) {
     mbmi->txfm_size = TX_8X8;
     cpi->t8x8_count++;
@@ -1929,6 +1932,9 @@ void vp8cx_encode_inter_macroblock (VP8_COMP *cpi, MACROBLOCK *x,
   } else
 #endif
   if (cpi->common.txfm_mode == ALLOW_8X8
+#if !CONFIG_HYBRIDTRANSFORM8X8
+      && mbmi->mode != I8X8_PRED
+#endif
       && mbmi->mode != B_PRED
       && mbmi->mode != SPLITMV) {
     mbmi->txfm_size = TX_8X8;
@@ -2109,6 +2115,9 @@ void vp8cx_encode_inter_superblock(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t,
 
   /* test code: set transform size based on mode selection */
   if (cpi->common.txfm_mode == ALLOW_8X8
+#if !CONFIG_HYBRIDTRANSFORM8X8
+      && x->e_mbd.mode_info_context->mbmi.mode != I8X8_PRED
+#endif
       && x->e_mbd.mode_info_context->mbmi.mode != B_PRED
       && x->e_mbd.mode_info_context->mbmi.mode != SPLITMV) {
     x->e_mbd.mode_info_context->mbmi.txfm_size = TX_8X8;