Separating 8x8DCT use from hybridtransform8x8

With this change, even if hybridtransform8x8 experiment is off,
8x8 dct is used for the I8x8 mode. However note that the gains
observed with the hybridtransform8x8 experiment will now be less,
since part of the gain is now merged in.

Change-Id: I9afb3880906fd0a1368a374041fc08efcf060c54

7 files changed, 52 insertions, 159 deletions

diff --git a/vp8/common/blockd.h b/vp8/common/blockd.h
index 96345c2bd..6f980ad65 100644
--- a/vp8/common/blockd.h
+++ b/vp8/common/blockd.h
@@ -89,10 +89,6 @@ typedef enum
 #endif
 } INTERPOLATIONFILTERTYPE;
 
-#if 0//CONFIG_SWITCHABLE_INTERP
-#define VP8_SWITCHABLE_FILTERS 2 /* number of switchable filters */
-#endif
-
 typedef enum
 {
   DC_PRED,            /* average of above and left pixels */
@@ -479,6 +475,40 @@ static void txfm_map(BLOCKD *b, B_PREDICTION_MODE bmode) {
       break;
   }
 }
+
+static TX_TYPE get_tx_type(MACROBLOCKD *xd, const BLOCKD *b) {
+  TX_TYPE tx_type = DCT_DCT;
+  int ib = (b - xd->block);
+  if (ib >= 16) return tx_type;
+#if CONFIG_HYBRIDTRANSFORM16X16
+  if (xd->mode_info_context->mbmi.txfm_size == TX_16X16) {
+    if (xd->mode_info_context->mbmi.mode < I8X8_PRED &&
+        xd->q_index < ACTIVE_HT16)
+      tx_type = b->bmi.as_mode.tx_type;
+    return tx_type;
+  }
+#endif
+#if CONFIG_HYBRIDTRANSFORM8X8
+  if (xd->mode_info_context->mbmi.txfm_size  == TX_8X8) {
+    BLOCKD *bb;
+    ib = (ib & 8) + ((ib & 4) >> 1);
+    bb = xd->block + ib;
+    if (xd->mode_info_context->mbmi.mode == I8X8_PRED)
+      tx_type = bb->bmi.as_mode.tx_type;
+    return tx_type;
+  }
+#endif
+#if CONFIG_HYBRIDTRANSFORM
+  if (xd->mode_info_context->mbmi.txfm_size  == TX_4X4) {
+    if (xd->mode_info_context->mbmi.mode == B_PRED &&
+        xd->q_index < ACTIVE_HT) {
+      tx_type = b->bmi.as_mode.tx_type;
+    }
+    return tx_type;
+  }
+#endif
+  return tx_type;
+}
 #endif
 
 extern void vp8_build_block_doffsets(MACROBLOCKD *xd);
diff --git a/vp8/common/entropy.h b/vp8/common/entropy.h
index f82df9681..ec9f90ef5 100644
--- a/vp8/common/entropy.h
+++ b/vp8/common/entropy.h
@@ -62,11 +62,7 @@ extern vp8_extra_bit_struct vp8_extra_bits[12];    /* indexed by token value */
 /* Outside dimension.  0 = Y no DC, 1 = Y2, 2 = UV, 3 = Y with DC */
 #define BLOCK_TYPES 4
 
-#if CONFIG_HYBRIDTRANSFORM8X8
 #define BLOCK_TYPES_8X8 4
-#else
-#define BLOCK_TYPES_8X8 3
-#endif
 
 #define BLOCK_TYPES_16X16 4
 
diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c
index e2d022388..e9b0f1ad0 100644
--- a/vp8/decoder/decodframe.c
+++ b/vp8/decoder/decodframe.c
@@ -243,9 +243,6 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
     else
 #endif
       if (pbi->common.txfm_mode == ALLOW_8X8 &&
-#if !CONFIG_HYBRIDTRANSFORM8X8
-          xd->mode_info_context->mbmi.mode != I8X8_PRED &&
-#endif
         xd->mode_info_context->mbmi.mode != B_PRED)
       xd->mode_info_context->mbmi.txfm_size = TX_8X8;
     else
@@ -261,9 +258,6 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
     else
 #endif
       if (pbi->common.txfm_mode == ALLOW_8X8 &&
-#if !CONFIG_HYBRIDTRANSFORM8X8
-        xd->mode_info_context->mbmi.mode != I8X8_PRED &&
-#endif
         xd->mode_info_context->mbmi.mode != B_PRED &&
         xd->mode_info_context->mbmi.mode != SPLITMV)
       xd->mode_info_context->mbmi.txfm_size = TX_8X8;
@@ -622,13 +616,10 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
 #if CONFIG_SUPERBLOCKS
     if (!xd->mode_info_context->mbmi.encoded_as_sb) {
 #endif
-      if ((tx_type == TX_8X8
-#if CONFIG_HYBRIDTRANSFORM8X8
-           && xd->mode_info_context->mbmi.mode != I8X8_PRED
-#endif
-           )
+      if ((tx_type == TX_8X8 &&
+           xd->mode_info_context->mbmi.mode != I8X8_PRED)
 #if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
-      || tx_type == TX_16X16
+          || tx_type == TX_16X16
 #endif
       )
     DEQUANT_INVOKE(&pbi->dequant, idct_add_uv_block_8x8) //
diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c
index 0d125ec94..9b793e20a 100644
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1833,9 +1833,6 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
   else
 #endif
   if (cpi->common.txfm_mode == ALLOW_8X8
-#if !CONFIG_HYBRIDTRANSFORM8X8
-      && mbmi->mode != I8X8_PRED
-#endif
       && mbmi->mode != B_PRED) {
     mbmi->txfm_size = TX_8X8;
     cpi->t8x8_count++;
@@ -1852,8 +1849,9 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
   else
     vp8_encode_intra16x16mby(IF_RTCD(&cpi->rtcd), x);
 
-  if (mbmi->mode != I8X8_PRED)
+  if (mbmi->mode != I8X8_PRED) {
     vp8_encode_intra16x16mbuv(IF_RTCD(&cpi->rtcd), x);
+  }
 
   if (output_enabled) {
     // Tokenize
@@ -1932,9 +1930,6 @@ void vp8cx_encode_inter_macroblock (VP8_COMP *cpi, MACROBLOCK *x,
   } else
 #endif
   if (cpi->common.txfm_mode == ALLOW_8X8
-#if !CONFIG_HYBRIDTRANSFORM8X8
-      && mbmi->mode != I8X8_PRED
-#endif
       && mbmi->mode != B_PRED
       && mbmi->mode != SPLITMV) {
     mbmi->txfm_size = TX_8X8;
@@ -2115,9 +2110,6 @@ void vp8cx_encode_inter_superblock(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t,
 
   /* test code: set transform size based on mode selection */
   if (cpi->common.txfm_mode == ALLOW_8X8
-#if !CONFIG_HYBRIDTRANSFORM8X8
-      && x->e_mbd.mode_info_context->mbmi.mode != I8X8_PRED
-#endif
       && x->e_mbd.mode_info_context->mbmi.mode != B_PRED
       && x->e_mbd.mode_info_context->mbmi.mode != SPLITMV) {
     x->e_mbd.mode_info_context->mbmi.txfm_size = TX_8X8;
diff --git a/vp8/encoder/encodeintra.c b/vp8/encoder/encodeintra.c
index 85b8e4f3d..c2f123c92 100644
--- a/vp8/encoder/encodeintra.c
+++ b/vp8/encoder/encodeintra.c
@@ -22,10 +22,6 @@
 #include "encodeintra.h"
 
 
-#ifdef ENC_DEBUG
-extern int enc_debug;
-#endif
-
 #if CONFIG_RUNTIME_CPU_DETECT
 #define IF_RTCD(x) (x)
 #else
@@ -208,39 +204,8 @@ void vp8_encode_intra16x16mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
   else
     vp8_inverse_transform_mby(IF_RTCD(&rtcd->common->idct), &x->e_mbd);
 
-#ifdef ENC_DEBUG
-  if (enc_debug) {
-    int i;
-    printf("Intra qcoeff:\n");
-    printf("%d %d:\n", x->e_mbd.mb_to_left_edge, x->e_mbd.mb_to_top_edge);
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.qcoeff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("Intra dqcoeff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.dqcoeff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("Intra diff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.diff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("Intra predictor:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.predictor[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("eobs:\n");
-    for (i = 0; i < 25; i++)
-      printf("%d ", x->e_mbd.block[i].eob);
-    printf("\n");
-  }
-#endif
-
   RECON_INVOKE(&rtcd->common->recon, recon_mby)
-  (IF_RTCD(&rtcd->common->recon), &x->e_mbd);
+      (IF_RTCD(&rtcd->common->recon), &x->e_mbd);
 
 }
 
@@ -270,37 +235,6 @@ void vp8_encode_intra16x16mbuv(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
   else
     vp8_quantize_mbuv(x);
 
-#ifdef ENC_DEBUG
-  if (enc_debug) {
-    int i;
-    printf("vp8_encode_intra16x16mbuv\n");
-    printf("%d %d:\n", x->e_mbd.mb_to_left_edge, x->e_mbd.mb_to_top_edge);
-    printf("qcoeff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.qcoeff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("dqcoeff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.dqcoeff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("diff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.diff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("predictor:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.predictor[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("eobs:\n");
-    for (i = 0; i < 25; i++)
-      printf("%d ", x->e_mbd.block[i].eob);
-    printf("\n");
-  }
-#endif
   if (x->optimize) {
     if (tx_type == TX_8X8)
       vp8_optimize_mbuv_8x8(x, rtcd);
@@ -335,7 +269,6 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
   }
 #endif
 
-#if CONFIG_HYBRIDTRANSFORM8X8
   {
     MACROBLOCKD *xd = &x->e_mbd;
     int idx = (ib & 0x02) ? (ib + 2) : ib;
@@ -343,12 +276,18 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
     // generate residual blocks
     vp8_subtract_4b_c(be, b, 16);
 
+#if CONFIG_HYBRIDTRANSFORM8X8
     txfm_map(b, pred_mode_conv(b->bmi.as_mode.first));
     vp8_fht_c(be->src_diff, (x->block + idx)->coeff, 32,
               b->bmi.as_mode.tx_type, 8);
     x->quantize_b_8x8(x->block + idx, xd->block + idx);
     vp8_ihtllm_c(xd->block[idx].dqcoeff, xd->block[ib].diff, 32,
                  b->bmi.as_mode.tx_type, 8);
+#else
+    x->vp8_short_fdct8x8(be->src_diff, (x->block + idx)->coeff, 32);
+    x->quantize_b_8x8(x->block + idx, xd->block + idx);
+    vp8_idct_idct8(xd->block[idx].dqcoeff, xd->block[ib].diff, 32);
+#endif
 
     // reconstruct submacroblock
     for (i = 0; i < 4; i++) {
@@ -357,19 +296,6 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
                     b->dst_stride);
     }
   }
-#else
-  for (i = 0; i < 4; i++) {
-    b = &x->e_mbd.block[ib + iblock[i]];
-    be = &x->block[ib + iblock[i]];
-    ENCODEMB_INVOKE(&rtcd->encodemb, subb)(be, b, 16);
-    x->vp8_short_fdct4x4(be->src_diff, be->coeff, 32);
-    x->quantize_b(be, b);
-    vp8_inverse_transform_b(IF_RTCD(&rtcd->common->idct), b, 32);
-    RECON_INVOKE(&rtcd->common->recon, recon)(b->predictor,
-                                              b->diff, *(b->base_dst) + b->dst,
-                                              b->dst_stride);
-  }
-#endif
 }
 
 extern const int vp8_i8x8_block[4];
@@ -380,7 +306,6 @@ void vp8_encode_intra8x8mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
     ib = vp8_i8x8_block[i];
     vp8_encode_intra8x8(rtcd, x, ib);
   }
-
 }
 
 void vp8_encode_intra_uv4x4(const VP8_ENCODER_RTCD *rtcd,
diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c
index 49f5b99b1..fd03fdb40 100644
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -1332,11 +1332,9 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
   DECLARE_ALIGNED_ARRAY(16, unsigned char,  best_predictor, 16 * 8);
   DECLARE_ALIGNED_ARRAY(16, short, best_dqcoeff, 16 * 4);
 
-#if CONFIG_HYBRIDTRANSFORM8X8
   // perform transformation of dimension 8x8
   // note the input and output index mapping
   int idx = (ib & 0x02) ? (ib + 2) : ib;
-#endif
 
   for (mode = DC_PRED; mode <= TM_PRED; mode++) {
 #if CONFIG_COMP_INTRA_PRED
@@ -1367,52 +1365,25 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
       txfm_map(b, pred_mode_conv(mode));
       vp8_fht_c(be->src_diff, (x->block + idx)->coeff, 32,
                 b->bmi.as_mode.tx_type, 8);
+
+#else
+      x->vp8_short_fdct8x8(be->src_diff, (x->block + idx)->coeff, 32);
+#endif
+
       x->quantize_b_8x8(x->block + idx, xd->block + idx);
 
       // compute quantization mse of 8x8 block
       distortion = vp8_block_error_c((x->block + idx)->coeff,
                                      (xd->block + idx)->dqcoeff, 64)>>2;
-
       ta0 = *(a + vp8_block2above_8x8[idx]);
       tl0 = *(l + vp8_block2left_8x8 [idx]);
 
       rate_t = cost_coeffs(x, xd->block + idx, PLANE_TYPE_Y_WITH_DC,
                            &ta0, &tl0, TX_8X8);
+
       rate += rate_t;
       ta1 = ta0;
       tl1 = tl0;
-#else
-      x->vp8_short_fdct8x4(be->src_diff, be->coeff, 32);
-      x->vp8_short_fdct8x4(be->src_diff + 64, be->coeff + 64, 32);
-
-      x->quantize_b_pair(x->block + ib, x->block + ib + 1,
-                         xd->block + ib, xd->block + ib + 1);
-      x->quantize_b_pair(x->block + ib + 4, x->block + ib + 5,
-                         xd->block + ib + 4, xd->block + ib + 5);
-
-      distortion = ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), berr)
-                   ((x->block + ib)->coeff, (xd->block + ib)->dqcoeff, 16) >> 2;
-      distortion += ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), berr)
-                    ((x->block + ib + 1)->coeff, (xd->block + ib + 1)->dqcoeff, 16) >> 2;
-      distortion += ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), berr)
-                    ((x->block + ib + 4)->coeff, (xd->block + ib + 4)->dqcoeff, 16) >> 2;
-      distortion += ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), berr)
-                    ((x->block + ib + 5)->coeff, (xd->block + ib + 5)->dqcoeff, 16) >> 2;
-
-      ta0 = *(a + vp8_block2above[ib]);
-      ta1 = *(a + vp8_block2above[ib + 1]);
-      tl0 = *(l + vp8_block2above[ib]);
-      tl1 = *(l + vp8_block2above[ib + 4]);
-      rate_t = cost_coeffs(x, xd->block + ib, PLANE_TYPE_Y_WITH_DC,
-                           &ta0, &tl0, TX_4X4);
-      rate_t += cost_coeffs(x, xd->block + ib + 1, PLANE_TYPE_Y_WITH_DC,
-                            &ta1, &tl0, TX_4X4);
-      rate_t += cost_coeffs(x, xd->block + ib + 4, PLANE_TYPE_Y_WITH_DC,
-                            &ta0, &tl1, TX_4X4);
-      rate_t += cost_coeffs(x, xd->block + ib + 5, PLANE_TYPE_Y_WITH_DC,
-                            &ta1, &tl1, TX_4X4);
-      rate += rate_t;
-#endif
 
       this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
       if (this_rd < best_rd) {
diff --git a/vp8/encoder/tokenize.c b/vp8/encoder/tokenize.c
index d4d410a6e..d992e1fe6 100644
--- a/vp8/encoder/tokenize.c
+++ b/vp8/encoder/tokenize.c
@@ -55,12 +55,6 @@ const TOKENVALUE *vp8_dct_value_tokens_ptr;
 static int dct_value_cost[DCT_MAX_VALUE * 2];
 const int *vp8_dct_value_cost_ptr;
 
-#ifdef ENC_DEBUG
-extern int mb_row_debug;
-extern int mb_col_debug;
-extern int enc_debug;
-#endif
-
 static void fill_value_tokens() {
 
   TOKENVALUE *const t = dct_value_tokens + DCT_MAX_VALUE;
@@ -218,12 +212,6 @@ static void tokenize2nd_order_b_8x8
 
     t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0));
 
-#ifdef ENC_DEBUG
-    if (t->skip_eob_node && vp8_coef_encodings[x].Len == 1)
-      printf("Trouble 2 x=%d Len=%d skip=%d eob=%d c=%d band=%d type=%d: [%d %d %d]\n",
-             x, vp8_coef_encodings[x].Len, t->skip_eob_node, eob, c, band, type,
-             cpi->count, mb_row_debug, mb_col_debug);
-#endif
     if (!dry_run)
       ++cpi->coef_counts_8x8       [type] [band] [pt] [x];
   } while (pt = vp8_prev_token_class[x], ++t, c < eob  &&  ++c < seg_eob);