Use transform-size selection for i8x8_pred also.

Change-Id: Iecb282fc89f9b5145ef31c5eda294ad42bc32a5d

7 files changed, 221 insertions, 99 deletions

diff --git a/vp8/decoder/decodemv.c b/vp8/decoder/decodemv.c
index 8a231b220..fa15af9e9 100644
--- a/vp8/decoder/decodemv.c
+++ b/vp8/decoder/decodemv.c
@@ -172,10 +172,10 @@ static void vp8_kfread_modes(VP8D_COMP *pbi,
 
 #if CONFIG_TX_SELECT
   if (cm->txfm_mode == TX_MODE_SELECT && m->mbmi.mb_skip_coeff == 0 &&
-      m->mbmi.mode <= TM_PRED) {
+      m->mbmi.mode <= I8X8_PRED) {
     // FIXME(rbultje) code ternary symbol once all experiments are merged
     m->mbmi.txfm_size = vp8_read(bc, cm->prob_tx[0]);
-    if (m->mbmi.txfm_size != TX_4X4)
+    if (m->mbmi.txfm_size != TX_4X4 && m->mbmi.mode != I8X8_PRED)
       m->mbmi.txfm_size += vp8_read(bc, cm->prob_tx[1]);
   } else
 #endif
@@ -1281,11 +1281,11 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi,
 
 #if CONFIG_TX_SELECT
   if (cm->txfm_mode == TX_MODE_SELECT && mbmi->mb_skip_coeff == 0 &&
-      ((mbmi->ref_frame == INTRA_FRAME && mbmi->mode <= TM_PRED) ||
+      ((mbmi->ref_frame == INTRA_FRAME && mbmi->mode <= I8X8_PRED) ||
        (mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) {
     // FIXME(rbultje) code ternary symbol once all experiments are merged
     mbmi->txfm_size = vp8_read(bc, cm->prob_tx[0]);
-    if (mbmi->txfm_size != TX_4X4)
+    if (mbmi->txfm_size != TX_4X4 && mbmi->mode != I8X8_PRED)
       mbmi->txfm_size += vp8_read(bc, cm->prob_tx[1]);
   } else
 #endif
diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c
index b2aca859b..3fa6660aa 100644
--- a/vp8/decoder/decodframe.c
+++ b/vp8/decoder/decodframe.c
@@ -364,10 +364,8 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
   if (mode == I8X8_PRED) {
     for (i = 0; i < 4; i++) {
       int ib = vp8_i8x8_block[i];
-#if !CONFIG_HYBRIDTRANSFORM8X8
       const int iblock[4] = {0, 1, 4, 5};
       int j;
-#endif
       int i8x8mode;
       BLOCKD *b;
 
@@ -381,30 +379,40 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
 
       b = &xd->block[ib];
       i8x8mode = b->bmi.as_mode.first;
-      RECON_INVOKE(RTCD_VTABLE(recon), intra8x8_predict)
-      (b, i8x8mode, b->predictor);
+      RECON_INVOKE(RTCD_VTABLE(recon), intra8x8_predict)(b, i8x8mode,
+                                                         b->predictor);
 
+      if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
 #if CONFIG_HYBRIDTRANSFORM8X8
-      vp8_ht_dequant_idct_add_8x8_c(b->bmi.as_mode.tx_type,
-                                    q, dq, pre, dst, 16, stride);
-      q += 64;
+        vp8_ht_dequant_idct_add_8x8_c(b->bmi.as_mode.tx_type,
+                                      q, dq, pre, dst, 16, stride);
+        q += 64;
 #else
-      vp8_dequant_idct_add_8x8_c(q, dq, pre, dst, 16, stride);
-      q += 64;
+        vp8_dequant_idct_add_8x8_c(q, dq, pre, dst, 16, stride);
+        q += 64;
 #endif
+      } else {
+        for (j = 0; j < 4; j++) {
+          b = &xd->block[ib + iblock[j]];
+          vp8_dequant_idct_add_c(b->qcoeff, b->dequant, b->predictor,
+                                 *(b->base_dst) + b->dst, 16, b->dst_stride);
+        }
+      }
 
       b = &xd->block[16 + i];
-      RECON_INVOKE(RTCD_VTABLE(recon), intra_uv4x4_predict)
-      (b, i8x8mode, b->predictor);
-      DEQUANT_INVOKE(&pbi->dequant, idct_add)
-      (b->qcoeff, b->dequant,  b->predictor,
-       *(b->base_dst) + b->dst, 8, b->dst_stride);
+      RECON_INVOKE(RTCD_VTABLE(recon), intra_uv4x4_predict)(b, i8x8mode,
+                                                            b->predictor);
+      DEQUANT_INVOKE(&pbi->dequant, idct_add)(b->qcoeff, b->dequant,
+                                              b->predictor,
+                                              *(b->base_dst) + b->dst, 8,
+                                              b->dst_stride);
       b = &xd->block[20 + i];
-      RECON_INVOKE(RTCD_VTABLE(recon), intra_uv4x4_predict)
-      (b, i8x8mode, b->predictor);
-      DEQUANT_INVOKE(&pbi->dequant, idct_add)
-      (b->qcoeff, b->dequant,  b->predictor,
-       *(b->base_dst) + b->dst, 8, b->dst_stride);
+      RECON_INVOKE(RTCD_VTABLE(recon), intra_uv4x4_predict)(b, i8x8mode,
+                                                            b->predictor);
+      DEQUANT_INVOKE(&pbi->dequant, idct_add)(b->qcoeff, b->dequant,
+                                              b->predictor,
+                                              *(b->base_dst) + b->dst, 8,
+                                              b->dst_stride);
     }
   } else if (mode == B_PRED) {
     for (i = 0; i < 16; i++) {
diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c
index f58fd7b73..812565915 100644
--- a/vp8/encoder/bitstream.c
+++ b/vp8/encoder/bitstream.c
@@ -239,6 +239,18 @@ static void update_mbintra_mode_probs(VP8_COMP *cpi) {
   }
 }
 
+static __inline int get_prob(int num, int den) {
+  int p;
+  if (den <= 0)
+    return 128;
+  p = (num * 255 + (den >> 1)) / den;
+  if (p > 255)
+    return 255;
+  else if (p < 1)
+    return 1;
+  return p;
+}
+
 void update_skip_probs(VP8_COMP *cpi) {
   VP8_COMMON *const pc = & cpi->common;
   int prob_skip_false[3] = {0, 0, 0};
@@ -1289,7 +1301,7 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) {
         }
 
 #if CONFIG_TX_SELECT
-        if (((rf == INTRA_FRAME && mode <= TM_PRED) ||
+        if (((rf == INTRA_FRAME && mode <= I8X8_PRED) ||
              (rf != INTRA_FRAME && mode != SPLITMV)) &&
             pc->txfm_mode == TX_MODE_SELECT &&
             !((pc->mb_no_coeff_skip && mi->mb_skip_coeff) ||
@@ -1298,7 +1310,7 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) {
           TX_SIZE sz = mi->txfm_size;
           // FIXME(rbultje) code ternary symbol once all experiments are merged
           vp8_write(w, sz != TX_4X4, pc->prob_tx[0]);
-          if (sz != TX_4X4)
+          if (sz != TX_4X4 && mode != I8X8_PRED)
             vp8_write(w, sz != TX_8X8, pc->prob_tx[1]);
         }
 #endif
@@ -1467,14 +1479,14 @@ static void write_kfmodes(VP8_COMP *cpi) {
           write_uv_mode(bc, m->mbmi.uv_mode, c->kf_uv_mode_prob[ym]);
 
 #if CONFIG_TX_SELECT
-        if (ym <= TM_PRED && c->txfm_mode == TX_MODE_SELECT &&
+        if (ym <= I8X8_PRED && c->txfm_mode == TX_MODE_SELECT &&
             !((c->mb_no_coeff_skip && m->mbmi.mb_skip_coeff) ||
               (segfeature_active(xd, segment_id, SEG_LVL_EOB) &&
                get_segdata(xd, segment_id, SEG_LVL_EOB) == 0))) {
           TX_SIZE sz = m->mbmi.txfm_size;
           // FIXME(rbultje) code ternary symbol once all experiments are merged
           vp8_write(bc, sz != TX_4X4, c->prob_tx[0]);
-          if (sz != TX_4X4)
+          if (sz != TX_4X4 && ym <= TM_PRED)
             vp8_write(bc, sz != TX_8X8, c->prob_tx[1]);
         }
 #endif
@@ -2626,32 +2638,13 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size)
 
 #if CONFIG_TX_SELECT
   {
-    int cnt = cpi->txfm_count[0] + cpi->txfm_count[1] + cpi->txfm_count[2];
-    if (cnt && pc->txfm_mode == TX_MODE_SELECT) {
-      int prob = (255 * (cpi->txfm_count[1] + cpi->txfm_count[2]) + (cnt >> 1)) / cnt;
-      if (prob <= 1) {
-        pc->prob_tx[0] = 1;
-      } else if (prob >= 255) {
-        pc->prob_tx[0] = 255;
-      } else {
-        pc->prob_tx[0] = prob;
-      }
-      pc->prob_tx[0] = 256 - pc->prob_tx[0];
+    if (pc->txfm_mode == TX_MODE_SELECT) {
+      pc->prob_tx[0] = get_prob(cpi->txfm_count[0] + cpi->txfm_count_8x8p[0],
+                                cpi->txfm_count[0] + cpi->txfm_count[1] + cpi->txfm_count[2] +
+                                cpi->txfm_count_8x8p[0] + cpi->txfm_count_8x8p[1]);
+      pc->prob_tx[1] = get_prob(cpi->txfm_count[1], cpi->txfm_count[1] + cpi->txfm_count[2]);
     } else {
       pc->prob_tx[0] = 128;
-    }
-    cnt -= cpi->txfm_count[0];
-    if (cnt && pc->txfm_mode == TX_MODE_SELECT) {
-      int prob = (255 * cpi->txfm_count[2] + (cnt >> 1)) / cnt;
-      if (prob <= 1) {
-        pc->prob_tx[1] = 1;
-      } else if (prob >= 255) {
-        pc->prob_tx[1] = 255;
-      } else {
-        pc->prob_tx[1] = prob;
-      }
-      pc->prob_tx[1] = 256 - pc->prob_tx[1];
-    } else {
       pc->prob_tx[1] = 128;
     }
     vp8_write_literal(bc, pc->txfm_mode, 2);
diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c
index c951ac8a6..1606a19ba 100644
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1396,6 +1396,7 @@ static void encode_frame_internal(VP8_COMP *cpi) {
   vpx_memset(cpi->comp_pred_count, 0, sizeof(cpi->comp_pred_count));
 #if CONFIG_TX_SELECT
   vpx_memset(cpi->txfm_count, 0, sizeof(cpi->txfm_count));
+  vpx_memset(cpi->txfm_count_8x8p, 0, sizeof(cpi->txfm_count_8x8p));
   vpx_memset(cpi->rd_tx_select_diff, 0, sizeof(cpi->rd_tx_select_diff));
 #endif
   {
@@ -1598,13 +1599,14 @@ void vp8_encode_frame(VP8_COMP *cpi) {
 
 #if CONFIG_TX_SELECT
     if (cpi->common.txfm_mode == TX_MODE_SELECT) {
-      const int count4x4 = cpi->txfm_count[TX_4X4];
+      const int count4x4 = cpi->txfm_count[TX_4X4] + cpi->txfm_count_8x8p[TX_4X4];
       const int count8x8 = cpi->txfm_count[TX_8X8];
+      const int count8x8_8x8p = cpi->txfm_count_8x8p[TX_8X8];
       const int count16x16 = cpi->txfm_count[TX_16X16];
 
       if (count4x4 == 0 && count16x16 == 0) {
         cpi->common.txfm_mode = ALLOW_8X8;
-      } else if (count8x8 == 0 && count16x16 == 0) {
+      } else if (count8x8 == 0 && count16x16 == 0 && count8x8_8x8p == 0) {
         cpi->common.txfm_mode = ONLY_4X4;
       } else if (count8x8 == 0 && count4x4 == 0) {
         cpi->common.txfm_mode = ALLOW_16X16;
@@ -1946,6 +1948,8 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
            get_segdata(&x->e_mbd, segment_id, SEG_LVL_EOB) == 0))) {
       if (mbmi->mode != B_PRED && mbmi->mode != I8X8_PRED) {
         cpi->txfm_count[mbmi->txfm_size]++;
+      } else if (mbmi->mode == I8X8_PRED) {
+        cpi->txfm_count_8x8p[mbmi->txfm_size]++;
       }
     } else
 #endif
@@ -2138,6 +2142,8 @@ void vp8cx_encode_inter_macroblock (VP8_COMP *cpi, MACROBLOCK *x,
       if (mbmi->mode != B_PRED && mbmi->mode != I8X8_PRED &&
           mbmi->mode != SPLITMV) {
         cpi->txfm_count[mbmi->txfm_size]++;
+      } else if (mbmi->mode == I8X8_PRED) {
+        cpi->txfm_count_8x8p[mbmi->txfm_size]++;
       }
     } else
 #endif
diff --git a/vp8/encoder/encodeintra.c b/vp8/encoder/encodeintra.c
index eee8efdd2..323df005b 100644
--- a/vp8/encoder/encodeintra.c
+++ b/vp8/encoder/encodeintra.c
@@ -238,7 +238,8 @@ void vp8_encode_intra16x16mbuv(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
 
 void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
                          MACROBLOCK *x, int ib) {
-  BLOCKD *b = &x->e_mbd.block[ib];
+  MACROBLOCKD *xd = &x->e_mbd;
+  BLOCKD *b = &xd->block[ib];
   BLOCK *be = &x->block[ib];
   const int iblock[4] = {0, 1, 4, 5};
   int i;
@@ -255,8 +256,7 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
   }
 #endif
 
-  {
-    MACROBLOCKD *xd = &x->e_mbd;
+  if (x->e_mbd.mode_info_context->mbmi.txfm_size == TX_8X8) {
     int idx = (ib & 0x02) ? (ib + 2) : ib;
 
     // generate residual blocks
@@ -274,14 +274,23 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
     x->quantize_b_8x8(x->block + idx, xd->block + idx);
     vp8_idct_idct8(xd->block[idx].dqcoeff, xd->block[ib].diff, 32);
 #endif
-
-    // reconstruct submacroblock
+  } else {
     for (i = 0; i < 4; i++) {
       b = &xd->block[ib + iblock[i]];
-      vp8_recon_b_c(b->predictor, b->diff, *(b->base_dst) + b->dst,
-                    b->dst_stride);
+      be = &x->block[ib + iblock[i]];
+      ENCODEMB_INVOKE(&rtcd->encodemb, subb)(be, b, 16);
+      x->vp8_short_fdct4x4(be->src_diff, be->coeff, 32);
+      x->quantize_b(be, b);
+      vp8_inverse_transform_b(IF_RTCD(&rtcd->common->idct), b, 32);
     }
   }
+
+  // reconstruct submacroblock
+  for (i = 0; i < 4; i++) {
+    b = &xd->block[ib + iblock[i]];
+    vp8_recon_b_c(b->predictor, b->diff, *(b->base_dst) + b->dst,
+                  b->dst_stride);
+  }
 }
 
 void vp8_encode_intra8x8mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
diff --git a/vp8/encoder/onyx_int.h b/vp8/encoder/onyx_int.h
index fe029217b..696da14ed 100644
--- a/vp8/encoder/onyx_int.h
+++ b/vp8/encoder/onyx_int.h
@@ -470,7 +470,8 @@ typedef struct VP8_COMP {
   int single_pred_count[COMP_PRED_CONTEXTS];
 #if CONFIG_TX_SELECT
   // FIXME contextualize
-  int txfm_count[TX_SIZE_MAX + 1];
+  int txfm_count[TX_SIZE_MAX];
+  int txfm_count_8x8p[TX_SIZE_MAX - 1];
   int64_t rd_tx_select_diff[NB_TXFM_MODES];
   int rd_tx_select_threshes[4][NB_TXFM_MODES];
 #endif
diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c
index dc6f050c3..64824f01f 100644
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -1565,30 +1565,63 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
 
       vp8_subtract_4b_c(be, b, 16);
 
+      if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
 #if CONFIG_HYBRIDTRANSFORM8X8
-      txfm_map(b, pred_mode_conv(mode));
-      vp8_fht_c(be->src_diff, (x->block + idx)->coeff, 32,
-                b->bmi.as_mode.tx_type, 8);
+        txfm_map(b, pred_mode_conv(mode));
+        vp8_fht_c(be->src_diff, (x->block + idx)->coeff, 32,
+                  b->bmi.as_mode.tx_type, 8);
 
 #else
-      x->vp8_short_fdct8x8(be->src_diff, (x->block + idx)->coeff, 32);
+        x->vp8_short_fdct8x8(be->src_diff, (x->block + idx)->coeff, 32);
 #endif
+        x->quantize_b_8x8(x->block + idx, xd->block + idx);
 
-      x->quantize_b_8x8(x->block + idx, xd->block + idx);
+        // compute quantization mse of 8x8 block
+        distortion = vp8_block_error_c((x->block + idx)->coeff,
+                                       (xd->block + idx)->dqcoeff, 64);
+        ta0 = *(a + vp8_block2above_8x8[idx]);
+        tl0 = *(l + vp8_block2left_8x8 [idx]);
 
-      // compute quantization mse of 8x8 block
-      distortion = vp8_block_error_c((x->block + idx)->coeff,
-                                     (xd->block + idx)->dqcoeff, 64)>>2;
-      ta0 = *(a + vp8_block2above_8x8[idx]);
-      tl0 = *(l + vp8_block2left_8x8 [idx]);
+        rate_t = cost_coeffs(x, xd->block + idx, PLANE_TYPE_Y_WITH_DC,
+                             &ta0, &tl0, TX_8X8);
 
-      rate_t = cost_coeffs(x, xd->block + idx, PLANE_TYPE_Y_WITH_DC,
-                           &ta0, &tl0, TX_8X8);
-
-      rate += rate_t;
-      ta1 = ta0;
-      tl1 = tl0;
+        rate += rate_t;
+        ta1 = ta0;
+        tl1 = tl0;
+      } else {
+        x->vp8_short_fdct8x4(be->src_diff, be->coeff, 32);
+        x->vp8_short_fdct8x4((be + 4)->src_diff, (be + 4)->coeff, 32);
+
+        x->quantize_b_pair(x->block + ib, x->block + ib + 1,
+                           xd->block + ib, xd->block + ib + 1);
+        x->quantize_b_pair(x->block + ib + 4, x->block + ib + 5,
+                           xd->block + ib + 4, xd->block + ib + 5);
+
+        distortion = vp8_block_error_c((x->block + ib)->coeff,
+                                       (xd->block + ib)->dqcoeff, 16);
+        distortion += vp8_block_error_c((x->block + ib + 1)->coeff,
+                                        (xd->block + ib + 1)->dqcoeff, 16);
+        distortion += vp8_block_error_c((x->block + ib + 4)->coeff,
+                                        (xd->block + ib + 4)->dqcoeff, 16);
+        distortion += vp8_block_error_c((x->block + ib + 5)->coeff,
+                                        (xd->block + ib + 5)->dqcoeff, 16);
+
+        ta0 = *(a + vp8_block2above[ib]);
+        ta1 = *(a + vp8_block2above[ib + 1]);
+        tl0 = *(l + vp8_block2above[ib]);
+        tl1 = *(l + vp8_block2above[ib + 4]);
+        rate_t = cost_coeffs(x, xd->block + ib, PLANE_TYPE_Y_WITH_DC,
+                             &ta0, &tl0, TX_4X4);
+        rate_t += cost_coeffs(x, xd->block + ib + 1, PLANE_TYPE_Y_WITH_DC,
+                              &ta1, &tl0, TX_4X4);
+        rate_t += cost_coeffs(x, xd->block + ib + 4, PLANE_TYPE_Y_WITH_DC,
+                              &ta0, &tl1, TX_4X4);
+        rate_t += cost_coeffs(x, xd->block + ib + 5, PLANE_TYPE_Y_WITH_DC,
+                              &ta1, &tl1, TX_4X4);
+        rate += rate_t;
+      }
 
+      distortion >>= 2;
       this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
       if (this_rd < best_rd) {
         *bestrate = rate;
@@ -1617,17 +1650,18 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
 #endif
   vp8_encode_intra8x8(IF_RTCD(&cpi->rtcd), x, ib);
 
-#if CONFIG_HYBRIDTRANSFORM8X8
-  *(a + vp8_block2above_8x8[idx])     = besta0;
-  *(a + vp8_block2above_8x8[idx] + 1) = besta1;
-  *(l + vp8_block2left_8x8 [idx])     = bestl0;
-  *(l + vp8_block2left_8x8 [idx] + 1) = bestl1;
-#else
-  *(a + vp8_block2above[ib])   = besta0;
-  *(a + vp8_block2above[ib + 1]) = besta1;
-  *(l + vp8_block2above[ib])   = bestl0;
-  *(l + vp8_block2above[ib + 4]) = bestl1;
-#endif
+  if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
+    *(a + vp8_block2above_8x8[idx])     = besta0;
+    *(a + vp8_block2above_8x8[idx] + 1) = besta1;
+    *(l + vp8_block2left_8x8 [idx])     = bestl0;
+    *(l + vp8_block2left_8x8 [idx] + 1) = bestl1;
+  } else {
+    *(a + vp8_block2above[ib])     = besta0;
+    *(a + vp8_block2above[ib + 1]) = besta1;
+    *(l + vp8_block2above[ib])     = bestl0;
+    *(l + vp8_block2above[ib + 4]) = bestl1;
+  }
+
   return best_rd;
 }
 
@@ -3395,7 +3429,7 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
     int other_cost = 0;
     int compmode_cost = 0;
     int mode_excluded = 0;
-    int64_t txfm_cache[NB_TXFM_MODES];
+    int64_t txfm_cache[NB_TXFM_MODES] = { 0 };
 
     // These variables hold are rolling total cost and distortion for this mode
     rate2 = 0;
@@ -3579,13 +3613,16 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
         }
         break;
         case I8X8_PRED: {
-          int64_t tmp_rd;
-          mbmi->txfm_size = TX_8X8; // FIXME wrong in case of hybridtransform8x8
-          tmp_rd = rd_pick_intra8x8mby_modes(cpi, x, &rate, &rate_y,
-                                             &distortion, best_yrd);
-          rate2 += rate;
-          distortion2 += distortion;
-
+#if CONFIG_TX_SELECT
+          int cost0 = vp8_cost_bit(cm->prob_tx[0], 0);
+          int cost1 = vp8_cost_bit(cm->prob_tx[0], 1);
+          int64_t tmp_rd_4x4s, tmp_rd_8x8s;
+#endif
+          int64_t tmp_rd_4x4, tmp_rd_8x8, tmp_rd;
+          int r4x4, tok4x4, d4x4, r8x8, tok8x8, d8x8;
+          mbmi->txfm_size = TX_4X4;
+          tmp_rd_4x4 = rd_pick_intra8x8mby_modes(cpi, x, &r4x4, &tok4x4,
+                                                 &d4x4, best_yrd);
           mode8x8[0][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.first;
           mode8x8[0][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.first;
           mode8x8[0][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.first;
@@ -3596,6 +3633,72 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
           mode8x8[1][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.second;
           mode8x8[1][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.second;
 #endif
+          mbmi->txfm_size = TX_8X8;
+          tmp_rd_8x8 = rd_pick_intra8x8mby_modes(cpi, x, &r8x8, &tok8x8,
+                                                 &d8x8, best_yrd);
+          txfm_cache[ONLY_4X4]  = tmp_rd_4x4;
+          txfm_cache[ALLOW_8X8] = tmp_rd_8x8;
+#if CONFIG_TX16X16
+          txfm_cache[ALLOW_16X16] = tmp_rd_8x8;
+#endif
+#if CONFIG_TX_SELECT
+          tmp_rd_4x4s = tmp_rd_4x4 + RDCOST(x->rdmult, x->rddiv, cost0, 0);
+          tmp_rd_8x8s = tmp_rd_8x8 + RDCOST(x->rdmult, x->rddiv, cost1, 0);
+          txfm_cache[TX_MODE_SELECT] = tmp_rd_4x4s < tmp_rd_8x8s ? tmp_rd_4x4s : tmp_rd_8x8s;
+          if (cm->txfm_mode == TX_MODE_SELECT) {
+            if (tmp_rd_4x4s < tmp_rd_8x8s) {
+              rate = r4x4 + cost0;
+              rate_y = tok4x4 + cost0;
+              distortion = d4x4;
+              mbmi->txfm_size = TX_4X4;
+              tmp_rd = tmp_rd_4x4s;
+            } else {
+              rate = r8x8 + cost1;
+              rate_y = tok8x8 + cost1;
+              distortion = d8x8;
+              mbmi->txfm_size = TX_8X8;
+              tmp_rd = tmp_rd_8x8s;
+
+              mode8x8[0][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.first;
+              mode8x8[0][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.first;
+              mode8x8[0][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.first;
+              mode8x8[0][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.first;
+#if CONFIG_COMP_INTRA_PRED
+              mode8x8[1][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.second;
+              mode8x8[1][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.second;
+              mode8x8[1][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.second;
+              mode8x8[1][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.second;
+#endif
+            }
+          } else
+#endif
+          if (cm->txfm_mode == ONLY_4X4) {
+            rate = r4x4;
+            rate_y = tok4x4;
+            distortion = d4x4;
+            mbmi->txfm_size = TX_4X4;
+            tmp_rd = tmp_rd_4x4;
+          } else {
+            rate = r8x8;
+            rate_y = tok8x8;
+            distortion = d8x8;
+            mbmi->txfm_size = TX_8X8;
+            tmp_rd = tmp_rd_8x8;
+
+            mode8x8[0][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.first;
+            mode8x8[0][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.first;
+            mode8x8[0][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.first;
+            mode8x8[0][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.first;
+#if CONFIG_COMP_INTRA_PRED
+            mode8x8[1][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.second;
+            mode8x8[1][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.second;
+            mode8x8[1][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.second;
+            mode8x8[1][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.second;
+#endif
+          }
+
+          rate2 += rate;
+          distortion2 += distortion;
 
           /* TODO: uv rate maybe over-estimated here since there is UV intra
                    mode coded in I8X8_PRED prediction */
@@ -4026,8 +4129,7 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
       if (!mode_excluded && this_rd != INT64_MAX) {
         for (i = 0; i < NB_TXFM_MODES; i++) {
           int64_t adj_rd;
-          if (this_mode != B_PRED && this_mode != I8X8_PRED &&
-              this_mode != SPLITMV) {
+          if (this_mode != B_PRED && this_mode != SPLITMV) {
             adj_rd = this_rd + txfm_cache[i] - txfm_cache[cm->txfm_mode];
           } else {
             adj_rd = this_rd;
@@ -4264,6 +4366,8 @@ void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x,
   mbmi->mode_rdopt = I8X8_PRED;
 #endif
 
+  // FIXME(rbultje) support transform-size selection
+  mbmi->txfm_size = (cm->txfm_mode == ONLY_4X4) ? TX_4X4 : TX_8X8;
   error8x8 = rd_pick_intra8x8mby_modes(cpi, x, &rate8x8, &rate8x8_tokenonly,
                                        &dist8x8, error16x16);
   mode8x8[0][0]= xd->mode_info_context->bmi[0].as_mode.first;
@@ -4365,8 +4469,9 @@ void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x,
              sizeof(x->mb_context[xd->mb_index].txfm_rd_diff));
 #endif
     } else {
+      // FIXME(rbultje) support transform-size selection
       mbmi->mode = I8X8_PRED;
-      mbmi->txfm_size = TX_8X8;
+      mbmi->txfm_size = (cm->txfm_mode == ONLY_4X4) ? TX_4X4 : TX_8X8;
       set_i8x8_block_modes(x, mode8x8);
       rate = rate8x8 + rateuv;
       dist = dist8x8 + (distuv >> 2);