Merge "Hybrid transform cleanups" into experimental

2 files changed, 77 insertions, 17 deletions

diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c
index bbbbc7870..e25b64305 100644
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1833,7 +1833,6 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
   else
 #endif
   if (cpi->common.txfm_mode == ALLOW_8X8
-      && mbmi->mode != I8X8_PRED
       && mbmi->mode != B_PRED) {
     mbmi->txfm_size = TX_8X8;
     cpi->t8x8_count++;
@@ -1930,7 +1929,6 @@ void vp8cx_encode_inter_macroblock (VP8_COMP *cpi, MACROBLOCK *x,
   } else
 #endif
   if (cpi->common.txfm_mode == ALLOW_8X8
-      && mbmi->mode != I8X8_PRED
       && mbmi->mode != B_PRED
       && mbmi->mode != SPLITMV) {
     mbmi->txfm_size = TX_8X8;
@@ -2111,7 +2109,6 @@ void vp8cx_encode_inter_superblock(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t,
 
   /* test code: set transform size based on mode selection */
   if (cpi->common.txfm_mode == ALLOW_8X8
-      && x->e_mbd.mode_info_context->mbmi.mode != I8X8_PRED
       && x->e_mbd.mode_info_context->mbmi.mode != B_PRED
       && x->e_mbd.mode_info_context->mbmi.mode != SPLITMV) {
     x->e_mbd.mode_info_context->mbmi.txfm_size = TX_8X8;
diff --git a/vp8/encoder/tokenize.c b/vp8/encoder/tokenize.c
index d0c315283..5b1f8ffb4 100644
--- a/vp8/encoder/tokenize.c
+++ b/vp8/encoder/tokenize.c
@@ -42,6 +42,8 @@ void vp8_stuff_mb(VP8_COMP *cpi,
                   MACROBLOCKD *xd, TOKENEXTRA **t, int dry_run);
 void vp8_stuff_mb_8x8(VP8_COMP *cpi,
                       MACROBLOCKD *xd, TOKENEXTRA **t, int dry_run);
+void vp8_stuff_mb_8x8_4x4uv(VP8_COMP *cpi,
+                            MACROBLOCKD *xd, TOKENEXTRA **t, int dry_run);
 #if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
 void vp8_stuff_mb_16x16(VP8_COMP *cpi, MACROBLOCKD *xd,
                         TOKENEXTRA **t, int dry_run);
@@ -764,6 +766,10 @@ int mb_is_skippable_8x8(MACROBLOCKD *xd) {
   return (mby_is_skippable_8x8(xd) & mbuv_is_skippable_8x8(xd));
 }
 
+int mb_is_skippable_8x8_4x4uv(MACROBLOCKD *xd) {
+  return (mby_is_skippable_8x8(xd) & mbuv_is_skippable(xd));
+}
+
 #if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
 int mby_is_skippable_16x16(MACROBLOCKD *xd) {
   int skip = 1;
@@ -822,7 +828,10 @@ void vp8_tokenize_mb(VP8_COMP *cpi,
       break;
 #endif
     case TX_8X8:
-      xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8(xd);
+      if (xd->mode_info_context->mbmi.mode == I8X8_PRED)
+        xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8_4x4uv(xd);
+      else
+        xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8(xd);
       break;
     default:
       xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable(xd, has_y2_block);
@@ -838,9 +847,12 @@ void vp8_tokenize_mb(VP8_COMP *cpi,
         vp8_stuff_mb_16x16(cpi, xd, t, dry_run);
       else
 #endif
-      if (tx_type == TX_8X8)
-        vp8_stuff_mb_8x8(cpi, xd, t, dry_run);
-      else
+      if (tx_type == TX_8X8) {
+        if (xd->mode_info_context->mbmi.mode == I8X8_PRED)
+          vp8_stuff_mb_8x8_4x4uv(cpi, xd, t, dry_run);
+        else
+          vp8_stuff_mb_8x8(cpi, xd, t, dry_run);
+      } else
         vp8_stuff_mb(cpi, xd, t, dry_run);
     } else {
       vp8_fix_contexts(xd);
@@ -895,6 +907,10 @@ void vp8_tokenize_mb(VP8_COMP *cpi,
   if (tx_type == TX_8X8) {
     ENTROPY_CONTEXT *A = (ENTROPY_CONTEXT *)xd->above_context;
     ENTROPY_CONTEXT *L = (ENTROPY_CONTEXT *)xd->left_context;
+#if CONFIG_HYBRIDTRANSFORM8X8
+    if (xd->mode_info_context->mbmi.mode == I8X8_PRED)
+      plane_type = PLANE_TYPE_Y_WITH_DC;
+#endif
     for (b = 0; b < 16; b += 4) {
       tokenize1st_order_b_8x8(xd,
                               xd->block + b, t, plane_type, xd->frame_type,
@@ -904,21 +920,34 @@ void vp8_tokenize_mb(VP8_COMP *cpi,
       *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]);
       *(L + vp8_block2left_8x8[b] + 1)  = *(L + vp8_block2left_8x8[b]);
     }
-    for (b = 16; b < 24; b += 4) {
-      tokenize1st_order_b_8x8(xd,
-                              xd->block + b, t, 2, xd->frame_type,
-                              A + vp8_block2above_8x8[b],
-                              L + vp8_block2left_8x8[b],
-                              cpi, dry_run);
-      *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]);
-      *(L + vp8_block2left_8x8[b] + 1) = *(L + vp8_block2left_8x8[b]);
+#if CONFIG_HYBRIDTRANSFORM8X8
+    if (xd->mode_info_context->mbmi.mode == I8X8_PRED) {
+        tokenize1st_order_chroma(xd, t, PLANE_TYPE_UV, cpi, dry_run);
+    } else
+#endif
+    {
+      for (b = 16; b < 24; b += 4) {
+        tokenize1st_order_b_8x8(xd,
+                                xd->block + b, t, 2, xd->frame_type,
+                                A + vp8_block2above_8x8[b],
+                                L + vp8_block2left_8x8[b],
+                                cpi, dry_run);
+        *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]);
+        *(L + vp8_block2left_8x8[b] + 1) = *(L + vp8_block2left_8x8[b]);
+      }
     }
   } else {
 #if CONFIG_HYBRIDTRANSFORM
     if(active_ht) {
       tokenize1st_order_ht(xd, t, plane_type, cpi, dry_run);
-    } else {
+    } else
+#endif
+    {
+      tokenize1st_order_b(xd, t, plane_type, cpi, dry_run);
+    }
+  }
 
+  /*
 #if CONFIG_HYBRIDTRANSFORM8X8
       if (xd->mode_info_context->mbmi.mode == I8X8_PRED) {
         ENTROPY_CONTEXT *A = (ENTROPY_CONTEXT *)xd->above_context;
@@ -946,6 +975,7 @@ void vp8_tokenize_mb(VP8_COMP *cpi,
     tokenize1st_order_b(xd, t, plane_type, cpi, dry_run);
 #endif
   }
+  */
   if (dry_run)
     *t = t_backup;
 }
@@ -1341,7 +1371,6 @@ void vp8_stuff_mb_8x8(VP8_COMP *cpi,
     *t = t_backup;
 }
 
-
 #if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
 static __inline
 void stuff1st_order_b_16x16(const BLOCKD *const b,
@@ -1493,6 +1522,40 @@ void vp8_stuff_mb(VP8_COMP *cpi, MACROBLOCKD *xd,
   if (dry_run)
     *t = t_backup;
 }
+
+void vp8_stuff_mb_8x8_4x4uv(VP8_COMP *cpi,
+                            MACROBLOCKD *xd,
+                            TOKENEXTRA **t,
+                            int dry_run) {
+  ENTROPY_CONTEXT *A = (ENTROPY_CONTEXT *)xd->above_context;
+  ENTROPY_CONTEXT *L = (ENTROPY_CONTEXT *)xd->left_context;
+  int plane_type;
+  int b;
+  TOKENEXTRA *t_backup = *t;
+
+  stuff2nd_order_b_8x8(xd->block + 24, t, 1, xd->frame_type,
+                       A + vp8_block2above_8x8[24],
+                       L + vp8_block2left_8x8[24], cpi, dry_run);
+  plane_type = 0;
+
+  for (b = 0; b < 16; b += 4) {
+    stuff1st_order_b_8x8(xd->block + b, t, plane_type, xd->frame_type,
+                         A + vp8_block2above_8x8[b],
+                         L + vp8_block2left_8x8[b],
+                         cpi, dry_run);
+    *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]);
+    *(L + vp8_block2left_8x8[b] + 1)  = *(L + vp8_block2left_8x8[b]);
+  }
+  for (b = 16; b < 24; b++)
+    stuff1st_order_buv(t,
+                       A + vp8_block2above[b],
+                       L + vp8_block2left[b],
+                       cpi, dry_run);
+
+  if (dry_run)
+    *t = t_backup;
+}
+
 void vp8_fix_contexts(MACROBLOCKD *xd) {
   /* Clear entropy contexts for Y2 blocks */
   if ((xd->mode_info_context->mbmi.mode != B_PRED