Some cleanups and fixes.

Separates the logic on transform type selection previously spread out
over a number of files into a separate function. Currently the tx_type
field in b_mode_info is not used, but still left in there to eventually
use for signaling the transform type in the bitstream.

Also, now for tx_type = DCT_DCT, the regular integer DCT is used, as
opposed to the floating point DCT used in conjuction with hybrid
transform.

Results change somewhat due to the transform change, but are within
reasonable limits. The hd/std-hd sets are slightly up, while derf/yt
are slightly down.

Change-Id: I5776840c2239ca2da31ca6cfd7fd1148dc5f9e0f

1 files changed, 34 insertions, 117 deletions

diff --git a/vp8/encoder/tokenize.c b/vp8/encoder/tokenize.c
index c72c1e7e7..d46637a3e 100644
--- a/vp8/encoder/tokenize.c
+++ b/vp8/encoder/tokenize.c
@@ -171,6 +171,7 @@ static void tokenize1st_order_b_16x16(MACROBLOCKD *xd,
 
     t->skip_eob_node = pt == 0 && ((band > 0 && type != PLANE_TYPE_Y_NO_DC) ||
                                    (band > 1 && type == PLANE_TYPE_Y_NO_DC));
+    assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
     if (!dry_run) {
 #if CONFIG_HYBRIDTRANSFORM16X16
       if (tx_type != DCT_DCT)
@@ -310,8 +311,7 @@ static void tokenize1st_order_b_8x8(MACROBLOCKD *xd,
   TOKENEXTRA *t = *tp;        /* store tokens starting here */
   const short *qcoeff_ptr = b->qcoeff;
 #if CONFIG_HYBRIDTRANSFORM8X8
-  TX_TYPE tx_type = xd->mode_info_context->mbmi.mode == I8X8_PRED ?
-      get_tx_type(xd, b) : DCT_DCT;
+  TX_TYPE tx_type = get_tx_type(xd, b);
 #endif
   const int eob = b->eob;
   int seg_eob = 64;
@@ -427,103 +427,6 @@ static void tokenize1st_order_chroma_4x4(MACROBLOCKD *xd,
   }
 }
 
-#if CONFIG_HYBRIDTRANSFORM
-static void tokenize1st_order_ht_4x4(MACROBLOCKD *xd,
-                                     TOKENEXTRA **tp,
-                                     PLANE_TYPE type,
-                                     VP8_COMP *cpi,
-                                     int dry_run) {
-  unsigned int block;
-  const BLOCKD *b = xd->block;
-  int pt;             /* near block/prev token context index */
-  TOKENEXTRA *t = *tp;/* store tokens starting here */
-  ENTROPY_CONTEXT * a;
-  ENTROPY_CONTEXT * l;
-  int const *pt_scan ;
-  int seg_eob = 16;
-  int segment_id = xd->mode_info_context->mbmi.segment_id;
-
-  if ( segfeature_active( xd, segment_id, SEG_LVL_EOB ) ) {
-    seg_eob = get_segdata( xd, segment_id, SEG_LVL_EOB );
-  }
-
-  /* Luma */
-  for (block = 0; block < 16; block++, b++) {
-    const int eob = b->eob;
-    TX_TYPE tx_type = DCT_DCT;
-    const int tmp1 = vp8_block2above[block];
-    const int tmp2 = vp8_block2left[block];
-    const int16_t *qcoeff_ptr = b->qcoeff;
-    int c = (type == PLANE_TYPE_Y_NO_DC) ? 1 : 0;
-
-    a = (ENTROPY_CONTEXT *)xd->above_context + tmp1;
-    l = (ENTROPY_CONTEXT *)xd->left_context + tmp2;
-    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
-
-    if( xd->mode_info_context->mbmi.mode == B_PRED ) {
-      tx_type = get_tx_type(xd, b);
-    }
-
-    // assign scanning order for luma components coded in intra4x4 mode
-    if ((xd->mode_info_context->mbmi.mode == B_PRED) &&
-        (type == PLANE_TYPE_Y_WITH_DC)) {
-      switch (tx_type) {
-        case ADST_DCT:
-          pt_scan = vp8_row_scan;
-          break;
-        case DCT_ADST:
-          pt_scan = vp8_col_scan;
-          break;
-        default :
-          pt_scan = vp8_default_zig_zag1d;
-          break;
-      }
-    } else {
-      pt_scan = vp8_default_zig_zag1d;
-    }
-
-    do {
-      const int band = vp8_coef_bands[c];
-      int token;
-
-      if (c < eob) {
-        const int rc = pt_scan[c];
-        const int v = qcoeff_ptr[rc];
-
-        t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
-        token    = vp8_dct_value_tokens_ptr[v].Token;
-      } else
-        token = DCT_EOB_TOKEN;
-
-      t->Token = token;
-      if (tx_type != DCT_DCT)
-        t->context_tree = cpi->common.fc.hybrid_coef_probs[type][band][pt];
-      else
-        t->context_tree = cpi->common.fc.coef_probs[type][band][pt];
-
-      t->skip_eob_node = pt == 0 && ((band > 0 && type != PLANE_TYPE_Y_NO_DC) ||
-                                     (band > 1 && type == PLANE_TYPE_Y_NO_DC));
-      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-
-      if (!dry_run) {
-        if (tx_type != DCT_DCT)
-          ++cpi->hybrid_coef_counts[type][band][pt][token];
-        else
-          ++cpi->coef_counts       [type][band][pt][token];
-      }
-      pt = vp8_prev_token_class[token];
-      ++t;
-    } while (c < eob && ++c < seg_eob);
-
-    *tp = t;
-    pt = (c != !type); /* 0 <-> all coeff data is zero */
-    *a = *l = pt;
-  }
-
-  tokenize1st_order_chroma_4x4(xd, tp, cpi, dry_run);
-}
-#endif
-
 static void tokenize1st_order_b_4x4(MACROBLOCKD *xd,
                                     TOKENEXTRA **tp,
                                     PLANE_TYPE type,
@@ -536,6 +439,7 @@ static void tokenize1st_order_b_4x4(MACROBLOCKD *xd,
   ENTROPY_CONTEXT *a, *l;
   int seg_eob = 16;
   int segment_id = xd->mode_info_context->mbmi.segment_id;
+  int const *pt_scan = vp8_default_zig_zag1d;
 
   if (segfeature_active(xd, segment_id, SEG_LVL_EOB)) {
     seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);
@@ -547,6 +451,20 @@ static void tokenize1st_order_b_4x4(MACROBLOCKD *xd,
     const int16_t *qcoeff_ptr = b->qcoeff;
     int c = (type == PLANE_TYPE_Y_NO_DC) ? 1 : 0;
 
+#if CONFIG_HYBRIDTRANSFORM
+    TX_TYPE tx_type = get_tx_type(xd, &xd->block[block]);
+    switch (tx_type) {
+      case ADST_DCT:
+        pt_scan = vp8_row_scan;
+        break;
+      case DCT_ADST:
+        pt_scan = vp8_col_scan;
+        break;
+      default :
+        pt_scan = vp8_default_zig_zag1d;
+        break;
+    }
+#endif
     a = (ENTROPY_CONTEXT *)xd->above_context + vp8_block2above[block];
     l = (ENTROPY_CONTEXT *)xd->left_context + vp8_block2left[block];
     VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
@@ -558,7 +476,7 @@ static void tokenize1st_order_b_4x4(MACROBLOCKD *xd,
       int token;
 
       if (c < eob) {
-        const int rc = vp8_default_zig_zag1d[c];
+        const int rc = pt_scan[c];
         const int v = qcoeff_ptr[rc];
 
         t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
@@ -567,13 +485,24 @@ static void tokenize1st_order_b_4x4(MACROBLOCKD *xd,
         token = DCT_EOB_TOKEN;
 
       t->Token = token;
-      t->context_tree = cpi->common.fc.coef_probs[type][band][pt];
+#if CONFIG_HYBRIDTRANSFORM
+      if (tx_type != DCT_DCT)
+        t->context_tree = cpi->common.fc.hybrid_coef_probs[type][band][pt];
+      else
+#endif
+        t->context_tree = cpi->common.fc.coef_probs[type][band][pt];
 
       t->skip_eob_node = pt == 0 && ((band > 0 && type != PLANE_TYPE_Y_NO_DC) ||
                                      (band > 1 && type == PLANE_TYPE_Y_NO_DC));
       assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-      if (!dry_run)
-        ++cpi->coef_counts[type][band][pt][token];
+      if (!dry_run) {
+#if CONFIG_HYBRIDTRANSFORM
+        if (tx_type != DCT_DCT)
+          ++cpi->hybrid_coef_counts[type][band][pt][token];
+        else
+#endif
+          ++cpi->coef_counts[type][band][pt][token];
+      }
       pt = vp8_prev_token_class[token];
       ++t;
     } while (c < eob && ++c < seg_eob);
@@ -674,12 +603,6 @@ void vp8_tokenize_mb(VP8_COMP *cpi,
   int skip_inc;
   int segment_id = xd->mode_info_context->mbmi.segment_id;
 
-#if CONFIG_HYBRIDTRANSFORM
-    int QIndex = cpi->mb.q_index;
-    int active_ht = (QIndex < ACTIVE_HT) &&
-                    (xd->mode_info_context->mbmi.mode == B_PRED);
-#endif
-
   if (!segfeature_active(xd, segment_id, SEG_LVL_EOB) ||
       (get_segdata(xd, segment_id, SEG_LVL_EOB) != 0)) {
     skip_inc = 1;
@@ -784,12 +707,7 @@ void vp8_tokenize_mb(VP8_COMP *cpi,
       }
     }
   } else {
-#if CONFIG_HYBRIDTRANSFORM
-    if (active_ht)
-      tokenize1st_order_ht_4x4(xd, t, plane_type, cpi, dry_run);
-    else
-#endif
-      tokenize1st_order_b_4x4(xd, t, plane_type, cpi, dry_run);
+    tokenize1st_order_b_4x4(xd, t, plane_type, cpi, dry_run);
   }
   if (dry_run)
     *t = t_backup;
@@ -1078,8 +996,7 @@ static __inline void stuff1st_order_b_8x8(MACROBLOCKD *xd,
   int pt; /* near block/prev token context index */
   TOKENEXTRA *t = *tp;        /* store tokens starting here */
 #if CONFIG_HYBRIDTRANSFORM8X8
-  TX_TYPE tx_type = xd->mode_info_context->mbmi.mode == I8X8_PRED ?
-      get_tx_type(xd, b) : DCT_DCT;
+  TX_TYPE tx_type = get_tx_type(xd, b);
 #endif
   const int band = vp8_coef_bands_8x8[(type == PLANE_TYPE_Y_NO_DC) ? 1 : 0];
   VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);