Merge duplicate loops in tokenization code.

Also merge the three occurrences of 4x4 chroma block writing into a
single function, and call that function instead of duplicating the
4x4 chroma tokenization code in 3 places.

Change-Id: I7913538d1029f709b0e3ae49fff1148d3be9eeb9

1 files changed, 176 insertions, 380 deletions

diff --git a/vp8/encoder/tokenize.c b/vp8/encoder/tokenize.c
index 891dec606..4cfa3ecfc 100644
--- a/vp8/encoder/tokenize.c
+++ b/vp8/encoder/tokenize.c
@@ -141,7 +141,6 @@ static void tokenize1st_order_b_16x16(MACROBLOCKD *xd,
   int c = 0;                  /* start at DC unless type 0 */
   const int eob = b->eob;     /* one beyond last nonzero coeff */
   TOKENEXTRA *t = *tp;        /* store tokens starting here */
-  int x;
   const short *qcoeff_ptr = b->qcoeff;
 #if CONFIG_HYBRIDTRANSFORM16X16
   TX_TYPE tx_type = get_tx_type(xd, b);
@@ -159,17 +158,18 @@ static void tokenize1st_order_b_16x16(MACROBLOCKD *xd,
 
   do {
     const int band = vp8_coef_bands_16x16[c];
-    int v;
+    int x;
 
-    x = DCT_EOB_TOKEN;
     if (c < eob) {
-      int rc = vp8_default_zig_zag1d_16x16[c];
-      v = qcoeff_ptr[rc];
+      const int rc = vp8_default_zig_zag1d_16x16[c];
+      const int v = qcoeff_ptr[rc];
 
       assert(-DCT_MAX_VALUE <= v  &&  v < (DCT_MAX_VALUE));
 
       t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
       x        = vp8_dct_value_tokens_ptr[v].Token;
+    } else {
+      x = DCT_EOB_TOKEN;
     }
 
     t->Token = x;
@@ -189,29 +189,28 @@ static void tokenize1st_order_b_16x16(MACROBLOCKD *xd,
 #endif
         ++cpi->coef_counts_16x16[type][band][pt][x];
     }
-  } while (pt = vp8_prev_token_class[x], ++t, c < eob  &&  ++c < seg_eob);
+    pt = vp8_prev_token_class[x];
+    ++t;
+  } while (c < eob  &&  ++c < seg_eob);
 
   *tp = t;
   pt = (c != !type); /* 0 <-> all coeff data is zero */
   *a = *l = pt;
 }
 
-static void tokenize2nd_order_b_8x8
-(
-  MACROBLOCKD *xd,
-  const BLOCKD *const b,
-  TOKENEXTRA **tp,
-  const int type,     /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
-  const FRAME_TYPE frametype,
-  ENTROPY_CONTEXT *a,
-  ENTROPY_CONTEXT *l,
-  VP8_COMP *cpi,
-  int dry_run) {
+static void tokenize2nd_order_b_8x8(MACROBLOCKD *xd,
+                                    const BLOCKD *const b,
+                                    TOKENEXTRA **tp,
+                                    const int type,     /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
+                                    const FRAME_TYPE frametype,
+                                    ENTROPY_CONTEXT *a,
+                                    ENTROPY_CONTEXT *l,
+                                    VP8_COMP *cpi,
+                                    int dry_run) {
   int pt; /* near block/prev token context index */
   int c = 0;          /* start at DC */
   const int eob = b->eob;     /* one beyond last nonzero coeff */
   TOKENEXTRA *t = *tp;        /* store tokens starting here */
-  int x;
   const short *qcoeff_ptr = b->qcoeff;
 
   int seg_eob = 4;
@@ -227,18 +226,19 @@ static void tokenize2nd_order_b_8x8
 
   do {
     const int band = vp8_coef_bands[c];
-    int v = 0;
+    int x;
 
     if (c < eob) {
-      int rc = vp8_default_zig_zag1d[c];
-      v = qcoeff_ptr[rc];
+      const int rc = vp8_default_zig_zag1d[c];
+      const int v = qcoeff_ptr[rc];
 
       assert(-DCT_MAX_VALUE <= v  &&  v < (DCT_MAX_VALUE));
 
       t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
       x        = vp8_dct_value_tokens_ptr[v].Token;
-    } else
+    } else {
       x = DCT_EOB_TOKEN;
+    }
 
     t->Token = x;
     // printf("Token : %d\n", x);
@@ -249,13 +249,13 @@ static void tokenize2nd_order_b_8x8
 
     if (!dry_run)
       ++cpi->coef_counts_8x8       [type] [band] [pt] [x];
-  } while (pt = vp8_prev_token_class[x], ++t, c < eob  &&  ++c < seg_eob);
-
+    pt = vp8_prev_token_class[x];
+    ++t;
+  } while (c < eob && ++c < seg_eob);
 
   *tp = t;
   pt = (c != !type); /* 0 <-> all coeff data is zero */
   *a = *l = pt;
-
 }
 
 static void tokenize2nd_order_b_4x4(MACROBLOCKD *xd,
@@ -263,34 +263,36 @@ static void tokenize2nd_order_b_4x4(MACROBLOCKD *xd,
                                     VP8_COMP *cpi,
                                     int dry_run) {
   int pt;             /* near block/prev token context index */
-  int c;              /* start at DC */
+  int c = 0;          /* start at DC */
   TOKENEXTRA *t = *tp;/* store tokens starting here */
-  const BLOCKD *b;
-  const short *qcoeff_ptr;
+  const BLOCKD *b = xd->block + 24;
+  const short *qcoeff_ptr = b->qcoeff;
   ENTROPY_CONTEXT *a;
   ENTROPY_CONTEXT *l;
-  int band, rc, v, token;
-
+  const int eob = b->eob;
   int seg_eob = 16;
   int segment_id = xd->mode_info_context->mbmi.segment_id;
 
   if (segfeature_active(xd, segment_id, SEG_LVL_EOB))
     seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);
 
-  b = xd->block + 24;
-  qcoeff_ptr = b->qcoeff;
   a = (ENTROPY_CONTEXT *)xd->above_context + 8;
   l = (ENTROPY_CONTEXT *)xd->left_context + 8;
 
   VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
 
-  for (c = 0; c < b->eob; c++) {
-    rc = vp8_default_zig_zag1d[c];
-    band = vp8_coef_bands[c];
-    v = qcoeff_ptr[rc];
+  do {
+    const int band = vp8_coef_bands[c];
+    int token;
+
+    if (c < eob) {
+      const int rc = vp8_default_zig_zag1d[c];
+      const int v = qcoeff_ptr[rc];
 
-    t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
-    token    = vp8_dct_value_tokens_ptr[v].Token;
+      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
+      token    = vp8_dct_value_tokens_ptr[v].Token;
+    } else
+      token    = DCT_EOB_TOKEN;
 
     t->Token = token;
     t->context_tree = cpi->common.fc.coef_probs [1] [band] [pt];
@@ -300,43 +302,24 @@ static void tokenize2nd_order_b_4x4(MACROBLOCKD *xd,
 
     if (!dry_run)
       ++cpi->coef_counts       [1] [band] [pt] [token];
-
     pt = vp8_prev_token_class[token];
-    t++;
-  }
-
-  if (c < seg_eob) {
-    band = vp8_coef_bands[c];
-    t->Token = DCT_EOB_TOKEN;
-    t->context_tree = cpi->common.fc.coef_probs [1] [band] [pt];
-
-    t->skip_eob_node = ((pt == 0) && (band > 0));
-    assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-
-    if (!dry_run)
-      ++cpi->coef_counts       [1] [band] [pt] [DCT_EOB_TOKEN];
-
-    t++;
-  }
-
+    ++t;
+  } while (c < eob && ++c < seg_eob);
 
   *tp = t;
   pt = (c != 0); /* 0 <-> all coeff data is zero */
   *a = *l = pt;
-
 }
 
-static void tokenize1st_order_b_8x8
-(
-  MACROBLOCKD *xd,
-  const BLOCKD *const b,
-  TOKENEXTRA **tp,
-  const int type,     /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
-  const FRAME_TYPE frametype,
-  ENTROPY_CONTEXT *a,
-  ENTROPY_CONTEXT *l,
-  VP8_COMP *cpi,
-  int dry_run) {
+static void tokenize1st_order_b_8x8(MACROBLOCKD *xd,
+                                    const BLOCKD *const b,
+                                    TOKENEXTRA **tp,
+                                    const int type,     /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
+                                    const FRAME_TYPE frametype,
+                                    ENTROPY_CONTEXT *a,
+                                    ENTROPY_CONTEXT *l,
+                                    VP8_COMP *cpi,
+                                    int dry_run) {
   int pt; /* near block/prev token context index */
   int c = type ? 0 : 1;       /* start at DC unless type 0 */
   TOKENEXTRA *t = *tp;        /* store tokens starting here */
@@ -345,7 +328,7 @@ static void tokenize1st_order_b_8x8
   TX_TYPE tx_type = xd->mode_info_context->mbmi.mode == I8X8_PRED ?
       get_tx_type(xd, b) : DCT_DCT;
 #endif
-
+  const int eob = b->eob;
   int seg_eob = 64;
   int segment_id = xd->mode_info_context->mbmi.segment_id;
 
@@ -354,16 +337,20 @@ static void tokenize1st_order_b_8x8
 
   VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
 
-  //if (!dry_run) printf("8: %d\n", tx_type);
-  for (; c < b->eob; ++c) {
+  do {
     const int band = vp8_coef_bands_8x8[c];
-    int rc = vp8_default_zig_zag1d_8x8[c];
-    int v = qcoeff_ptr[rc], x;
+    int x;
+
+    if (c < eob) {
+      const int rc = vp8_default_zig_zag1d_8x8[c];
+      const int v = qcoeff_ptr[rc];
 
-    assert(-DCT_MAX_VALUE <= v  &&  v < (DCT_MAX_VALUE));
+      assert(-DCT_MAX_VALUE <= v  &&  v < (DCT_MAX_VALUE));
 
-    t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
-    x        = vp8_dct_value_tokens_ptr[v].Token;
+      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
+      x        = vp8_dct_value_tokens_ptr[v].Token;
+    } else
+      x = DCT_EOB_TOKEN;
 
     t->Token = x;
 #if CONFIG_HYBRIDTRANSFORM8X8
@@ -384,43 +371,75 @@ static void tokenize1st_order_b_8x8
 #endif
         ++cpi->coef_counts_8x8[type][band][pt][x];
     }
-
     pt = vp8_prev_token_class[x];
     ++t;
+  } while (c < eob && ++c < seg_eob);
+
+  *tp = t;
+  pt = (c != !type); /* 0 <-> all coeff data is zero */
+  *a = *l = pt;
+}
+
+static void tokenize1st_order_chroma_4x4(MACROBLOCKD *xd,
+                                         TOKENEXTRA **tp,
+                                         int type,           /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
+                                         VP8_COMP *cpi,
+                                         int dry_run) {
+  unsigned int block;
+  const BLOCKD *b = xd->block + 16;
+  int pt;             /* near block/prev token context index */
+  TOKENEXTRA *t = *tp;/* store tokens starting here */
+  ENTROPY_CONTEXT *a;
+  ENTROPY_CONTEXT *l;
+  int seg_eob = 16;
+  int segment_id = xd->mode_info_context->mbmi.segment_id;
+
+  if (segfeature_active(xd, segment_id, SEG_LVL_EOB)) {
+    seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);
   }
 
-  if (c < seg_eob) {
-    const int band = vp8_coef_bands_8x8[c];
-    t->Token = DCT_EOB_TOKEN;
+  /* Chroma */
+  for (block = 16; block < 24; block++, b++) {
+    const int eob = b->eob;
+    const int tmp1 = vp8_block2above[block];
+    const int tmp2 = vp8_block2left[block];
+    const int16_t *qcoeff_ptr = b->qcoeff;
+    int c = 0;
 
-#if CONFIG_HYBRIDTRANSFORM8X8
-    if (tx_type != DCT_DCT)
-      t->context_tree = cpi->common.fc.hybrid_coef_probs_8x8 [type] [band] [pt];
-    else
-#endif
-      t->context_tree = cpi->common.fc.coef_probs_8x8 [type] [band] [pt];
+    a = (ENTROPY_CONTEXT *)xd->above_context + tmp1;
+    l = (ENTROPY_CONTEXT *)xd->left_context + tmp2;
 
-    t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0));
-    if (vp8_coef_encodings[t->Token].Len - t->skip_eob_node <= 0) {
-      printf("type %d, seg-eob %d, eob %d, pt %d, c %d band %d\n", type, seg_eob, b->eob, pt, c, band);
-      fflush(stdout);
-    }
-    assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
+    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
 
-    if (!dry_run) {
-#if CONFIG_HYBRIDTRANSFORM8X8
-      if (tx_type != DCT_DCT)
-        ++cpi->hybrid_coef_counts_8x8[type][band][pt][DCT_EOB_TOKEN];
-      else
-#endif
-        ++cpi->coef_counts_8x8[type][band][pt][DCT_EOB_TOKEN];
-    }
-    ++t;
-  }
+    do {
+      const int band = vp8_coef_bands[c];
+      int token;
 
-  *tp = t;
-  pt = (c != !type); /* 0 <-> all coeff data is zero */
-  *a = *l = pt;
+      if (c < eob) {
+        const int rc = vp8_default_zig_zag1d[c];
+        const int v = qcoeff_ptr[rc];
+
+        t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
+        token    = vp8_dct_value_tokens_ptr[v].Token;
+      } else
+        token = DCT_EOB_TOKEN;
+
+      t->Token = token;
+      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];
+
+      t->skip_eob_node = ((pt == 0) && (band > 0));
+      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
+
+      if (!dry_run)
+        ++cpi->coef_counts       [2] [band] [pt] [token];
+      pt = vp8_prev_token_class[token];
+      ++t;
+    } while (c < eob && ++c < seg_eob);
+
+    *tp = t;
+    pt = (c != 0); /* 0 <-> all coeff data is zero */
+    *a = *l = pt;
+  }
 }
 
 #if CONFIG_HYBRIDTRANSFORM
@@ -430,19 +449,12 @@ static void tokenize1st_order_ht_4x4(MACROBLOCKD *xd,
                                      VP8_COMP *cpi,
                                      int dry_run) {
   unsigned int block;
-  const BLOCKD *b;
+  const BLOCKD *b = xd->block;
   int pt;             /* near block/prev token context index */
-  int c;
-  int token;
   TOKENEXTRA *t = *tp;/* store tokens starting here */
-  const short *qcoeff_ptr;
   ENTROPY_CONTEXT * a;
   ENTROPY_CONTEXT * l;
-  int band, rc, v;
-  int tmp1, tmp2;
-
   int const *pt_scan ;
-
   int seg_eob = 16;
   int segment_id = xd->mode_info_context->mbmi.segment_id;
 
@@ -450,33 +462,33 @@ static void tokenize1st_order_ht_4x4(MACROBLOCKD *xd,
     seg_eob = get_segdata( xd, segment_id, SEG_LVL_EOB );
   }
 
-  b = xd->block;
-
   /* Luma */
   for (block = 0; block < 16; block++, b++) {
-    B_PREDICTION_MODE b_mode;
+    const int eob = b->eob;
     TX_TYPE tx_type = DCT_DCT;
+    const int tmp1 = vp8_block2above[block];
+    const int tmp2 = vp8_block2left[block];
+    const int16_t *qcoeff_ptr = b->qcoeff;
+    int c = type ? 0 : 1;
+
+    a = (ENTROPY_CONTEXT *)xd->above_context + tmp1;
+    l = (ENTROPY_CONTEXT *)xd->left_context + tmp2;
+    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
 
     if( xd->mode_info_context->mbmi.mode == B_PRED ) {
-      b_mode = b->bmi.as_mode.first;
       tx_type = get_tx_type(xd, b);
     }
 
     // assign scanning order for luma components coded in intra4x4 mode
-    if( (xd->mode_info_context->mbmi.mode == B_PRED) &&
-        (type == PLANE_TYPE_Y_WITH_DC) ) {
-      switch(b_mode) {
-        case B_VE_PRED :
-        case B_VR_PRED :
+    if ((xd->mode_info_context->mbmi.mode == B_PRED) &&
+        (type == PLANE_TYPE_Y_WITH_DC)) {
+      switch (tx_type) {
+        case ADST_DCT:
           pt_scan = vp8_row_scan;
           break;
-
-        case B_HE_PRED :
-        case B_HD_PRED :
-        case B_HU_PRED :
+        case DCT_ADST:
           pt_scan = vp8_col_scan;
           break;
-
         default :
           pt_scan = vp8_default_zig_zag1d;
           break;
@@ -485,22 +497,18 @@ static void tokenize1st_order_ht_4x4(MACROBLOCKD *xd,
       pt_scan = vp8_default_zig_zag1d;
     }
 
-    tmp1 = vp8_block2above[block];
-    tmp2 = vp8_block2left[block];
-    qcoeff_ptr = b->qcoeff;
-    a = (ENTROPY_CONTEXT *)xd->above_context + tmp1;
-    l = (ENTROPY_CONTEXT *)xd->left_context + tmp2;
-    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
-
-    c = type ? 0 : 1;
+    do {
+      const int band = vp8_coef_bands[c];
+      int token;
 
-    for (; c < b->eob; c++) {
-      rc = pt_scan[c];
-      band = vp8_coef_bands[c];
-      v = qcoeff_ptr[rc];
+      if (c < eob) {
+        const int rc = pt_scan[c];
+        const int v = qcoeff_ptr[rc];
 
-      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
-      token    = vp8_dct_value_tokens_ptr[v].Token;
+        t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
+        token    = vp8_dct_value_tokens_ptr[v].Token;
+      } else
+        token = DCT_EOB_TOKEN;
 
       t->Token = token;
       if (tx_type != DCT_DCT)
@@ -518,185 +526,29 @@ static void tokenize1st_order_ht_4x4(MACROBLOCKD *xd,
         else
           ++cpi->coef_counts       [type] [band] [pt] [token];
       }
-
       pt = vp8_prev_token_class[token];
-      t++;
-    }
-
-    if (c < seg_eob) {
-      band = vp8_coef_bands[c];
-      t->Token = DCT_EOB_TOKEN;
-      if (tx_type != DCT_DCT)
-        t->context_tree = cpi->common.fc.hybrid_coef_probs [type] [band] [pt];
-      else
-        t->context_tree = cpi->common.fc.coef_probs [type] [band] [pt];
-
-      t->skip_eob_node = pt == 0 &&
-          ((band > 0 && type > 0) || (band > 1 && type == 0));
-      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-      if (!dry_run) {
-        if (tx_type != DCT_DCT)
-          ++cpi->hybrid_coef_counts[type] [band] [pt] [DCT_EOB_TOKEN];
-        else
-          ++cpi->coef_counts       [type] [band] [pt] [DCT_EOB_TOKEN];
-      }
-
-      t++;
-    }
+      ++t;
+    } while (c < eob && ++c < seg_eob);
 
     *tp = t;
     pt = (c != !type); /* 0 <-> all coeff data is zero */
     *a = *l = pt;
   }
 
-  // reset scanning order for chroma components
-  pt_scan = vp8_default_zig_zag1d ;
-
-  /* Chroma */
-  for (block = 16; block < 24; block++, b++) {
-    tmp1 = vp8_block2above[block];
-    tmp2 = vp8_block2left[block];
-    qcoeff_ptr = b->qcoeff;
-    a = (ENTROPY_CONTEXT *)xd->above_context + tmp1;
-    l = (ENTROPY_CONTEXT *)xd->left_context + tmp2;
-
-    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
-
-    for (c = 0; c < b->eob; c++) {
-      rc = pt_scan[c];
-      band = vp8_coef_bands[c];
-      v = qcoeff_ptr[rc];
-
-      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
-      token    = vp8_dct_value_tokens_ptr[v].Token;
-
-      t->Token = token;
-      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];
-
-      t->skip_eob_node = ((pt == 0) && (band > 0));
-      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-
-      if (!dry_run)
-        ++cpi->coef_counts       [2] [band] [pt] [token];
-
-      pt = vp8_prev_token_class[token];
-      t++;
-  }
-
-    if (c < seg_eob) {
-      band = vp8_coef_bands[c];
-      t->Token = DCT_EOB_TOKEN;
-      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];
-
-      t->skip_eob_node = ((pt == 0) && (band > 0));
-      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-      if (!dry_run)
-        ++cpi->coef_counts       [2] [band] [pt] [DCT_EOB_TOKEN];
-      t++;
-    }
-
-    *tp = t;
-    pt = (c != 0); /* 0 <-> all coeff data is zero */
-    *a = *l = pt;
-  }
+  tokenize1st_order_chroma_4x4(xd, tp, PLANE_TYPE_UV, cpi, dry_run);
 }
 #endif
 
-
-static void tokenize1st_order_chroma_4x4
-(
-  MACROBLOCKD *xd,
-  TOKENEXTRA **tp,
-  int type,           /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
-  VP8_COMP *cpi,
-  int dry_run) {
-  unsigned int block;
-  const BLOCKD *b;
-  int pt;             /* near block/prev token context index */
-  int c;
-  int token;
-  TOKENEXTRA *t = *tp;/* store tokens starting here */
-  const short *qcoeff_ptr;
-  ENTROPY_CONTEXT *a;
-  ENTROPY_CONTEXT *l;
-  int band, rc, v;
-  int tmp1, tmp2;
-
-  int seg_eob = 16;
-  int segment_id = xd->mode_info_context->mbmi.segment_id;
-
-  if (segfeature_active(xd, segment_id, SEG_LVL_EOB)) {
-    seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);
-  }
-
-  b = xd->block;
-  b += 16;
-
-  /* Chroma */
-  for (block = 16; block < 24; block++, b++) {
-    tmp1 = vp8_block2above[block];
-    tmp2 = vp8_block2left[block];
-    qcoeff_ptr = b->qcoeff;
-    a = (ENTROPY_CONTEXT *)xd->above_context + tmp1;
-    l = (ENTROPY_CONTEXT *)xd->left_context + tmp2;
-
-    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
-
-    for (c = 0; c < b->eob; c++) {
-      rc = vp8_default_zig_zag1d[c];
-      band = vp8_coef_bands[c];
-      v = qcoeff_ptr[rc];
-
-      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
-      token    = vp8_dct_value_tokens_ptr[v].Token;
-
-      t->Token = token;
-      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];
-
-      t->skip_eob_node = ((pt == 0) && (band > 0));
-      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-
-      if (!dry_run)
-        ++cpi->coef_counts       [2] [band] [pt] [token];
-
-      pt = vp8_prev_token_class[token];
-      t++;
-    }
-
-    if (c < seg_eob) {
-      band = vp8_coef_bands[c];
-      t->Token = DCT_EOB_TOKEN;
-      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];
-
-      t->skip_eob_node = ((pt == 0) && (band > 0));
-      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-
-      if (!dry_run)
-        ++cpi->coef_counts       [2] [band] [pt] [DCT_EOB_TOKEN];
-
-      t++;
-    }
-    *tp = t;
-    pt = (c != 0); /* 0 <-> all coeff data is zero */
-    *a = *l = pt;
-  }
-}
-
-static void tokenize1st_order_b_4x4
-(
-  MACROBLOCKD *xd,
-  TOKENEXTRA **tp,
-  int type,           /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
-  VP8_COMP *cpi,
-  int dry_run) {
+static void tokenize1st_order_b_4x4(MACROBLOCKD *xd,
+                                    TOKENEXTRA **tp,
+                                    int type,           /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
+                                    VP8_COMP *cpi,
+                                    int dry_run) {
   unsigned int block;
-  const BLOCKD *b;
+  const BLOCKD *b = xd->block;
   int pt;             /* near block/prev token context index */
-  int band, rc, v, c, token;
   TOKENEXTRA *t = *tp;/* store tokens starting here */
-  const short *qcoeff_ptr;
   ENTROPY_CONTEXT *a, *l;
-
   int seg_eob = 16;
   int segment_id = xd->mode_info_context->mbmi.segment_id;
 
@@ -704,25 +556,30 @@ static void tokenize1st_order_b_4x4
     seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);
   }
 
-  b = xd->block;
   /* Luma */
   for (block = 0; block < 16; block++, b++) {
-    qcoeff_ptr = b->qcoeff;
+    const int eob = b->eob;
+    const int16_t *qcoeff_ptr = b->qcoeff;
+    int c = type ? 0 : 1;
+
     a = (ENTROPY_CONTEXT *)xd->above_context + vp8_block2above[block];
     l = (ENTROPY_CONTEXT *)xd->left_context + vp8_block2left[block];
     VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
 
-    c = type ? 0 : 1;
-
     assert(b->eob <= 16);
 
-    for (; c < b->eob; c++) {
-      rc = vp8_default_zig_zag1d[c];
-      band = vp8_coef_bands[c];
-      v = qcoeff_ptr[rc];
+    do {
+      const int band = vp8_coef_bands[c];
+      int token;
 
-      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
-      token    = vp8_dct_value_tokens_ptr[v].Token;
+      if (c < eob) {
+        const int rc = vp8_default_zig_zag1d[c];
+        const int v = qcoeff_ptr[rc];
+
+        t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
+        token    = vp8_dct_value_tokens_ptr[v].Token;
+      } else
+        token = DCT_EOB_TOKEN;
 
       t->Token = token;
       t->context_tree = cpi->common.fc.coef_probs [type] [band] [pt];
@@ -732,79 +589,18 @@ static void tokenize1st_order_b_4x4
       assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
       if (!dry_run)
         ++cpi->coef_counts       [type] [band] [pt] [token];
-
       pt = vp8_prev_token_class[token];
-      t++;
-    }
-
-    if (c < seg_eob) {
-      band = vp8_coef_bands[c];
-      t->Token = DCT_EOB_TOKEN;
-      t->context_tree = cpi->common.fc.coef_probs [type] [band] [pt];
-
-      t->skip_eob_node = pt == 0 &&
-                         ((band > 0 && type > 0) || (band > 1 && type == 0));
-      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-      if (!dry_run)
-        ++cpi->coef_counts       [type] [band] [pt] [DCT_EOB_TOKEN];
+      ++t;
+    } while (c < eob && ++c < seg_eob);
 
-      t++;
-    }
     *tp = t;
     pt = (c != !type); /* 0 <-> all coeff data is zero */
     *a = *l = pt;
-
   }
-  /* Chroma */
-  for (block = 16; block < 24; block++, b++) {
-    qcoeff_ptr = b->qcoeff;
-    a = (ENTROPY_CONTEXT *)xd->above_context + vp8_block2above[block];
-    l = (ENTROPY_CONTEXT *)xd->left_context + vp8_block2left[block];
-
-    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
-
-    assert(b->eob <= 16);
-    for (c = 0; c < b->eob; c++) {
-      rc = vp8_default_zig_zag1d[c];
-      band = vp8_coef_bands[c];
-      v = qcoeff_ptr[rc];
-
-      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
-      token    = vp8_dct_value_tokens_ptr[v].Token;
-
-      t->Token = token;
-      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];
-
-      t->skip_eob_node = ((pt == 0) && (band > 0));
-      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
-
-      if (!dry_run)
-        ++cpi->coef_counts       [2] [band] [pt] [token];
-
-      pt = vp8_prev_token_class[token];
-      t++;
-    }
-
-    if (c < seg_eob) {
-      band = vp8_coef_bands[c];
-      t->Token = DCT_EOB_TOKEN;
-      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];
-
-      t->skip_eob_node = ((pt == 0) && (band > 0));
-      assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0);
 
-      if (!dry_run)
-        ++cpi->coef_counts       [2] [band] [pt] [DCT_EOB_TOKEN];
-
-      t++;
-    }
-    *tp = t;
-    pt = (c != 0); /* 0 <-> all coeff data is zero */
-    *a = *l = pt;
-  }
+  tokenize1st_order_chroma_4x4(xd, tp, PLANE_TYPE_UV, cpi, dry_run);
 }
 
-
 int mby_is_skippable_4x4(MACROBLOCKD *xd, int has_y2_block) {
   int skip = 1;
   int i = 0;