Add col/row-based coefficient scanning patterns for 1D 8x8/16x16 ADSTs.

These are mostly just for experimental purposes. I saw small gains (in
the 0.1% range) when playing with this on derf.

Change-Id: Ib21eed477bbb46bddcd73b21c5c708a5b46abedc

10 files changed, 284 insertions, 46 deletions

diff --git a/vp9/common/vp9_entropy.c b/vp9/common/vp9_entropy.c
index 1e13861db..22c2905d8 100644
--- a/vp9/common/vp9_entropy.c
+++ b/vp9/common/vp9_entropy.c
@@ -91,6 +91,28 @@ DECLARE_ALIGNED(64, const int, vp9_default_zig_zag1d_8x8[64]) = {
   58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63,
 };
 
+DECLARE_ALIGNED(16, const int, vp9_col_scan_8x8[64]) = {
+   0,  8, 16, 24, 32, 40, 48, 56,
+   1,  9, 17, 25, 33, 41, 49, 57,
+   2, 10, 18, 26, 34, 42, 50, 58,
+   3, 11, 19, 27, 35, 43, 51, 59,
+   4, 12, 20, 28, 36, 44, 52, 60,
+   5, 13, 21, 29, 37, 45, 53, 61,
+   6, 14, 22, 30, 38, 46, 54, 62,
+   7, 15, 23, 31, 39, 47, 55, 63,
+};
+
+DECLARE_ALIGNED(16, const int, vp9_row_scan_8x8[64]) = {
+   0,  1,  2,  3,  4,  5,  6,  7,
+   8,  9, 10, 11, 12, 13, 14, 15,
+  16, 17, 18, 19, 20, 21, 22, 23,
+  24, 25, 26, 27, 28, 29, 30, 31,
+  32, 33, 34, 35, 36, 37, 38, 39,
+  40, 41, 42, 43, 44, 45, 46, 47,
+  48, 49, 50, 51, 52, 53, 54, 55,
+  56, 57, 58, 59, 60, 61, 62, 63,
+};
+
 DECLARE_ALIGNED(16, const int, vp9_default_zig_zag1d_16x16[256]) = {
   0,   1,  16,  32,  17,   2,   3,  18,
   33,  48,  64,  49,  34,  19,   4,   5,
@@ -126,6 +148,44 @@ DECLARE_ALIGNED(16, const int, vp9_default_zig_zag1d_16x16[256]) = {
   237, 252, 253, 238, 223, 239, 254, 255,
 };
 
+DECLARE_ALIGNED(16, const int, vp9_col_scan_16x16[256]) = {
+    0,  16,  32,  48,  64,  80,  96, 112, 128, 144, 160, 176, 192, 208, 224, 240,
+    1,  17,  33,  49,  65,  81,  97, 113, 129, 145, 161, 177, 193, 209, 225, 241,
+    2,  18,  34,  50,  66,  82,  98, 114, 130, 146, 162, 178, 194, 210, 226, 242,
+    3,  19,  35,  51,  67,  83,  99, 115, 131, 147, 163, 179, 195, 211, 227, 243,
+    4,  20,  36,  52,  68,  84, 100, 116, 132, 148, 164, 180, 196, 212, 228, 244,
+    5,  21,  37,  53,  69,  85, 101, 117, 133, 149, 165, 181, 197, 213, 229, 245,
+    6,  22,  38,  54,  70,  86, 102, 118, 134, 150, 166, 182, 198, 214, 230, 246,
+    7,  23,  39,  55,  71,  87, 103, 119, 135, 151, 167, 183, 199, 215, 231, 247,
+    8,  24,  40,  56,  72,  88, 104, 120, 136, 152, 168, 184, 200, 216, 232, 248,
+    9,  25,  41,  57,  73,  89, 105, 121, 137, 153, 169, 185, 201, 217, 233, 249,
+   10,  26,  42,  58,  74,  90, 106, 122, 138, 154, 170, 186, 202, 218, 234, 250,
+   11,  27,  43,  59,  75,  91, 107, 123, 139, 155, 171, 187, 203, 219, 235, 251,
+   12,  28,  44,  60,  76,  92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252,
+   13,  29,  45,  61,  77,  93, 109, 125, 141, 157, 173, 189, 205, 221, 237, 253,
+   14,  30,  46,  62,  78,  94, 110, 126, 142, 158, 174, 190, 206, 222, 238, 254,
+   15,  31,  47,  63,  79,  95, 111, 127, 143, 159, 175, 191, 207, 223, 239, 255,
+};
+
+DECLARE_ALIGNED(16, const int, vp9_row_scan_16x16[256]) = {
+    0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,  13,  14,  15,
+   16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,  27,  28,  29,  30,  31,
+   32,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,  45,  46,  47,
+   48,  49,  50,  51,  52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,
+   64,  65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,  78,  79,
+   80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,  91,  92,  93,  94,  95,
+   96,  97,  98,  99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111,
+  112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127,
+  128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143,
+  144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
+  160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
+  176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191,
+  192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,
+  208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223,
+  224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
+  240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
+};
+
 DECLARE_ALIGNED(16, const int, vp9_default_zig_zag1d_32x32[1024]) = {
     0,    1,   32,   64,   33,    2,    3,   34,   65,   96,  128,   97,   66,   35,    4,    5,   36,   67,   98,  129,  160,  192,  161,  130,   99,   68,   37,    6,    7,   38,   69,  100,
   131,  162,  193,  224,  256,  225,  194,  163,  132,  101,   70,   39,    8,    9,   40,   71,  102,  133,  164,  195,  226,  257,  288,  320,  289,  258,  227,  196,  165,  134,  103,   72,
@@ -2080,8 +2140,16 @@ DECLARE_ALIGNED(16, int,
 DECLARE_ALIGNED(16, int,
                 vp9_row_scan_4x4_neighbors[16 * MAX_NEIGHBORS]);
 DECLARE_ALIGNED(16, int,
+                vp9_col_scan_8x8_neighbors[64 * MAX_NEIGHBORS]);
+DECLARE_ALIGNED(16, int,
+                vp9_row_scan_8x8_neighbors[64 * MAX_NEIGHBORS]);
+DECLARE_ALIGNED(16, int,
                 vp9_default_zig_zag1d_8x8_neighbors[64 * MAX_NEIGHBORS]);
 DECLARE_ALIGNED(16, int,
+                vp9_col_scan_16x16_neighbors[256 * MAX_NEIGHBORS]);
+DECLARE_ALIGNED(16, int,
+                vp9_row_scan_16x16_neighbors[256 * MAX_NEIGHBORS]);
+DECLARE_ALIGNED(16, int,
                 vp9_default_zig_zag1d_16x16_neighbors[256 * MAX_NEIGHBORS]);
 DECLARE_ALIGNED(16, int,
                 vp9_default_zig_zag1d_32x32_neighbors[1024 * MAX_NEIGHBORS]);
@@ -2117,8 +2185,13 @@ static void init_scan_neighbors(const int *scan, int l, int *neighbors,
       // use the combination of the two as a context.
       int a = find_in_scan(scan, l, (i - 1) * l + j);
       int b = find_in_scan(scan, l,  i      * l + j - 1);
-      if (scan == vp9_col_scan_4x4 || scan == vp9_row_scan_4x4) {
-        neighbors[max_neighbors * n + 0] = MAX(a, b);
+      if (scan == vp9_col_scan_4x4 || scan == vp9_col_scan_8x8 ||
+          scan == vp9_col_scan_16x16) {
+        neighbors[max_neighbors * n + 0] = a;
+        neighbors[max_neighbors * n + 1] = -1;
+      } else if (scan == vp9_row_scan_4x4 || scan == vp9_row_scan_8x8 ||
+                 scan == vp9_row_scan_16x16) {
+        neighbors[max_neighbors * n + 0] = b;
         neighbors[max_neighbors * n + 1] = -1;
       } else {
         neighbors[max_neighbors * n + 0] = a;
@@ -2149,8 +2222,16 @@ void vp9_init_neighbors() {
                       vp9_col_scan_4x4_neighbors, MAX_NEIGHBORS);
   init_scan_neighbors(vp9_default_zig_zag1d_8x8, 8,
                       vp9_default_zig_zag1d_8x8_neighbors, MAX_NEIGHBORS);
+  init_scan_neighbors(vp9_row_scan_8x8, 8,
+                      vp9_row_scan_8x8_neighbors, MAX_NEIGHBORS);
+  init_scan_neighbors(vp9_col_scan_8x8, 8,
+                      vp9_col_scan_8x8_neighbors, MAX_NEIGHBORS);
   init_scan_neighbors(vp9_default_zig_zag1d_16x16, 16,
                       vp9_default_zig_zag1d_16x16_neighbors, MAX_NEIGHBORS);
+  init_scan_neighbors(vp9_row_scan_16x16, 16,
+                      vp9_row_scan_16x16_neighbors, MAX_NEIGHBORS);
+  init_scan_neighbors(vp9_col_scan_16x16, 16,
+                      vp9_col_scan_16x16_neighbors, MAX_NEIGHBORS);
   init_scan_neighbors(vp9_default_zig_zag1d_32x32, 32,
                       vp9_default_zig_zag1d_32x32_neighbors, MAX_NEIGHBORS);
 }
@@ -2168,9 +2249,21 @@ const int *vp9_get_coef_neighbors_handle(const int *scan, int *pad) {
   } else if (scan == vp9_default_zig_zag1d_8x8) {
     *pad = MAX_NEIGHBORS;
     return vp9_default_zig_zag1d_8x8_neighbors;
+  } else if (scan == vp9_row_scan_8x8) {
+    *pad = 2;
+    return vp9_row_scan_8x8_neighbors;
+  } else if (scan == vp9_col_scan_8x8) {
+    *pad = 2;
+    return vp9_col_scan_8x8_neighbors;
   } else if (scan == vp9_default_zig_zag1d_16x16) {
     *pad = MAX_NEIGHBORS;
     return vp9_default_zig_zag1d_16x16_neighbors;
+  } else if (scan == vp9_row_scan_16x16) {
+    *pad = 2;
+    return vp9_row_scan_16x16_neighbors;
+  } else if (scan == vp9_col_scan_16x16) {
+    *pad = 2;
+    return vp9_col_scan_16x16_neighbors;
   } else if (scan == vp9_default_zig_zag1d_32x32) {
     *pad = MAX_NEIGHBORS;
     return vp9_default_zig_zag1d_32x32_neighbors;
diff --git a/vp9/common/vp9_entropy.h b/vp9/common/vp9_entropy.h
index ec8cb72e1..64f595047 100644
--- a/vp9/common/vp9_entropy.h
+++ b/vp9/common/vp9_entropy.h
@@ -102,7 +102,15 @@ extern DECLARE_ALIGNED(16, const int, vp9_col_scan_4x4[16]);
 extern DECLARE_ALIGNED(16, const int, vp9_row_scan_4x4[16]);
 
 extern DECLARE_ALIGNED(64, const int, vp9_default_zig_zag1d_8x8[64]);
+
+extern DECLARE_ALIGNED(16, const int, vp9_col_scan_8x8[64]);
+extern DECLARE_ALIGNED(16, const int, vp9_row_scan_8x8[64]);
+
 extern DECLARE_ALIGNED(16, const int, vp9_default_zig_zag1d_16x16[256]);
+
+extern DECLARE_ALIGNED(16, const int, vp9_col_scan_16x16[256]);
+extern DECLARE_ALIGNED(16, const int, vp9_row_scan_16x16[256]);
+
 extern DECLARE_ALIGNED(16, const int, vp9_default_zig_zag1d_32x32[1024]);
 
 void vp9_coef_tree_initialize(void);
diff --git a/vp9/decoder/vp9_detokenize.c b/vp9/decoder/vp9_detokenize.c
index 3def217ee..b6eec373e 100644
--- a/vp9/decoder/vp9_detokenize.c
+++ b/vp9/decoder/vp9_detokenize.c
@@ -132,7 +132,8 @@ static int decode_coefs(VP9D_COMP *dx, const MACROBLOCKD *xd,
   switch (txfm_size) {
     default:
     case TX_4X4: {
-      const TX_TYPE tx_type = get_tx_type_4x4(xd, block_idx);
+      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
+                              get_tx_type_4x4(xd, block_idx) : DCT_DCT;
       switch (tx_type) {
         default:
           scan = vp9_default_zig_zag1d_4x4;
@@ -151,16 +152,47 @@ static int decode_coefs(VP9D_COMP *dx, const MACROBLOCKD *xd,
       default_eob = 16;
       break;
     }
-    case TX_8X8:
-      scan = vp9_default_zig_zag1d_8x8;
+    case TX_8X8: {
+      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
+      const int sz = 3 + sb_type, x = block_idx & ((1 << sz) - 1);
+      const int y = block_idx - x;
+      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
+                              get_tx_type_8x8(xd, y + (x >> 1)) : DCT_DCT;
+      switch (tx_type) {
+        default:
+          scan = vp9_default_zig_zag1d_8x8;
+          break;
+        case ADST_DCT:
+          scan = vp9_row_scan_8x8;
+          break;
+        case DCT_ADST:
+          scan = vp9_col_scan_8x8;
+          break;
+      }
       coef_probs  = fc->coef_probs_8x8;
       coef_counts = fc->coef_counts_8x8;
       above_ec = (A0[aidx] + A0[aidx + 1]) != 0;
       left_ec  = (L0[lidx] + L0[lidx + 1]) != 0;
       default_eob = 64;
       break;
-    case TX_16X16:
-      scan = vp9_default_zig_zag1d_16x16;
+    }
+    case TX_16X16: {
+      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
+      const int sz = 4 + sb_type, x = block_idx & ((1 << sz) - 1);
+      const int y = block_idx - x;
+      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
+                              get_tx_type_16x16(xd, y + (x >> 2)) : DCT_DCT;
+      switch (tx_type) {
+        default:
+          scan = vp9_default_zig_zag1d_16x16;
+          break;
+        case ADST_DCT:
+          scan = vp9_row_scan_16x16;
+          break;
+        case DCT_ADST:
+          scan = vp9_col_scan_16x16;
+          break;
+      }
       coef_probs  = fc->coef_probs_16x16;
       coef_counts = fc->coef_counts_16x16;
       if (type == PLANE_TYPE_UV) {
@@ -174,6 +206,7 @@ static int decode_coefs(VP9D_COMP *dx, const MACROBLOCKD *xd,
       }
       default_eob = 256;
       break;
+    }
     case TX_32X32:
       scan = vp9_default_zig_zag1d_32x32;
       coef_probs = fc->coef_probs_32x32;
diff --git a/vp9/encoder/vp9_block.h b/vp9/encoder/vp9_block.h
index 439006156..491ea62b5 100644
--- a/vp9/encoder/vp9_block.h
+++ b/vp9/encoder/vp9_block.h
@@ -173,8 +173,8 @@ struct macroblock {
   void (*fwd_txm16x16)(int16_t *input, int16_t *output, int pitch);
   void (*quantize_b_4x4)(MACROBLOCK *x, int b_idx);
   void (*quantize_b_4x4_pair)(MACROBLOCK *x, int b_idx1, int b_idx2);
-  void (*quantize_b_16x16)(MACROBLOCK *x, int b_idx);
-  void (*quantize_b_8x8)(MACROBLOCK *x, int b_idx);
+  void (*quantize_b_16x16)(MACROBLOCK *x, int b_idx, TX_TYPE tx_type);
+  void (*quantize_b_8x8)(MACROBLOCK *x, int b_idx, TX_TYPE tx_type);
 };
 
 #endif  // VP9_ENCODER_VP9_BLOCK_H_
diff --git a/vp9/encoder/vp9_encodeintra.c b/vp9/encoder/vp9_encodeintra.c
index 9e5bcea16..eddacb872 100644
--- a/vp9/encoder/vp9_encodeintra.c
+++ b/vp9/encoder/vp9_encodeintra.c
@@ -155,12 +155,12 @@ void vp9_encode_intra8x8(MACROBLOCK *x, int ib) {
     tx_type = get_tx_type_8x8(xd, ib);
     if (tx_type != DCT_DCT) {
       vp9_short_fht8x8(be->src_diff, (x->block + idx)->coeff, 16, tx_type);
-      x->quantize_b_8x8(x, idx);
+      x->quantize_b_8x8(x, idx, tx_type);
       vp9_short_iht8x8(xd->block[idx].dqcoeff, xd->block[ib].diff,
                             16, tx_type);
     } else {
       x->fwd_txm8x8(be->src_diff, (x->block + idx)->coeff, 32);
-      x->quantize_b_8x8(x, idx);
+      x->quantize_b_8x8(x, idx, DCT_DCT);
       vp9_short_idct8x8(xd->block[idx].dqcoeff, xd->block[ib].diff, 32);
     }
   } else {
diff --git a/vp9/encoder/vp9_encodemb.c b/vp9/encoder/vp9_encodemb.c
index f51f4c57a..3ad429a9e 100644
--- a/vp9/encoder/vp9_encodemb.c
+++ b/vp9/encoder/vp9_encodemb.c
@@ -598,20 +598,40 @@ static void optimize_b(VP9_COMMON *const cm,
       }
       break;
     }
-    case TX_8X8:
-      scan = vp9_default_zig_zag1d_8x8;
+    case TX_8X8: {
+      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
+      const int sz = 3 + sb_type, x = ib & ((1 << sz) - 1), y = ib - x;
+      const TX_TYPE tx_type = get_tx_type_8x8(xd, y + (x >> 1));
+      if (tx_type == DCT_ADST) {
+        scan = vp9_col_scan_8x8;
+      } else if (tx_type == ADST_DCT) {
+        scan = vp9_row_scan_8x8;
+      } else {
+        scan = vp9_default_zig_zag1d_8x8;
+      }
       default_eob = 64;
 #if CONFIG_CODE_NONZEROCOUNT
       nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
 #endif
       break;
-    case TX_16X16:
-      scan = vp9_default_zig_zag1d_16x16;
+    }
+    case TX_16X16: {
+      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
+      const int sz = 4 + sb_type, x = ib & ((1 << sz) - 1), y = ib - x;
+      const TX_TYPE tx_type = get_tx_type_16x16(xd, y + (x >> 2));
+      if (tx_type == DCT_ADST) {
+        scan = vp9_col_scan_16x16;
+      } else if (tx_type == ADST_DCT) {
+        scan = vp9_row_scan_16x16;
+      } else {
+        scan = vp9_default_zig_zag1d_16x16;
+      }
       default_eob = 256;
 #if CONFIG_CODE_NONZEROCOUNT
       nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
 #endif
       break;
+    }
     case TX_32X32:
       scan = vp9_default_zig_zag1d_32x32;
       default_eob = 1024;
diff --git a/vp9/encoder/vp9_quantize.c b/vp9/encoder/vp9_quantize.c
index 13958c03d..881fce50f 100644
--- a/vp9/encoder/vp9_quantize.c
+++ b/vp9/encoder/vp9_quantize.c
@@ -196,13 +196,26 @@ void vp9_quantize_mb_4x4(MACROBLOCK *x) {
   vp9_quantize_mbuv_4x4(x);
 }
 
-void vp9_regular_quantize_b_8x8(MACROBLOCK *mb, int b_idx) {
+void vp9_regular_quantize_b_8x8(MACROBLOCK *mb, int b_idx, TX_TYPE tx_type) {
   MACROBLOCKD *const xd = &mb->e_mbd;
   int16_t *qcoeff_ptr = xd->qcoeff + 16 * b_idx;
   int16_t *dqcoeff_ptr = xd->dqcoeff + 16 * b_idx;
   const int c_idx = plane_idx(xd, b_idx);
   BLOCK *const b = &mb->block[c_idx];
   BLOCKD *const d = &xd->block[c_idx];
+  const int *pt_scan;
+
+  switch (tx_type) {
+    case ADST_DCT:
+      pt_scan = vp9_row_scan_8x8;
+      break;
+    case DCT_ADST:
+      pt_scan = vp9_col_scan_8x8;
+      break;
+    default:
+      pt_scan = vp9_default_zig_zag1d_8x8;
+      break;
+  }
 
   vpx_memset(qcoeff_ptr, 0, 64 * sizeof(int16_t));
   vpx_memset(dqcoeff_ptr, 0, 64 * sizeof(int16_t));
@@ -254,7 +267,7 @@ void vp9_regular_quantize_b_8x8(MACROBLOCK *mb, int b_idx) {
       }
     }
     for (i = 1; i < 64; i++) {
-      rc   = vp9_default_zig_zag1d_8x8[i];
+      rc   = pt_scan[i];
       z    = coeff_ptr[rc];
       zbin = (zbin_ptr[1] + zbin_boost_ptr[zero_run] + zbin_oq_value);
       // The original code was incrementing zero_run while keeping it at
@@ -303,7 +316,8 @@ void vp9_quantize_mby_8x8(MACROBLOCK *x) {
   }
 #endif
   for (i = 0; i < 16; i += 4) {
-    x->quantize_b_8x8(x, i);
+    TX_TYPE tx_type = get_tx_type_8x8(&x->e_mbd, (i & 8) + ((i & 4) >> 1));
+    x->quantize_b_8x8(x, i, tx_type);
   }
 }
 
@@ -316,7 +330,7 @@ void vp9_quantize_mbuv_8x8(MACROBLOCK *x) {
   }
 #endif
   for (i = 16; i < 24; i += 4)
-    x->quantize_b_8x8(x, i);
+    x->quantize_b_8x8(x, i, DCT_DCT);
 }
 
 void vp9_quantize_mb_8x8(MACROBLOCK *x) {
@@ -325,13 +339,14 @@ void vp9_quantize_mb_8x8(MACROBLOCK *x) {
 }
 
 void vp9_quantize_mby_16x16(MACROBLOCK *x) {
+  TX_TYPE tx_type = get_tx_type_16x16(&x->e_mbd, 0);
 #if CONFIG_CODE_NONZEROCOUNT
   int i;
   for (i = 0; i < 16; i++) {
     x->e_mbd.nzcs[i] = 0;
   }
 #endif
-  x->quantize_b_16x16(x, 0);
+  x->quantize_b_16x16(x, 0, tx_type);
 }
 
 void vp9_quantize_mb_16x16(MACROBLOCK *x) {
@@ -400,11 +415,24 @@ static void quantize(int16_t *zbin_boost_orig_ptr,
 #endif
 }
 
-void vp9_regular_quantize_b_16x16(MACROBLOCK *mb, int b_idx) {
+void vp9_regular_quantize_b_16x16(MACROBLOCK *mb, int b_idx, TX_TYPE tx_type) {
   MACROBLOCKD *const xd = &mb->e_mbd;
   const int c_idx = plane_idx(xd, b_idx);
   BLOCK *const b = &mb->block[c_idx];
   BLOCKD *const d = &xd->block[c_idx];
+  const int *pt_scan;
+
+  switch (tx_type) {
+    case ADST_DCT:
+      pt_scan = vp9_row_scan_16x16;
+      break;
+    case DCT_ADST:
+      pt_scan = vp9_col_scan_16x16;
+      break;
+    default:
+      pt_scan = vp9_default_zig_zag1d_16x16;
+      break;
+  }
 
   quantize(b->zrun_zbin_boost,
            mb->coeff + 16 * b_idx,
@@ -418,7 +446,7 @@ void vp9_regular_quantize_b_16x16(MACROBLOCK *mb, int b_idx) {
 #if CONFIG_CODE_NONZEROCOUNT
            &xd->nzcs[b_idx],
 #endif
-           vp9_default_zig_zag1d_16x16, 1);
+           pt_scan, 1);
 }
 
 void vp9_regular_quantize_b_32x32(MACROBLOCK *mb, int b_idx) {
@@ -450,15 +478,21 @@ void vp9_quantize_sby_32x32(MACROBLOCK *x) {
 void vp9_quantize_sby_16x16(MACROBLOCK *x) {
   int n;
 
-  for (n = 0; n < 4; n++)
-    x->quantize_b_16x16(x, n * 16);
+  for (n = 0; n < 4; n++) {
+    TX_TYPE tx_type = get_tx_type_16x16(&x->e_mbd,
+                                        (16 * (n & 2)) + ((n & 1) * 4));
+    x->quantize_b_16x16(x, n * 16, tx_type);
+  }
 }
 
 void vp9_quantize_sby_8x8(MACROBLOCK *x) {
   int n;
 
-  for (n = 0; n < 16; n++)
-    x->quantize_b_8x8(x, n * 4);
+  for (n = 0; n < 16; n++) {
+    TX_TYPE tx_type = get_tx_type_8x8(&x->e_mbd,
+                                      (4 * (n & 12)) + ((n & 3) * 2));
+    x->quantize_b_8x8(x, n * 4, tx_type);
+  }
 }
 
 void vp9_quantize_sby_4x4(MACROBLOCK *x) {
@@ -476,15 +510,15 @@ void vp9_quantize_sby_4x4(MACROBLOCK *x) {
 }
 
 void vp9_quantize_sbuv_16x16(MACROBLOCK *x) {
-  x->quantize_b_16x16(x, 64);
-  x->quantize_b_16x16(x, 80);
+  x->quantize_b_16x16(x, 64, DCT_DCT);
+  x->quantize_b_16x16(x, 80, DCT_DCT);
 }
 
 void vp9_quantize_sbuv_8x8(MACROBLOCK *x) {
   int i;
 
   for (i = 64; i < 96; i += 4)
-    x->quantize_b_8x8(x, i);
+    x->quantize_b_8x8(x, i, DCT_DCT);
 }
 
 void vp9_quantize_sbuv_4x4(MACROBLOCK *x) {
@@ -504,15 +538,21 @@ void vp9_quantize_sb64y_32x32(MACROBLOCK *x) {
 void vp9_quantize_sb64y_16x16(MACROBLOCK *x) {
   int n;
 
-  for (n = 0; n < 16; n++)
-    x->quantize_b_16x16(x, n * 16);
+  for (n = 0; n < 16; n++) {
+    TX_TYPE tx_type = get_tx_type_16x16(&x->e_mbd,
+                                        (16 * (n & 12)) + ((n & 3) * 4));
+    x->quantize_b_16x16(x, n * 16, tx_type);
+  }
 }
 
 void vp9_quantize_sb64y_8x8(MACROBLOCK *x) {
   int n;
 
-  for (n = 0; n < 64; n++)
-    x->quantize_b_8x8(x, n * 4);
+  for (n = 0; n < 64; n++) {
+    TX_TYPE tx_type = get_tx_type_8x8(&x->e_mbd,
+                                      (4 * (n & 56)) + ((n & 7) * 2));
+    x->quantize_b_8x8(x, n * 4, tx_type);
+  }
 }
 
 void vp9_quantize_sb64y_4x4(MACROBLOCK *x) {
@@ -538,14 +578,14 @@ void vp9_quantize_sb64uv_16x16(MACROBLOCK *x) {
   int i;
 
   for (i = 256; i < 384; i += 16)
-    x->quantize_b_16x16(x, i);
+    x->quantize_b_16x16(x, i, DCT_DCT);
 }
 
 void vp9_quantize_sb64uv_8x8(MACROBLOCK *x) {
   int i;
 
   for (i = 256; i < 384; i += 4)
-    x->quantize_b_8x8(x, i);
+    x->quantize_b_8x8(x, i, DCT_DCT);
 }
 
 void vp9_quantize_sb64uv_4x4(MACROBLOCK *x) {
diff --git a/vp9/encoder/vp9_quantize.h b/vp9/encoder/vp9_quantize.h
index 739254025..6ba6cbdd9 100644
--- a/vp9/encoder/vp9_quantize.h
+++ b/vp9/encoder/vp9_quantize.h
@@ -29,8 +29,8 @@
 void vp9_ht_quantize_b_4x4(MACROBLOCK *mb, int b_ix, TX_TYPE type);
 void vp9_regular_quantize_b_4x4(MACROBLOCK *mb, int b_idx);
 void vp9_regular_quantize_b_4x4_pair(MACROBLOCK *mb, int b_idx1, int b_idx2);
-void vp9_regular_quantize_b_8x8(MACROBLOCK *mb, int b_idx);
-void vp9_regular_quantize_b_16x16(MACROBLOCK *mb, int b_idx);
+void vp9_regular_quantize_b_8x8(MACROBLOCK *mb, int b_idx, TX_TYPE tx_type);
+void vp9_regular_quantize_b_16x16(MACROBLOCK *mb, int b_idx, TX_TYPE tx_type);
 void vp9_regular_quantize_b_32x32(MACROBLOCK *mb, int b_idx);
 
 void vp9_quantize_mb_4x4(MACROBLOCK *x);
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index f68afabed..8abcd1141 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -488,10 +488,20 @@ static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
       }
       break;
     }
-    case TX_8X8:
+    case TX_8X8: {
+      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
+      const int sz = 3 + sb_type, x = ib & ((1 << sz) - 1), y = ib - x;
+      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
+                              get_tx_type_8x8(xd, y + (x >> 1)) : DCT_DCT;
       a_ec = (a[0] + a[1]) != 0;
       l_ec = (l[0] + l[1]) != 0;
-      scan = vp9_default_zig_zag1d_8x8;
+      if (tx_type == ADST_DCT) {
+        scan = vp9_row_scan_8x8;
+      } else if (tx_type == DCT_ADST) {
+        scan = vp9_col_scan_8x8;
+      } else {
+        scan = vp9_default_zig_zag1d_8x8;
+      }
 #if CONFIG_CODE_NONZEROCOUNT
       nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
 #else
@@ -499,8 +509,19 @@ static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
 #endif
       seg_eob = 64;
       break;
-    case TX_16X16:
-      scan = vp9_default_zig_zag1d_16x16;
+    }
+    case TX_16X16: {
+      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
+      const int sz = 4 + sb_type, x = ib & ((1 << sz) - 1), y = ib - x;
+      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
+                              get_tx_type_16x16(xd, y + (x >> 2)) : DCT_DCT;
+      if (tx_type == ADST_DCT) {
+        scan = vp9_row_scan_16x16;
+      } else if (tx_type == DCT_ADST) {
+        scan = vp9_col_scan_16x16;
+      } else {
+        scan = vp9_default_zig_zag1d_16x16;
+      }
 #if CONFIG_CODE_NONZEROCOUNT
       nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
 #else
@@ -515,6 +536,7 @@ static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
         l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
       }
       break;
+    }
     case TX_32X32:
       scan = vp9_default_zig_zag1d_32x32;
 #if CONFIG_CODE_NONZEROCOUNT
@@ -1498,7 +1520,7 @@ static int64_t rd_pick_intra8x8block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
         vp9_short_fht8x8(be->src_diff, (x->block + idx)->coeff, 16, tx_type);
       else
         x->fwd_txm8x8(be->src_diff, (x->block + idx)->coeff, 32);
-      x->quantize_b_8x8(x, idx);
+      x->quantize_b_8x8(x, idx, tx_type);
 
       // compute quantization mse of 8x8 block
       distortion = vp9_block_error_c((x->block + idx)->coeff,
@@ -2503,7 +2525,7 @@ static int64_t encode_inter_mb_segment_8x8(VP9_COMMON *const cm,
       if (xd->mode_info_context->mbmi.txfm_size == TX_4X4) {
         if (otherrd) {
           x->fwd_txm8x8(be->src_diff, be2->coeff, 32);
-          x->quantize_b_8x8(x, idx);
+          x->quantize_b_8x8(x, idx, DCT_DCT);
           thisdistortion = vp9_block_error_c(be2->coeff, bd2->dqcoeff, 64);
           otherdist += thisdistortion;
           xd->mode_info_context->mbmi.txfm_size = TX_8X8;
@@ -2557,7 +2579,7 @@ static int64_t encode_inter_mb_segment_8x8(VP9_COMMON *const cm,
           }
         }
         x->fwd_txm8x8(be->src_diff, be2->coeff, 32);
-        x->quantize_b_8x8(x, idx);
+        x->quantize_b_8x8(x, idx, DCT_DCT);
         thisdistortion = vp9_block_error_c(be2->coeff, bd2->dqcoeff, 64);
         *distortion += thisdistortion;
         *labelyrate += cost_coeffs(cm, x, idx, PLANE_TYPE_Y_WITH_DC,
diff --git a/vp9/encoder/vp9_tokenize.c b/vp9/encoder/vp9_tokenize.c
index c770a5c59..6e2b8474d 100644
--- a/vp9/encoder/vp9_tokenize.c
+++ b/vp9/encoder/vp9_tokenize.c
@@ -181,15 +181,29 @@ static void tokenize_b(VP9_COMP *cpi,
       probs = cpi->common.fc.coef_probs_4x4;
       break;
     }
-    case TX_8X8:
+    case TX_8X8: {
+      const int sz = 3 + sb_type, x = ib & ((1 << sz) - 1), y = ib - x;
+      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
+                              get_tx_type_8x8(xd, y + (x >> 1)) : DCT_DCT;
       a_ec = (a[0] + a[1]) != 0;
       l_ec = (l[0] + l[1]) != 0;
       seg_eob = 64;
       scan = vp9_default_zig_zag1d_8x8;
+      if (tx_type != DCT_DCT) {
+        if (tx_type == ADST_DCT) {
+          scan = vp9_row_scan_8x8;
+        } else if (tx_type == DCT_ADST) {
+          scan = vp9_col_scan_8x8;
+        }
+      }
       counts = cpi->coef_counts_8x8;
       probs = cpi->common.fc.coef_probs_8x8;
       break;
-    case TX_16X16:
+    }
+    case TX_16X16: {
+      const int sz = 4 + sb_type, x = ib & ((1 << sz) - 1), y = ib - x;
+      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
+                              get_tx_type_16x16(xd, y + (x >> 2)) : DCT_DCT;
       if (type != PLANE_TYPE_UV) {
         a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
         l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
@@ -199,9 +213,17 @@ static void tokenize_b(VP9_COMP *cpi,
       }
       seg_eob = 256;
       scan = vp9_default_zig_zag1d_16x16;
+      if (tx_type != DCT_DCT) {
+        if (tx_type == ADST_DCT) {
+          scan = vp9_row_scan_16x16;
+        } else if (tx_type == DCT_ADST) {
+          scan = vp9_col_scan_16x16;
+        }
+      }
       counts = cpi->coef_counts_16x16;
       probs = cpi->common.fc.coef_probs_16x16;
       break;
+    }
     case TX_32X32:
       if (type != PLANE_TYPE_UV) {
         a_ec = (a[0] + a[1] + a[2] + a[3] +