changes intra coding to be based on txfm block

This commit changed the encoding and decoding of intra blocks to be
based on transform block. In each prediction block, the intra coding
iterates thorough each transform block based on raster scan order.

This commit also fixed a bug in D135 prediction code.

TODO next:
The RD mode/txfm_size selection should take this into account when
computing RD values.

Change-Id: I6d1be2faa4c4948a52e830b6a9a84a6b2b6850f6

5 files changed, 65 insertions, 36 deletions

diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index 88943330f..9f3268021 100644
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -871,8 +871,9 @@ static void pack_inter_mode_mvs(VP9_COMP *cpi, MODE_INFO *m,
   if (((rf == INTRA_FRAME && mode != I4X4_PRED) ||
        (rf != INTRA_FRAME && mode != SPLITMV)) &&
       pc->txfm_mode == TX_MODE_SELECT &&
-      !(skip_coeff || vp9_segfeature_active(xd, segment_id,
-                                            SEG_LVL_SKIP)))
+      !(rf != INTRA_FRAME &&
+        (skip_coeff || vp9_segfeature_active(xd, segment_id,
+                                            SEG_LVL_SKIP))))
 #endif
   {
     TX_SIZE sz = mi->txfm_size;
@@ -941,7 +942,8 @@ static void write_mb_modes_kf(const VP9_COMP *cpi,
       !(skip_coeff || vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))) {
 #else
   if (ym != I4X4_PRED && c->txfm_mode == TX_MODE_SELECT &&
-      !(skip_coeff || vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))) {
+    !(m->mbmi.ref_frame != INTRA_FRAME && (skip_coeff ||
+      vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP)))) {
 #endif
     TX_SIZE sz = m->mbmi.txfm_size;
     // FIXME(rbultje) code ternary symbol once all experiments are merged
diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 277f92ce7..73eec5656 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -1703,19 +1703,14 @@ static void encode_superblock(VP9_COMP *cpi, TOKENEXTRA **t,
 #if CONFIG_AB4X4
   if (mbmi->ref_frame == INTRA_FRAME &&
       bsize < BLOCK_SIZE_SB8X8) {
-#else
-  if (mbmi->mode == I4X4_PRED) {
-    assert(bsize == BLOCK_SIZE_SB8X8 && mbmi->txfm_size == TX_4X4);
-#endif
-    vp9_encode_intra4x4mby(cm, x, BLOCK_SIZE_SB8X8);
+    vp9_encode_intra_block_y(cm, x, BLOCK_SIZE_SB8X8);
     vp9_build_intra_predictors_sbuv_s(xd, BLOCK_SIZE_SB8X8);
     vp9_encode_sbuv(cm, x, BLOCK_SIZE_SB8X8);
-
-    if (output_enabled)
-      sum_intra_stats(cpi, x);
-  } else if (mbmi->ref_frame == INTRA_FRAME) {
-    vp9_build_intra_predictors_sby_s(xd, bsize);
-    vp9_build_intra_predictors_sbuv_s(xd, bsize);
+#else
+  if (xd->mode_info_context->mbmi.ref_frame == INTRA_FRAME) {
+    vp9_encode_intra_block_y(cm, x, bsize);
+    vp9_encode_intra_block_uv(cm, x, bsize);
+#endif
     if (output_enabled)
       sum_intra_stats(cpi, x);
   } else {
@@ -1741,10 +1736,10 @@ static void encode_superblock(VP9_COMP *cpi, TOKENEXTRA **t,
   if (mbmi->ref_frame == INTRA_FRAME &&
       bsize < BLOCK_SIZE_SB8X8) {
 #else
-  if (mbmi->mode == I4X4_PRED) {
-    assert(bsize == BLOCK_SIZE_SB8X8);
+  if (xd->mode_info_context->mbmi.ref_frame == INTRA_FRAME) {
 #endif
-    vp9_tokenize_sb(cpi, xd, t, !output_enabled, BLOCK_SIZE_SB8X8);
+    vp9_tokenize_sb(cpi, xd, t, !output_enabled,
+                    (bsize < BLOCK_SIZE_SB8X8) ? BLOCK_SIZE_SB8X8 : bsize);
   } else if (!x->skip) {
     vp9_encode_sb(cm, x, (bsize < BLOCK_SIZE_SB8X8) ? BLOCK_SIZE_SB8X8 : bsize);
     vp9_tokenize_sb(cpi, xd, t, !output_enabled,
@@ -1771,8 +1766,9 @@ static void encode_superblock(VP9_COMP *cpi, TOKENEXTRA **t,
 
   if (output_enabled) {
     if (cm->txfm_mode == TX_MODE_SELECT &&
+        (mbmi->ref_frame == INTRA_FRAME ||
         !(mbmi->mb_skip_coeff ||
-          vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))) {
+          vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP)))) {
       if (bsize >= BLOCK_SIZE_SB32X32) {
         cpi->txfm_count_32x32p[mbmi->txfm_size]++;
       } else if (bsize >= BLOCK_SIZE_MB16X16) {
@@ -1783,18 +1779,23 @@ static void encode_superblock(VP9_COMP *cpi, TOKENEXTRA **t,
     } else {
       int x, y;
       TX_SIZE sz = (cm->txfm_mode == TX_MODE_SELECT) ? TX_32X32 : cm->txfm_mode;
-
-      if (sz == TX_32X32 && bsize < BLOCK_SIZE_SB32X32)
-        sz = TX_16X16;
-      if (sz == TX_16X16 && bsize < BLOCK_SIZE_MB16X16)
-        sz = TX_8X8;
+       // The new intra coding scheme requires no change of transform size
+      if (mi->mbmi.ref_frame != INTRA_FRAME) {
+        if (sz == TX_32X32 && bsize < BLOCK_SIZE_SB32X32)
+          sz = TX_16X16;
+        if (sz == TX_16X16 && bsize < BLOCK_SIZE_MB16X16)
+          sz = TX_8X8;
 #if CONFIG_AB4X4
-      if (sz == TX_8X8 && bsize < BLOCK_SIZE_SB8X8)
+        if (sz == TX_8X8 && bsize < BLOCK_SIZE_SB8X8)
 #else
-      if (sz == TX_8X8 && (mbmi->mode == SPLITMV ||
-                           mbmi->mode == I4X4_PRED))
+        if (sz == TX_8X8 && mbmi->mode == SPLITMV)
 #endif
+          sz = TX_4X4;
+      } else if (mbmi->mode != I4X4_PRED) {
+        sz = mbmi->txfm_size;
+      } else {
         sz = TX_4X4;
+      }
 
       for (y = 0; y < bh; y++) {
         for (x = 0; x < bw; x++) {
diff --git a/vp9/encoder/vp9_encodeintra.h b/vp9/encoder/vp9_encodeintra.h
index 22a046e35..7da164c6a 100644
--- a/vp9/encoder/vp9_encodeintra.h
+++ b/vp9/encoder/vp9_encodeintra.h
@@ -16,6 +16,9 @@
 int vp9_encode_intra(VP9_COMP *cpi, MACROBLOCK *x, int use_16x16_pred);
 void vp9_encode_intra16x16mby(VP9_COMMON *const cm, MACROBLOCK *x);
 void vp9_encode_intra16x16mbuv(VP9_COMMON *const cm, MACROBLOCK *x);
-void vp9_encode_intra4x4mby(VP9_COMMON *const cm, MACROBLOCK *mb,
-                            BLOCK_SIZE_TYPE bs);
+void vp9_encode_intra_block_y(VP9_COMMON *const cm, MACROBLOCK *mb,
+                              BLOCK_SIZE_TYPE bs);
+void vp9_encode_intra_block_uv(VP9_COMMON *const cm, MACROBLOCK *mb,
+                               BLOCK_SIZE_TYPE bs);
+
 #endif  // VP9_ENCODER_VP9_ENCODEINTRA_H_
diff --git a/vp9/encoder/vp9_encodemb.c b/vp9/encoder/vp9_encodemb.c
index d9cd09163..a4991f21a 100644
--- a/vp9/encoder/vp9_encodemb.c
+++ b/vp9/encoder/vp9_encodemb.c
@@ -628,16 +628,23 @@ static void encode_block_intra(int plane, int block, BLOCK_SIZE_TYPE bsize,
 
   const int txfm_b_size = 4 << tx_size;
   int ib = raster_block;
+  int tx_ib = ib >> tx_size;
+  int plane_b_size;
 
   TX_TYPE tx_type;
+  int mode, b_mode;
 
-  if (tx_size <= TX_16X16)
-    tx_type = txfm_map(xd->mode_info_context->bmi[ib].as_mode.first);
+  mode = plane == 0? xd->mode_info_context->mbmi.mode:
+                     xd->mode_info_context->mbmi.uv_mode;
+  if (bsize <= BLOCK_SIZE_SB8X8 && mode == I4X4_PRED && plane == 0)
+    b_mode = xd->mode_info_context->bmi[ib].as_mode.first;
   else
-    tx_type = DCT_DCT;
+    b_mode = mode;
 
-  vp9_predict_intra_block(&x->e_mbd, ib, bsize, tx_size,
-                          xd->mode_info_context->bmi[ib].as_mode.first,
+  assert(b_mode >= B_DC_PRED && b_mode <= B_TM_PRED);
+
+  plane_b_size = b_width_log2(bsize) - xd->plane[plane].subsampling_x;
+  vp9_predict_intra_block(xd, tx_ib, plane_b_size, tx_size, b_mode,
                           dst, xd->plane[plane].dst.stride);
   vp9_subtract_block(txfm_b_size, txfm_b_size,
                      src_diff, bw,
@@ -650,7 +657,6 @@ static void encode_block_intra(int plane, int block, BLOCK_SIZE_TYPE bsize,
   if (x->optimize)
     vp9_optimize_b(plane, block, bsize, ss_txfrm_size, args->cm, x, args->ctx);
     */
-
   switch (ss_txfrm_size / 2) {
     case TX_32X32:
         vp9_short_idct32x32_add(BLOCK_OFFSET(xd->plane[plane].dqcoeff,
@@ -695,8 +701,8 @@ static void encode_block_intra(int plane, int block, BLOCK_SIZE_TYPE bsize,
   }
 }
 
-void vp9_encode_intra4x4mby(VP9_COMMON *const cm, MACROBLOCK *x,
-                            BLOCK_SIZE_TYPE bsize) {
+void vp9_encode_intra_block_y(VP9_COMMON *const cm, MACROBLOCK *x,
+                              BLOCK_SIZE_TYPE bsize) {
   MACROBLOCKD* const xd = &x->e_mbd;
   struct optimize_ctx ctx;
   struct encode_b_args arg = {cm, x, &ctx};
@@ -704,4 +710,12 @@ void vp9_encode_intra4x4mby(VP9_COMMON *const cm, MACROBLOCK *x,
   foreach_transformed_block_in_plane(xd, bsize, 0,
                                      encode_block_intra, &arg);
 }
+void vp9_encode_intra_block_uv(VP9_COMMON *const cm, MACROBLOCK *x,
+                              BLOCK_SIZE_TYPE bsize) {
+  MACROBLOCKD* const xd = &x->e_mbd;
+  struct optimize_ctx ctx;
+  struct encode_b_args arg = {cm, x, &ctx};
+
+  foreach_transformed_block_uv(xd, bsize, encode_block_intra, &arg);
+}
 
diff --git a/vp9/encoder/vp9_onyx_if.c b/vp9/encoder/vp9_onyx_if.c
index a8e1a1727..e3c7930ab 100644
--- a/vp9/encoder/vp9_onyx_if.c
+++ b/vp9/encoder/vp9_onyx_if.c
@@ -3170,6 +3170,15 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
   // Pick the loop filter level for the frame.
   loopfilter_frame(cpi, cm);
 
+#if WRITE_RECON_BUFFER
+  if (cm->show_frame)
+    write_cx_frame_to_file(cm->frame_to_show,
+                           cm->current_video_frame + 2000);
+  else
+    write_cx_frame_to_file(cm->frame_to_show,
+                           cm->current_video_frame + 3000);
+#endif
+
   // build the bitstream
   cpi->dummy_packing = 0;
   vp9_pack_bitstream(cpi, dest, size);