changes intra coding to be based on txfm block

This commit changed the encoding and decoding of intra blocks to be
based on transform block. In each prediction block, the intra coding
iterates thorough each transform block based on raster scan order.

This commit also fixed a bug in D135 prediction code.

TODO next:
The RD mode/txfm_size selection should take this into account when
computing RD values.

Change-Id: I6d1be2faa4c4948a52e830b6a9a84a6b2b6850f6

3 files changed, 93 insertions, 8 deletions

diff --git a/vp9/decoder/vp9_decodemv.c b/vp9/decoder/vp9_decodemv.c
index ac66fb644..1ddd21c12 100644
--- a/vp9/decoder/vp9_decodemv.c
+++ b/vp9/decoder/vp9_decodemv.c
@@ -154,11 +154,11 @@ static void kfread_modes(VP9D_COMP *pbi, MODE_INFO *m,
   m->mbmi.uv_mode = read_uv_mode(r, cm->kf_uv_mode_prob[m->mbmi.mode]);
 
   if (cm->txfm_mode == TX_MODE_SELECT &&
-      !m->mbmi.mb_skip_coeff &&
+    !(m->mbmi.mb_skip_coeff && m->mbmi.ref_frame != INTRA_FRAME)
 #if CONFIG_AB4X4
-      m->mbmi.sb_type >= BLOCK_SIZE_SB8X8
+      && m->mbmi.sb_type >= BLOCK_SIZE_SB8X8
 #else
-      m->mbmi.mode != I4X4_PRED
+      && m->mbmi.mode != I4X4_PRED
 #endif
       ) {
     const int allow_16x16 = m->mbmi.sb_type >= BLOCK_SIZE_MB16X16;
@@ -880,7 +880,8 @@ static void read_mb_modes_mv(VP9D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi,
   if (cm->txfm_mode == TX_MODE_SELECT && mbmi->mb_skip_coeff == 0 &&
       bsize >= BLOCK_SIZE_SB8X8) {
 #else
-  if (cm->txfm_mode == TX_MODE_SELECT && mbmi->mb_skip_coeff == 0 &&
+  if (cm->txfm_mode == TX_MODE_SELECT &&
+      (mbmi->mb_skip_coeff == 0 || mbmi->ref_frame == INTRA_FRAME) &&
       ((mbmi->ref_frame == INTRA_FRAME && mbmi->mode != I4X4_PRED) ||
        (mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) {
 #endif
diff --git a/vp9/decoder/vp9_decodframe.c b/vp9/decoder/vp9_decodframe.c
index 449e4269e..eea7658a8 100644
--- a/vp9/decoder/vp9_decodframe.c
+++ b/vp9/decoder/vp9_decodframe.c
@@ -239,6 +239,59 @@ static void decode_block(int plane, int block, BLOCK_SIZE_TYPE bsize,
   }
 }
 
+static void decode_block_intra(int plane, int block, BLOCK_SIZE_TYPE bsize,
+                               int ss_txfrm_size, void *arg) {
+  MACROBLOCKD* const xd = arg;
+  int16_t* const qcoeff = BLOCK_OFFSET(xd->plane[plane].qcoeff, block, 16);
+  const int stride = xd->plane[plane].dst.stride;
+  const int raster_block = txfrm_block_to_raster_block(xd, bsize, plane,
+                                                       block, ss_txfrm_size);
+  uint8_t* const dst = raster_block_offset_uint8(xd, bsize, plane,
+                                                 raster_block,
+                                                 xd->plane[plane].dst.buf,
+                                                 stride);
+  const TX_SIZE tx_size = (TX_SIZE)(ss_txfrm_size / 2);
+  TX_TYPE tx_type;
+  int mode, b_mode;
+  int plane_b_size;
+  int tx_ib = raster_block >> tx_size;
+  mode = plane == 0? xd->mode_info_context->mbmi.mode:
+                     xd->mode_info_context->mbmi.uv_mode;
+
+  if (bsize <= BLOCK_SIZE_SB8X8 && mode == I4X4_PRED && plane == 0)
+    b_mode = xd->mode_info_context->bmi[raster_block].as_mode.first;
+  else
+    b_mode = mode;
+
+  plane_b_size = b_width_log2(bsize) - xd->plane[plane].subsampling_x;
+  vp9_predict_intra_block(xd, tx_ib, plane_b_size, tx_size,
+                          b_mode, dst, xd->plane[plane].dst.stride);
+
+  switch (ss_txfrm_size / 2) {
+    case TX_4X4:
+      tx_type = plane == 0 ? get_tx_type_4x4(xd, raster_block) : DCT_DCT;
+      if (tx_type == DCT_DCT)
+        xd->itxm_add(qcoeff, dst, stride, xd->plane[plane].eobs[block]);
+      else
+        vp9_iht_add_c(tx_type, qcoeff, dst, stride,
+                      xd->plane[plane].eobs[block]);
+      break;
+    case TX_8X8:
+      tx_type = plane == 0 ? get_tx_type_8x8(xd, raster_block) : DCT_DCT;
+      vp9_iht_add_8x8_c(tx_type, qcoeff, dst, stride,
+                        xd->plane[plane].eobs[block]);
+      break;
+    case TX_16X16:
+      tx_type = plane == 0 ? get_tx_type_16x16(xd, raster_block) : DCT_DCT;
+      vp9_iht_add_16x16_c(tx_type, qcoeff, dst, stride,
+                          xd->plane[plane].eobs[block]);
+      break;
+    case TX_32X32:
+      vp9_idct_add_32x32(qcoeff, dst, stride, xd->plane[plane].eobs[block]);
+      break;
+  }
+}
+
 static void decode_atom_intra(VP9D_COMP *pbi, MACROBLOCKD *xd,
                               vp9_reader *r,
                               BLOCK_SIZE_TYPE bsize) {
@@ -295,6 +348,26 @@ static void decode_atom(VP9D_COMP *pbi, MACROBLOCKD *xd,
     foreach_transformed_block(xd, bsize, decode_block, xd);
 }
 
+static void decode_sb_intra(VP9D_COMP *pbi, MACROBLOCKD *xd,
+                          int mi_row, int mi_col,
+                          vp9_reader *r, BLOCK_SIZE_TYPE bsize) {
+  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
+  if (mbmi->mb_skip_coeff) {
+    vp9_reset_sb_tokens_context(xd, bsize);
+  } else {
+    // re-initialize macroblock dequantizer before detokenization
+    if (xd->segmentation_enabled)
+      mb_init_dequantizer(&pbi->common, xd);
+
+    if (!vp9_reader_has_error(r)) {
+      vp9_decode_tokens(pbi, xd, r, bsize);
+    }
+  }
+
+  foreach_transformed_block(xd, bsize, decode_block_intra, xd);
+}
+
+
 static void decode_sb(VP9D_COMP *pbi, MACROBLOCKD *xd, int mi_row, int mi_col,
                       vp9_reader *r, BLOCK_SIZE_TYPE bsize) {
   const int bwl = mi_width_log2(bsize), bhl = mi_height_log2(bsize);
@@ -422,12 +495,13 @@ static void decode_modes_b(VP9D_COMP *pbi, int mi_row, int mi_col,
   vp9_decode_mb_mode_mv(pbi, xd, mi_row, mi_col, r);
   set_refs(pbi, mi_row, mi_col);
 
+  if (xd->mode_info_context->mbmi.ref_frame == INTRA_FRAME)
+    decode_sb_intra(pbi, xd, mi_row, mi_col, r, bsize);
 #if CONFIG_AB4X4
-  if (bsize < BLOCK_SIZE_SB8X8)
+  else if (bsize < BLOCK_SIZE_SB8X8)
 #else
-  if (bsize == BLOCK_SIZE_SB8X8 &&
-      (xd->mode_info_context->mbmi.mode == SPLITMV ||
-       xd->mode_info_context->mbmi.mode == I4X4_PRED))
+  else if (bsize == BLOCK_SIZE_SB8X8 &&
+      xd->mode_info_context->mbmi.mode == SPLITMV)
 #endif
     decode_atom(pbi, xd, mi_row, mi_col, r, BLOCK_SIZE_SB8X8);
   else
diff --git a/vp9/decoder/vp9_onyxd_if.c b/vp9/decoder/vp9_onyxd_if.c
index 9761bd6f8..5d5a543a3 100644
--- a/vp9/decoder/vp9_onyxd_if.c
+++ b/vp9/decoder/vp9_onyxd_if.c
@@ -352,6 +352,16 @@ int vp9_receive_compressed_data(VP9D_PTR ptr,
       vp9_loop_filter_frame(cm, &pbi->mb, cm->filter_level, 0,
                             cm->dering_enabled);
     }
+
+#if WRITE_RECON_BUFFER == 2
+    if (cm->show_frame)
+      write_dx_frame_to_file(cm->frame_to_show,
+                             cm->current_video_frame + 2000);
+    else
+      write_dx_frame_to_file(cm->frame_to_show,
+                             cm->current_video_frame + 3000);
+#endif
+
     vp9_extend_frame_borders(cm->frame_to_show,
                              cm->subsampling_x, cm->subsampling_y);
   }