Modify the setting of transform skip flags in non-rd mode

While searching for the best mode in non-rd case, SSE of
a partition block is calculated and the transform size is set.
This patch rewrites the skip checking conditions based on
transform size instead of partition size to be more precise.

Small gains were seen in rtc set borg test (speed 6).
AVG PSNR: 0.087%, overall PSNR: 0.073%, SSIM: 0.146%.
No noticeable speed change.

Change-Id: I5603ca5339c784dfa02263f4005988ccd8c32f6e

1 files changed, 26 insertions, 12 deletions

diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index 2f9cccb99..4a6074e51 100644
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -224,15 +224,6 @@ static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
   *var_y = var;
   *sse_y = sse;
 
-  x->skip_txfm[0] = 0;
-  // Check if all ac coefficients can be quantized to zero.
-  if (var < ac_thr || var == 0) {
-    x->skip_txfm[0] = 2;
-    // Check if dc coefficient can be quantized to zero.
-    if (sse - var < dc_thr || sse == var)
-      x->skip_txfm[0] = 1;
-  }
-
   if (cpi->common.tx_mode == TX_MODE_SELECT) {
     if (sse > (var << 2))
       xd->mi[0].src_mi->mbmi.tx_size =
@@ -254,6 +245,32 @@ static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
             tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
   }
 
+  // Evaluate if the partition block is a skippable block in Y plane.
+  {
+    const BLOCK_SIZE unit_size =
+        txsize_to_bsize[xd->mi[0].src_mi->mbmi.tx_size];
+    const unsigned int num_blk_log2 =
+        (b_width_log2_lookup[bsize] - b_width_log2_lookup[unit_size]) +
+        (b_height_log2_lookup[bsize] - b_height_log2_lookup[unit_size]);
+    const unsigned int sse_tx = sse >> num_blk_log2;
+    const unsigned int var_tx = var >> num_blk_log2;
+
+    x->skip_txfm[0] = 0;
+    // Check if all ac coefficients can be quantized to zero.
+    if (var_tx < ac_thr || var == 0) {
+      x->skip_txfm[0] = 2;
+      // Check if dc coefficient can be quantized to zero.
+      if (sse_tx - var_tx < dc_thr || sse == var)
+        x->skip_txfm[0] = 1;
+    }
+  }
+
+  if (x->skip_txfm[0] == 1) {
+    *out_rate_sum = 0;
+    *out_dist_sum = sse << 4;
+    return;
+  }
+
 #if CONFIG_VP9_HIGHBITDEPTH
   if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
     vp9_model_rd_from_var_lapndz(sse - var, num_pels_log2_lookup[bsize],
@@ -285,9 +302,6 @@ static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
 
   *out_rate_sum += rate;
   *out_dist_sum += dist << 4;
-
-  if (*out_rate_sum == 0)
-    x->skip_txfm[0] = 1;
 }
 
 static void model_rd_for_sb_uv(VP9_COMP *cpi, BLOCK_SIZE bsize,