Modify block transform skipping check

Block transform skipping was implemented based on DCT's energy
conservation property. Modified the thresholds using zero bin
parameters. AC and DC coefficients were checked separately to
allow better identifying of skippable blocks.

Borg test at speed 3 showed:
stdhd set: psnr gain: 0.153%, ssim gain: 0.051%;
derf set: psnr gain: 0.023%, ssim gain: 0.036%

For most test clips, the encoding speedup is 1% - 2%.
parkrun(720p): 7.5% speedup, park_joy(1080p): 3.5% speedup.

Change-Id: If28eb81113a077414f5ca7b021c14f9069b373bb

2 files changed, 30 insertions, 16 deletions

diff --git a/vp9/encoder/vp9_quantize.c b/vp9/encoder/vp9_quantize.c
index 3d2c40970..0aeeef01f 100644
--- a/vp9/encoder/vp9_quantize.c
+++ b/vp9/encoder/vp9_quantize.c
@@ -641,13 +641,14 @@ void vp9_init_plane_quantizers(VP9_COMP *cpi, MACROBLOCK *x) {
   x->plane[0].quant_shift = quants->y_quant_shift[qindex];
   x->plane[0].zbin = quants->y_zbin[qindex];
   x->plane[0].round = quants->y_round[qindex];
-  x->plane[0].quant_thred[0] = cm->y_dequant[qindex][0] *
-                                  cm->y_dequant[qindex][0];
-  x->plane[0].quant_thred[1] = cm->y_dequant[qindex][1] *
-                                  cm->y_dequant[qindex][1];
   x->plane[0].zbin_extra = (int16_t)((cm->y_dequant[qindex][1] * zbin) >> 7);
   xd->plane[0].dequant = cm->y_dequant[qindex];
 
+  x->plane[0].quant_thred[0] = (x->plane[0].zbin[0] + x->plane[0].zbin_extra) *
+      (x->plane[0].zbin[0] + x->plane[0].zbin_extra);
+  x->plane[0].quant_thred[1] = (x->plane[0].zbin[1] + x->plane[0].zbin_extra) *
+      (x->plane[0].zbin[1] + x->plane[0].zbin_extra);
+
   // UV
   for (i = 1; i < 3; i++) {
     x->plane[i].quant = quants->uv_quant[qindex];
@@ -656,12 +657,15 @@ void vp9_init_plane_quantizers(VP9_COMP *cpi, MACROBLOCK *x) {
     x->plane[i].quant_shift = quants->uv_quant_shift[qindex];
     x->plane[i].zbin = quants->uv_zbin[qindex];
     x->plane[i].round = quants->uv_round[qindex];
-    x->plane[i].quant_thred[0] = cm->y_dequant[qindex][0] *
-                                    cm->y_dequant[qindex][0];
-    x->plane[i].quant_thred[1] = cm->y_dequant[qindex][1] *
-                                    cm->y_dequant[qindex][1];
     x->plane[i].zbin_extra = (int16_t)((cm->uv_dequant[qindex][1] * zbin) >> 7);
     xd->plane[i].dequant = cm->uv_dequant[qindex];
+
+    x->plane[i].quant_thred[0] =
+        (x->plane[i].zbin[0] + x->plane[i].zbin_extra) *
+        (x->plane[i].zbin[0] + x->plane[i].zbin_extra);
+    x->plane[i].quant_thred[1] =
+        (x->plane[i].zbin[1] + x->plane[i].zbin_extra) *
+        (x->plane[i].zbin[1] + x->plane[i].zbin_extra);
   }
 
   x->skip_block = vp9_segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP);
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index bf0de4b29..be7c18e1b 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -180,7 +180,7 @@ static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize,
   unsigned int sse;
   unsigned int var = 0;
   unsigned int sum_sse = 0;
-  const int shift = 8;
+  const int shift = 6;
   int rate;
   int64_t dist;
 
@@ -212,12 +212,16 @@ static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize,
         sum_sse += sse;
 
         if (!x->select_tx_size) {
-          if (x->bsse[(i << 2) + block_idx] < p->quant_thred[0] >> shift)
-            x->skip_txfm[(i << 2) + block_idx] = 1;
-          else if (var < p->quant_thred[1] >> shift)
+          // Check if all ac coefficients can be quantized to zero.
+          if (var < p->quant_thred[1] >> shift) {
             x->skip_txfm[(i << 2) + block_idx] = 2;
-          else
+
+            // Check if dc coefficient can be quantized to zero.
+            if (sse - var < p->quant_thred[0] >> shift)
+              x->skip_txfm[(i << 2) + block_idx] = 1;
+          } else {
             x->skip_txfm[(i << 2) + block_idx] = 0;
+          }
         }
 
         if (i == 0)
@@ -484,9 +488,15 @@ static void block_rd_txfm(int plane, int block, BLOCK_SIZE plane_bsize,
       vp9_xform_quant_dc(x, plane, block, plane_bsize, tx_size);
       args->sse  = x->bsse[(plane << 2) + (block >> (tx_size << 1))] << 4;
       args->dist = args->sse;
-      if (!x->plane[plane].eobs[block])
-        args->dist = args->sse - ((coeff[0] * coeff[0] -
-            (coeff[0] - dqcoeff[0]) * (coeff[0] - dqcoeff[0])) >> 2);
+      if (x->plane[plane].eobs[block]) {
+        int64_t dc_correct = coeff[0] * coeff[0] -
+            (coeff[0] - dqcoeff[0]) * (coeff[0] - dqcoeff[0]);
+
+        if (tx_size != TX_32X32)
+          dc_correct >>= 2;
+
+        args->dist = args->sse - dc_correct;
+      }
     } else {
       // skip forward transform
       x->plane[plane].eobs[block] = 0;