Enable variable block size test in non-RD mode decision

This is an initial attempt to allow variable block size partition
in non-RD coding flow. It tests 8x8, 16x16 and 32x32 block size per
64x64 block, all using non-RD mode decision and the associated rate
distortion costs from modeling, then selects the best block size to
encode the entire 64x64 block. Such operations are triggered every
other 3 frames. The blocks of intermediate frames will reuse the
collocated block's partition type.

It improves the compression performance by 13.2%. Note that the gains
are not evenly distributed. For many hard clips, the compression
performance is improved by 20% to 28%. Local speed test shows that
it will also increase runtime by 50%, as compared to speed -7. It is
now enabled in speed -6 setting.

Change-Id: Ib4fb8659d21621c9075b3c369ddaa9ecb0a4b204

1 files changed, 44 insertions, 0 deletions

diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index a38592240..257abdea5 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -2693,6 +2693,7 @@ static void nonrd_pick_sb_modes(VP9_COMP *cpi, const TileInfo *const tile,
   MACROBLOCKD *const xd = &x->e_mbd;
   set_offsets(cpi, tile, mi_row, mi_col, bsize);
   xd->mi_8x8[0]->mbmi.sb_type = bsize;
+
   if (!frame_is_intra_only(cm)) {
     vp9_pick_inter_mode(cpi, x, tile, mi_row, mi_col,
                         rate, dist, bsize);
@@ -2825,6 +2826,8 @@ static void encode_nonrd_sb_row(VP9_COMP *cpi, const TileInfo *const tile,
     int64_t dummy_dist;
     const int idx_str = cm->mode_info_stride * mi_row + mi_col;
     MODE_INFO **mi_8x8 = cm->mi_grid_visible + idx_str;
+    MODE_INFO **prev_mi_8x8 = cm->prev_mi_grid_visible + idx_str;
+
     BLOCK_SIZE bsize = cpi->sf.partition_search_type == FIXED_PARTITION ?
         cpi->sf.always_this_block_size :
         get_nonrd_var_based_fixed_partition(cpi, mi_row, mi_col);
@@ -2834,6 +2837,47 @@ static void encode_nonrd_sb_row(VP9_COMP *cpi, const TileInfo *const tile,
     // Set the partition type of the 64X64 block
     if (cpi->sf.partition_search_type == VAR_BASED_PARTITION)
       choose_partitioning(cpi, tile, mi_row, mi_col);
+    else if (cpi->sf.partition_search_type == REFERENCE_PARTITION) {
+      if (cpi->sf.partition_check) {
+        MACROBLOCK *x = &cpi->mb;
+        int rate1, rate2, rate3;
+        int64_t dist1, dist2, dist3;
+        set_fixed_partitioning(cpi, tile, mi_8x8, mi_row, mi_col, BLOCK_8X8);
+        nonrd_use_partition(cpi, tile, mi_8x8, tp, mi_row, mi_col, BLOCK_64X64,
+                            0, &rate1, &dist1);
+        set_fixed_partitioning(cpi, tile, mi_8x8, mi_row, mi_col, BLOCK_16X16);
+        nonrd_use_partition(cpi, tile, mi_8x8, tp, mi_row, mi_col, BLOCK_64X64,
+                            0, &rate2, &dist2);
+        set_fixed_partitioning(cpi, tile, mi_8x8, mi_row, mi_col, BLOCK_32X32);
+        nonrd_use_partition(cpi, tile, mi_8x8, tp, mi_row, mi_col, BLOCK_64X64,
+                            0, &rate3, &dist3);
+
+        if (RDCOST(x->rdmult, x->rddiv, rate1, dist1) <
+            RDCOST(x->rdmult, x->rddiv, rate2, dist2)) {
+          if (RDCOST(x->rdmult, x->rddiv, rate1, dist1) <
+              RDCOST(x->rdmult, x->rddiv, rate3, dist3))
+            set_fixed_partitioning(cpi, tile, mi_8x8, mi_row, mi_col,
+                                   BLOCK_8X8);
+          else
+            set_fixed_partitioning(cpi, tile, mi_8x8, mi_row, mi_col,
+                                   BLOCK_32X32);
+        } else {
+          if (RDCOST(x->rdmult, x->rddiv, rate2, dist2) <
+              RDCOST(x->rdmult, x->rddiv, rate3, dist3))
+            set_fixed_partitioning(cpi, tile, mi_8x8, mi_row, mi_col,
+                                   BLOCK_16X16);
+          else
+            set_fixed_partitioning(cpi, tile, mi_8x8, mi_row, mi_col,
+                                   BLOCK_32X32);
+        }
+
+      } else {
+        if (!sb_has_motion(cm, prev_mi_8x8))
+          copy_partitioning(cm, mi_8x8, prev_mi_8x8);
+        else
+          set_fixed_partitioning(cpi, tile, mi_8x8, mi_row, mi_col, bsize);
+      }
+    }
     else
       set_fixed_partitioning(cpi, tile, mi_8x8, mi_row, mi_col, bsize);