hybrid transform of 16x16 dimension

Enable ADST/DCT of dimension 16x16 for I16X16 modes. This change provides
benefits mostly for hd sequences.

Set up the framework for selectable transform dimension.

Also allowing quantization parameter threshold to control the use
of hybrid transform (This is currently disabled by setting threshold
always above the quantization parameter. Adaptive thresholding can
be built upon this, which will further improve the coding performance.)

The coding performance gains (with respect to the codec that has all
other configuration settings turned on) are

derf:   0.013
yt:     0.086
hd:     0.198
std-hd: 0.501

Change-Id: Ibb4263a61fc74e0b3c345f54d73e8c73552bf926

1 files changed, 97 insertions, 96 deletions

diff --git a/vp8/common/default_coef_probs.h b/vp8/common/default_coef_probs.h
index 940e971b7..3fcacb278 100644
--- a/vp8/common/default_coef_probs.h
+++ b/vp8/common/default_coef_probs.h
@@ -488,7 +488,7 @@ vp8_default_coef_probs_8x8[BLOCK_TYPES_8X8]
   }
 #endif
 };
-#if CONFIG_TX16X16
+#if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
 static const vp8_prob
 vp8_default_coef_probs_16x16[BLOCK_TYPES_16X16]
                             [COEF_BANDS]
@@ -546,103 +546,104 @@ vp8_default_coef_probs_16x16[BLOCK_TYPES_16X16]
     }
   },
   { /* block Type 1 */
-    { /* Coeff Band 0 */
-      { 134, 152, 233, 224, 234, 52, 255, 166, 128, 128, 128},
-      { 97, 132, 185, 234, 186, 189, 197, 171, 255, 212, 128},
-      { 84, 110, 185, 237, 182, 182, 145, 145, 255, 255, 128}
-    },
-    { /* Coeff Band 1 */
-      { 1, 124, 213, 247, 192, 212, 255, 255, 128, 128, 128},
-      { 88, 111, 178, 254, 189, 211, 255, 255, 128, 128, 128},
-      { 12, 59, 129, 236, 150, 179, 239, 195, 255, 255, 128},
-      { 12, 59, 129, 236, 150, 179, 239, 195, 255, 255, 128}
-    },
-    { /* Coeff Band 2 */
-      { 1, 102, 225, 255, 210, 240, 128, 128, 128, 128, 128},
-      { 110, 78, 195, 254, 200, 191, 255, 255, 128, 128, 128},
-      { 37, 63, 177, 255, 194, 195, 128, 128, 128, 128, 128},
-      { 37, 63, 177, 255, 194, 195, 128, 128, 128, 128, 128}
-    },
-    { /* Coeff Band 3 */
-      { 1, 1, 229, 255, 202, 224, 128, 128, 128, 128, 128},
-      { 150, 1, 192, 255, 206, 226, 128, 128, 128, 128, 128},
-      { 75, 1, 138, 255, 172, 228, 128, 128, 128, 128, 128},
-      { 75, 1, 138, 255, 172, 228, 128, 128, 128, 128, 128}
-    },
-    { /* Coeff Band 4 */
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}
-    },
-    { /* Coeff Band 5 */
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}
-    },
-    { /* Coeff Band 6 */
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}
-    },
-    { /* Coeff Band 7 */
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
-      { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}
-    }
+      { /* Coeff Band 0 */
+        { 1, 30, 103, 204, 142, 168, 235, 161, 255, 228, 128},
+        { 1, 35, 90, 192, 130, 161, 227, 158, 255, 226, 255},
+        { 1, 36, 78, 180, 122, 156, 221, 153, 255, 222, 255},
+        { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}
+      },
+      { /* Coeff Band 1 */
+        { 1, 163, 228, 253, 212, 194, 255, 205, 128, 128, 128},
+        { 67, 160, 226, 253, 210, 202, 245, 172, 255, 255, 128},
+        { 51, 147, 219, 251, 207, 207, 255, 217, 128, 128, 128},
+        { 25, 107, 175, 245, 183, 190, 254, 209, 255, 255, 128}
+      },
+      { /* Coeff Band 2 */
+        { 1, 66, 170, 240, 177, 186, 252, 203, 255, 245, 128},
+        { 23, 64, 145, 230, 161, 177, 252, 198, 255, 255, 128},
+        { 6, 51, 99, 208, 135, 163, 249, 178, 255, 248, 128},
+        { 1, 33, 59, 161, 104, 151, 238, 164, 255, 237, 128}
+      },
+      { /* Coeff Band 3 */
+        { 1, 76, 216, 250, 198, 199, 255, 226, 255, 255, 128},
+        { 86, 83, 200, 247, 189, 193, 255, 224, 255, 255, 128},
+        { 30, 75, 164, 242, 172, 184, 254, 218, 255, 255, 128},
+        { 3, 54, 103, 227, 140, 172, 253, 201, 255, 255, 128}
+      },
+      { /* Coeff Band 4 */
+        { 1, 241, 247, 255, 233, 223, 255, 255, 128, 128, 128},
+        { 78, 212, 242, 255, 226, 230, 255, 255, 128, 128, 128},
+        { 10, 167, 224, 255, 217, 225, 255, 128, 128, 128, 128},
+        { 1, 104, 176, 250, 166, 219, 255, 255, 128, 128, 128}
+      },
+      { /* Coeff Band 5 */
+        { 1, 194, 241, 254, 228, 214, 248, 237, 255, 255, 128},
+        { 95, 133, 228, 254, 218, 215, 255, 229, 128, 128, 128},
+        { 24, 119, 201, 252, 202, 205, 255, 229, 128, 128, 128},
+        { 1, 88, 155, 246, 183, 193, 255, 205, 128, 128, 128}
+      },
+      { /* Coeff Band 6 */
+        { 1, 204, 236, 255, 222, 220, 255, 239, 128, 128, 128},
+        { 126, 105, 225, 254, 214, 217, 255, 254, 128, 128, 128},
+        { 44, 86, 190, 251, 197, 204, 255, 233, 128, 128, 128},
+        { 6, 71, 130, 240, 164, 188, 255, 246, 128, 128, 128}
+      },
+      { /* Coeff Band 7 */
+        { 1, 195, 250, 255, 239, 197, 128, 128, 128, 128, 128},
+        { 167, 102, 249, 255, 234, 255, 128, 128, 128, 128, 128},
+        { 65, 91, 222, 255, 217, 255, 128, 128, 128, 128, 128},
+        { 1, 59, 128, 255, 154, 255, 128, 128, 128, 128, 128}
+      }
   },
   { /* block Type 2 */
-    { /* Coeff Band 0 */
-      { 11, 181, 226, 199, 183, 255, 255, 255, 128, 128, 128},
-      { 2, 147, 185, 248, 163, 180, 255, 236, 128, 128, 128},
-      { 1, 123, 157, 238, 154, 176, 255, 226, 255, 255, 128},
-      { 1, 123, 157, 238, 154, 176, 255, 226, 255, 255, 128}
-    },
-    { /* Coeff Band 1 */
-      { 1, 150, 191, 246, 174, 188, 255, 235, 128, 128, 128},
-      { 1, 125, 166, 245, 165, 185, 255, 234, 128, 128, 128},
-      { 1, 79, 125, 240, 148, 179, 255, 234, 255, 255, 128},
-      { 1, 79, 125, 240, 148, 179, 255, 234, 255, 255, 128}
-    },
-    { /* Coeff Band 2 */
-      { 1, 146, 184, 242, 167, 183, 255, 230, 255, 255, 128},
-      { 1, 119, 160, 239, 156, 178, 255, 231, 255, 255, 128},
-      { 1, 75, 115, 234, 142, 173, 255, 225, 255, 255, 128},
-      { 1, 75, 115, 234, 142, 173, 255, 225, 255, 255, 128}
-    },
-    { /* Coeff Band 3 */
-      { 1, 150, 188, 244, 169, 183, 255, 233, 255, 255, 128},
-      { 1, 123, 162, 243, 161, 180, 255, 233, 128, 128, 128},
-      { 1, 76, 120, 238, 148, 178, 255, 230, 255, 255, 128},
-      { 1, 76, 120, 238, 148, 178, 255, 230, 255, 255, 128}
-    },
-    { /* Coeff Band 4 */
-      { 1, 163, 202, 252, 188, 204, 255, 248, 128, 128, 128},
-      { 1, 136, 180, 251, 181, 201, 255, 246, 128, 128, 128},
-      { 1, 92, 146, 249, 170, 197, 255, 245, 128, 128, 128},
-      { 1, 92, 146, 249, 170, 197, 255, 245, 128, 128, 128}
-    },
-    { /* Coeff Band 5 */
-      { 1, 156, 195, 249, 179, 193, 255, 241, 255, 255, 128},
-      { 1, 128, 169, 248, 171, 192, 255, 242, 255, 255, 128},
-      { 1, 84, 132, 245, 158, 187, 255, 240, 255, 255, 128},
-      { 1, 84, 132, 245, 158, 187, 255, 240, 255, 255, 128}
-    },
-    { /* Coeff Band 6 */
-      { 1, 36, 71, 251, 192, 201, 255, 243, 255, 255, 128},
-      { 1, 49, 185, 250, 184, 199, 255, 242, 128, 128, 128},
-      { 1, 95, 147, 247, 168, 190, 255, 239, 255, 255, 128},
-      { 1, 95, 147, 247, 168, 190, 255, 239, 255, 255, 128}
-    },
-    { /* Coeff Band 7 */
-      { 1, 19, 98, 255, 218, 222, 255, 255, 128, 128, 128},
-      { 36, 50, 210, 255, 212, 221, 255, 255, 128, 128, 128},
-      { 6, 117, 180, 254, 199, 216, 255, 251, 128, 128, 128},
-      { 6, 117, 180, 254, 199, 216, 255, 251, 128, 128, 128}
-    }
+      { /* Coeff Band 0 */
+        { 1, 30, 103, 204, 142, 168, 235, 161, 255, 228, 128},
+        { 1, 35, 90, 192, 130, 161, 227, 158, 255, 226, 255},
+        { 1, 36, 78, 180, 122, 156, 221, 153, 255, 222, 255},
+        { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}
+      },
+      { /* Coeff Band 1 */
+        { 1, 163, 228, 253, 212, 194, 255, 205, 128, 128, 128},
+        { 67, 160, 226, 253, 210, 202, 245, 172, 255, 255, 128},
+        { 51, 147, 219, 251, 207, 207, 255, 217, 128, 128, 128},
+        { 25, 107, 175, 245, 183, 190, 254, 209, 255, 255, 128}
+      },
+      { /* Coeff Band 2 */
+        { 1, 66, 170, 240, 177, 186, 252, 203, 255, 245, 128},
+        { 23, 64, 145, 230, 161, 177, 252, 198, 255, 255, 128},
+        { 6, 51, 99, 208, 135, 163, 249, 178, 255, 248, 128},
+        { 1, 33, 59, 161, 104, 151, 238, 164, 255, 237, 128}
+      },
+      { /* Coeff Band 3 */
+        { 1, 76, 216, 250, 198, 199, 255, 226, 255, 255, 128},
+        { 86, 83, 200, 247, 189, 193, 255, 224, 255, 255, 128},
+        { 30, 75, 164, 242, 172, 184, 254, 218, 255, 255, 128},
+        { 3, 54, 103, 227, 140, 172, 253, 201, 255, 255, 128}
+      },
+      { /* Coeff Band 4 */
+        { 1, 241, 247, 255, 233, 223, 255, 255, 128, 128, 128},
+        { 78, 212, 242, 255, 226, 230, 255, 255, 128, 128, 128},
+        { 10, 167, 224, 255, 217, 225, 255, 128, 128, 128, 128},
+        { 1, 104, 176, 250, 166, 219, 255, 255, 128, 128, 128}
+      },
+      { /* Coeff Band 5 */
+        { 1, 194, 241, 254, 228, 214, 248, 237, 255, 255, 128},
+        { 95, 133, 228, 254, 218, 215, 255, 229, 128, 128, 128},
+        { 24, 119, 201, 252, 202, 205, 255, 229, 128, 128, 128},
+        { 1, 88, 155, 246, 183, 193, 255, 205, 128, 128, 128}
+      },
+      { /* Coeff Band 6 */
+        { 1, 204, 236, 255, 222, 220, 255, 239, 128, 128, 128},
+        { 126, 105, 225, 254, 214, 217, 255, 254, 128, 128, 128},
+        { 44, 86, 190, 251, 197, 204, 255, 233, 128, 128, 128},
+        { 6, 71, 130, 240, 164, 188, 255, 246, 128, 128, 128}
+      },
+      { /* Coeff Band 7 */
+        { 1, 195, 250, 255, 239, 197, 128, 128, 128, 128, 128},
+        { 167, 102, 249, 255, 234, 255, 128, 128, 128, 128, 128},
+        { 65, 91, 222, 255, 217, 255, 128, 128, 128, 128, 128},
+        { 1, 59, 128, 255, 154, 255, 128, 128, 128, 128, 128}
+      }
   },
   { /* block Type 3 */
     { /* Coeff Band 0 */