Optimize vp9_short_idct4x4llm function

Wrote a SSE2 vp9_short_idct4x4llm to improve the decoder
performance.

Change-Id: I90b9d48c4bf37aaf47995bffe7e584e6d4a2c000

6 files changed, 128 insertions, 26 deletions

diff --git a/vp9/common/vp9_idct.h b/vp9/common/vp9_idct.h
index 17d0134fa..bc79b5cae 100644
--- a/vp9/common/vp9_idct.h
+++ b/vp9/common/vp9_idct.h
@@ -84,11 +84,4 @@ typedef struct {
   transform_1d cols, rows;  // vertical and horizontal
 } transform_2d;
 
-#define ROUND_POWER_OF_TWO(value, n) (((value) + (1 << ((n) - 1))) >> (n))
-
-/* If we don't want to use ROUND_POWER_OF_TWO macro
-static INLINE int16_t round_power_of_two(int16_t value, int n) {
-  return (value + (1 << (n - 1))) >> n;
-}*/
-
 #endif  // VP9_COMMON_VP9_IDCT_H_
diff --git a/vp9/common/vp9_rtcd_defs.sh b/vp9/common/vp9_rtcd_defs.sh
index 9cbf44c46..a444876fd 100644
--- a/vp9/common/vp9_rtcd_defs.sh
+++ b/vp9/common/vp9_rtcd_defs.sh
@@ -254,7 +254,7 @@ prototype void vp9_short_idct4x4llm_1 "int16_t *input, int16_t *output, int pitc
 specialize vp9_short_idct4x4llm_1
 
 prototype void vp9_short_idct4x4llm "int16_t *input, int16_t *output, int pitch"
-specialize vp9_short_idct4x4llm
+specialize vp9_short_idct4x4llm sse2
 
 prototype void vp9_short_idct8x8 "int16_t *input, int16_t *output, int pitch"
 specialize vp9_short_idct8x8
diff --git a/vp9/common/x86/vp9_idct_x86.h b/vp9/common/x86/vp9_idct_x86.h
index 8320cf87d..bd66d8c72 100644
--- a/vp9/common/x86/vp9_idct_x86.h
+++ b/vp9/common/x86/vp9_idct_x86.h
@@ -20,23 +20,10 @@
  */
 
 #if HAVE_MMX
-extern prototype_idct(vp9_short_idct4x4llm_1_mmx);
-extern prototype_idct(vp9_short_idct4x4llm_mmx);
-extern prototype_idct_scalar_add(vp9_dc_only_idct_add_mmx);
-
 extern prototype_second_order(vp9_short_inv_walsh4x4_mmx);
 extern prototype_second_order(vp9_short_inv_walsh4x4_1_mmx);
 
 #if !CONFIG_RUNTIME_CPU_DETECT
-#undef  vp9_idct_idct1
-#define vp9_idct_idct1 vp9_short_idct4x4llm_1_mmx
-
-#undef  vp9_idct_idct16
-#define vp9_idct_idct16 vp9_short_idct4x4llm_mmx
-
-#undef  vp9_idct_idct1_scalar_add
-#define vp9_idct_idct1_scalar_add vp9_dc_only_idct_add_mmx
-
 #undef vp9_idct_iwalsh16
 #define vp9_idct_iwalsh16 vp9_short_inv_walsh4x4_mmx
 
diff --git a/vp9/common/x86/vp9_idctllm_x86.c b/vp9/common/x86/vp9_idctllm_x86.c
index 667f5c1d3..7b3c57967 100644
--- a/vp9/common/x86/vp9_idctllm_x86.c
+++ b/vp9/common/x86/vp9_idctllm_x86.c
@@ -73,4 +73,129 @@ void vp9_dc_only_idct_add_sse2(int input_dc, uint8_t *pred_ptr,
   p1 = _mm_srli_si128(p1, 4);
   *(int *)dst_ptr = _mm_cvtsi128_si32(p1);
 }
+
+void vp9_short_idct4x4llm_sse2(int16_t *input, int16_t *output, int pitch) {
+  const __m128i zero = _mm_setzero_si128();
+  const __m128i eight = _mm_set1_epi16(8);
+  const __m128i cst = _mm_setr_epi16((short)cospi_16_64, (short)cospi_16_64,
+                                     (short)cospi_16_64, (short)-cospi_16_64,
+                                     (short)cospi_24_64, (short)-cospi_8_64,
+                                     (short)cospi_8_64, (short)cospi_24_64);
+  const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
+  const int half_pitch = pitch >> 1;
+  __m128i input0, input1, input2, input3;
+
+  // Rows
+  input0 = _mm_loadl_epi64((__m128i *)input);
+  input1 = _mm_loadl_epi64((__m128i *)(input + 4));
+  input2 = _mm_loadl_epi64((__m128i *)(input + 8));
+  input3 = _mm_loadl_epi64((__m128i *)(input + 12));
+
+  // Construct i3, i1, i3, i1, i2, i0, i2, i0
+  input0 = _mm_shufflelo_epi16(input0, 0xd8);
+  input1 = _mm_shufflelo_epi16(input1, 0xd8);
+  input2 = _mm_shufflelo_epi16(input2, 0xd8);
+  input3 = _mm_shufflelo_epi16(input3, 0xd8);
+
+  input0 = _mm_unpacklo_epi32(input0, input0);
+  input1 = _mm_unpacklo_epi32(input1, input1);
+  input2 = _mm_unpacklo_epi32(input2, input2);
+  input3 = _mm_unpacklo_epi32(input3, input3);
+
+  // Stage 1
+  input0 = _mm_madd_epi16(input0, cst);
+  input1 = _mm_madd_epi16(input1, cst);
+  input2 = _mm_madd_epi16(input2, cst);
+  input3 = _mm_madd_epi16(input3, cst);
+
+  input0 = _mm_add_epi32(input0, rounding);
+  input1 = _mm_add_epi32(input1, rounding);
+  input2 = _mm_add_epi32(input2, rounding);
+  input3 = _mm_add_epi32(input3, rounding);
+
+  input0 = _mm_srai_epi32(input0, DCT_CONST_BITS);
+  input1 = _mm_srai_epi32(input1, DCT_CONST_BITS);
+  input2 = _mm_srai_epi32(input2, DCT_CONST_BITS);
+  input3 = _mm_srai_epi32(input3, DCT_CONST_BITS);
+
+  // Stage 2
+  input0 = _mm_packs_epi32(input0, zero);
+  input1 = _mm_packs_epi32(input1, zero);
+  input2 = _mm_packs_epi32(input2, zero);
+  input3 = _mm_packs_epi32(input3, zero);
+
+  // Transpose
+  input1 = _mm_unpacklo_epi16(input0, input1);
+  input3 = _mm_unpacklo_epi16(input2, input3);
+  input0 = _mm_unpacklo_epi32(input1, input3);
+  input1 = _mm_unpackhi_epi32(input1, input3);
+
+  // Switch column2, column 3, and then, we got:
+  // input2: column1, column 0;  input3: column2, column 3.
+  input1 = _mm_shuffle_epi32(input1, 0x4e);
+  input2 = _mm_add_epi16(input0, input1);
+  input3 = _mm_sub_epi16(input0, input1);
+
+  // Columns
+  // Construct i3, i1, i3, i1, i2, i0, i2, i0
+  input0 = _mm_shufflelo_epi16(input2, 0xd8);
+  input1 = _mm_shufflehi_epi16(input2, 0xd8);
+  input2 = _mm_shufflehi_epi16(input3, 0xd8);
+  input3 = _mm_shufflelo_epi16(input3, 0xd8);
+
+  input0 = _mm_unpacklo_epi32(input0, input0);
+  input1 = _mm_unpackhi_epi32(input1, input1);
+  input2 = _mm_unpackhi_epi32(input2, input2);
+  input3 = _mm_unpacklo_epi32(input3, input3);
+
+  // Stage 1
+  input0 = _mm_madd_epi16(input0, cst);
+  input1 = _mm_madd_epi16(input1, cst);
+  input2 = _mm_madd_epi16(input2, cst);
+  input3 = _mm_madd_epi16(input3, cst);
+
+  input0 = _mm_add_epi32(input0, rounding);
+  input1 = _mm_add_epi32(input1, rounding);
+  input2 = _mm_add_epi32(input2, rounding);
+  input3 = _mm_add_epi32(input3, rounding);
+
+  input0 = _mm_srai_epi32(input0, DCT_CONST_BITS);
+  input1 = _mm_srai_epi32(input1, DCT_CONST_BITS);
+  input2 = _mm_srai_epi32(input2, DCT_CONST_BITS);
+  input3 = _mm_srai_epi32(input3, DCT_CONST_BITS);
+
+  // Stage 2
+  input0 = _mm_packs_epi32(input0, zero);
+  input1 = _mm_packs_epi32(input1, zero);
+  input2 = _mm_packs_epi32(input2, zero);
+  input3 = _mm_packs_epi32(input3, zero);
+
+  // Transpose
+  input1 = _mm_unpacklo_epi16(input0, input1);
+  input3 = _mm_unpacklo_epi16(input2, input3);
+  input0 = _mm_unpacklo_epi32(input1, input3);
+  input1 = _mm_unpackhi_epi32(input1, input3);
+
+  // Switch column2, column 3, and then, we got:
+  // input2: column1, column 0;  input3: column2, column 3.
+  input1 = _mm_shuffle_epi32(input1, 0x4e);
+  input2 = _mm_add_epi16(input0, input1);
+  input3 = _mm_sub_epi16(input0, input1);
+
+  // Final round and shift
+  input2 = _mm_add_epi16(input2, eight);
+  input3 = _mm_add_epi16(input3, eight);
+
+  input2 = _mm_srai_epi16(input2, 4);
+  input3 = _mm_srai_epi16(input3, 4);
+
+  // Store results
+  _mm_storel_epi64((__m128i *)output, input2);
+  input2 = _mm_srli_si128(input2, 8);
+  _mm_storel_epi64((__m128i *)(output + half_pitch), input2);
+
+  _mm_storel_epi64((__m128i *)(output + 3 * half_pitch), input3);
+  input3 = _mm_srli_si128(input3, 8);
+  _mm_storel_epi64((__m128i *)(output + 2 * half_pitch), input3);
+}
 #endif
diff --git a/vp9/decoder/vp9_dequantize.c b/vp9/decoder/vp9_dequantize.c
index 5a98b1150..8eda94e55 100644
--- a/vp9/decoder/vp9_dequantize.c
+++ b/vp9/decoder/vp9_dequantize.c
@@ -90,7 +90,7 @@ void vp9_dequant_idct_add_c(int16_t *input, const int16_t *dq, uint8_t *pred,
       input[i] *= dq[i];
 
     // the idct halves ( >> 1) the pitch
-    vp9_short_idct4x4llm_c(input, output, 4 << 1);
+    vp9_short_idct4x4llm(input, output, 4 << 1);
 
     vpx_memset(input, 0, 32);
 
@@ -112,7 +112,7 @@ void vp9_dequant_dc_idct_add_c(int16_t *input, const int16_t *dq, uint8_t *pred,
     input[i] *= dq[i];
 
   // the idct halves ( >> 1) the pitch
-  vp9_short_idct4x4llm_c(input, output, 4 << 1);
+  vp9_short_idct4x4llm(input, output, 4 << 1);
   vpx_memset(input, 0, 32);
   add_residual(output, pred, pitch, dest, stride, 4, 4);
 }
diff --git a/vp9/decoder/x86/vp9_idct_mmx.h b/vp9/decoder/x86/vp9_idct_mmx.h
index c0e9bfd06..7d9829175 100644
--- a/vp9/decoder/x86/vp9_idct_mmx.h
+++ b/vp9/decoder/x86/vp9_idct_mmx.h
@@ -16,9 +16,6 @@ void vp9_dequant_dc_idct_add_mmx(short *input, const short *dq,
                                  unsigned char *pred, unsigned char *dest,
                                  int pitch, int stride, int Dc);
 
-void vp9_dc_only_idct_add_mmx(short input_dc, const unsigned char *pred_ptr,
-                              unsigned char *dst_ptr, int pitch, int stride);
-
 void vp9_dequant_idct_add_mmx(short *input, const short *dq, unsigned char *pred,
                               unsigned char *dest, int pitch, int stride);