diff options
-rw-r--r-- | test/partial_idct_test.cc | 2 | ||||
-rw-r--r-- | vp9/common/vp9_idct.h | 3 | ||||
-rw-r--r-- | vp9/common/vp9_rtcd_defs.pl | 2 | ||||
-rw-r--r-- | vp9/common/x86/vp9_idct_intrin_sse2.c | 26 | ||||
-rw-r--r-- | vp9/common/x86/vp9_idct_intrin_sse2.h | 26 | ||||
-rw-r--r-- | vp9/common/x86/vp9_idct_intrin_ssse3.c | 377 |
6 files changed, 408 insertions, 28 deletions
diff --git a/test/partial_idct_test.cc b/test/partial_idct_test.cc index 79ef52109..b63e07b3a 100644 --- a/test/partial_idct_test.cc +++ b/test/partial_idct_test.cc @@ -193,7 +193,7 @@ INSTANTIATE_TEST_CASE_P( #if HAVE_SSSE3 && ARCH_X86_64 INSTANTIATE_TEST_CASE_P( - SSSE3, PartialIDctTest, + SSSE3_64, PartialIDctTest, ::testing::Values( make_tuple(&vp9_idct8x8_64_add_c, &vp9_idct8x8_12_add_ssse3, diff --git a/vp9/common/vp9_idct.h b/vp9/common/vp9_idct.h index d86877622..3253bcbf4 100644 --- a/vp9/common/vp9_idct.h +++ b/vp9/common/vp9_idct.h @@ -33,6 +33,9 @@ extern "C" { #define pair_set_epi16(a, b) \ _mm_set_epi16(b, a, b, a, b, a, b, a) +#define dual_set_epi16(a, b) \ + _mm_set_epi16(b, b, b, b, a, a, a, a) + // Constants: // for (int i = 1; i< 32; ++i) // printf("static const int cospi_%d_64 = %.0f;\n", i, diff --git a/vp9/common/vp9_rtcd_defs.pl b/vp9/common/vp9_rtcd_defs.pl index a18ca8ea6..09ce72ef2 100644 --- a/vp9/common/vp9_rtcd_defs.pl +++ b/vp9/common/vp9_rtcd_defs.pl @@ -360,7 +360,7 @@ specialize qw/vp9_idct16x16_256_add sse2 ssse3 neon_asm dspr2/; $vp9_idct16x16_256_add_neon_asm=vp9_idct16x16_256_add_neon; add_proto qw/void vp9_idct16x16_10_add/, "const int16_t *input, uint8_t *dest, int dest_stride"; -specialize qw/vp9_idct16x16_10_add sse2 neon_asm dspr2/; +specialize qw/vp9_idct16x16_10_add sse2 ssse3 neon_asm dspr2/; $vp9_idct16x16_10_add_neon_asm=vp9_idct16x16_10_add_neon; add_proto qw/void vp9_idct32x32_1024_add/, "const int16_t *input, uint8_t *dest, int dest_stride"; diff --git a/vp9/common/x86/vp9_idct_intrin_sse2.c b/vp9/common/x86/vp9_idct_intrin_sse2.c index ff9c43221..b60f8a06d 100644 --- a/vp9/common/x86/vp9_idct_intrin_sse2.c +++ b/vp9/common/x86/vp9_idct_intrin_sse2.c @@ -375,15 +375,6 @@ void vp9_iht4x4_16_add_sse2(const int16_t *input, uint8_t *dest, int stride, out3 = _mm_unpackhi_epi64(tr1_2, tr1_6); \ } -#define TRANSPOSE_8X4(in0, in1, in2, in3, out0, out1) \ - { \ - const __m128i tr0_0 = _mm_unpacklo_epi16(in0, in1); \ - const __m128i tr0_1 = _mm_unpacklo_epi16(in2, in3); \ - \ - in0 = _mm_unpacklo_epi32(tr0_0, tr0_1); /* i1 i0 */ \ - in1 = _mm_unpackhi_epi32(tr0_0, tr0_1); /* i3 i2 */ \ - } - #define TRANSPOSE_8X8_10(in0, in1, in2, in3, out0, out1) \ { \ const __m128i tr0_0 = _mm_unpacklo_epi16(in0, in1); \ @@ -612,23 +603,6 @@ void vp9_idct8x8_1_add_sse2(const int16_t *input, uint8_t *dest, int stride) { RECON_AND_STORE(dest, dc_value); } -static INLINE void array_transpose_4X8(__m128i *in, __m128i * out) { - const __m128i tr0_0 = _mm_unpacklo_epi16(in[0], in[1]); - const __m128i tr0_1 = _mm_unpacklo_epi16(in[2], in[3]); - const __m128i tr0_4 = _mm_unpacklo_epi16(in[4], in[5]); - const __m128i tr0_5 = _mm_unpacklo_epi16(in[6], in[7]); - - const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); - const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); - const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); - const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); - - out[0] = _mm_unpacklo_epi64(tr1_0, tr1_4); - out[1] = _mm_unpackhi_epi64(tr1_0, tr1_4); - out[2] = _mm_unpacklo_epi64(tr1_2, tr1_6); - out[3] = _mm_unpackhi_epi64(tr1_2, tr1_6); -} - static void idct8_sse2(__m128i *in) { const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); const __m128i stg1_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); diff --git a/vp9/common/x86/vp9_idct_intrin_sse2.h b/vp9/common/x86/vp9_idct_intrin_sse2.h index 1c62e3272..0f179b49a 100644 --- a/vp9/common/x86/vp9_idct_intrin_sse2.h +++ b/vp9/common/x86/vp9_idct_intrin_sse2.h @@ -45,6 +45,32 @@ static INLINE void array_transpose_8x8(__m128i *in, __m128i *res) { res[7] = _mm_unpackhi_epi64(tr1_6, tr1_7); } +#define TRANSPOSE_8X4(in0, in1, in2, in3, out0, out1) \ + { \ + const __m128i tr0_0 = _mm_unpacklo_epi16(in0, in1); \ + const __m128i tr0_1 = _mm_unpacklo_epi16(in2, in3); \ + \ + in0 = _mm_unpacklo_epi32(tr0_0, tr0_1); /* i1 i0 */ \ + in1 = _mm_unpackhi_epi32(tr0_0, tr0_1); /* i3 i2 */ \ + } + +static INLINE void array_transpose_4X8(__m128i *in, __m128i * out) { + const __m128i tr0_0 = _mm_unpacklo_epi16(in[0], in[1]); + const __m128i tr0_1 = _mm_unpacklo_epi16(in[2], in[3]); + const __m128i tr0_4 = _mm_unpacklo_epi16(in[4], in[5]); + const __m128i tr0_5 = _mm_unpacklo_epi16(in[6], in[7]); + + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); + + out[0] = _mm_unpacklo_epi64(tr1_0, tr1_4); + out[1] = _mm_unpackhi_epi64(tr1_0, tr1_4); + out[2] = _mm_unpacklo_epi64(tr1_2, tr1_6); + out[3] = _mm_unpackhi_epi64(tr1_2, tr1_6); +} + static INLINE void array_transpose_16x16(__m128i *res0, __m128i *res1) { __m128i tbuf[8]; array_transpose_8x8(res0, res0); diff --git a/vp9/common/x86/vp9_idct_intrin_ssse3.c b/vp9/common/x86/vp9_idct_intrin_ssse3.c index 0930e7805..73bf5d1d7 100644 --- a/vp9/common/x86/vp9_idct_intrin_ssse3.c +++ b/vp9/common/x86/vp9_idct_intrin_ssse3.c @@ -383,3 +383,380 @@ void vp9_idct16x16_256_add_ssse3(const int16_t *input, uint8_t *dest, dest += 8; write_buffer_8x16(dest, in1, stride); } + +static void idct16_10_r1(__m128i *in, __m128i *l) { + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i zero = _mm_setzero_si128(); + + const __m128i stg2_01 = dual_set_epi16(3212, 32610); + const __m128i stg2_67 = dual_set_epi16(-9512, 31358); + const __m128i stg3_01 = dual_set_epi16(6392, 32138); + const __m128i stg4_01 = dual_set_epi16(23170, 23170); + + + + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + const __m128i stg4_7 = pair_set_epi16(-cospi_8_64, cospi_24_64); + + __m128i stp1_0, stp1_1, stp1_4, stp1_6, + stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, + stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4; + + // Stage2 + { + const __m128i lo_1_15 = _mm_unpackhi_epi64(in[0], in[0]); + const __m128i lo_13_3 = _mm_unpackhi_epi64(in[1], in[1]); + + stp2_8 = _mm_mulhrs_epi16(lo_1_15, stg2_01); + stp2_11 = _mm_mulhrs_epi16(lo_13_3, stg2_67); + } + + // Stage3 + { + const __m128i lo_2_14 = _mm_unpacklo_epi64(in[1], in[1]); + stp1_4 = _mm_mulhrs_epi16(lo_2_14, stg3_01); + + stp1_13 = _mm_unpackhi_epi64(stp2_11, zero); + stp1_14 = _mm_unpackhi_epi64(stp2_8, zero); + } + + // Stage4 + { + const __m128i lo_0_8 = _mm_unpacklo_epi64(in[0], in[0]); + const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp1_14); + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp1_13); + + tmp0 = _mm_mulhrs_epi16(lo_0_8, stg4_01); + tmp1 = _mm_madd_epi16(lo_9_14, stg4_4); + tmp3 = _mm_madd_epi16(lo_9_14, stg4_5); + tmp2 = _mm_madd_epi16(lo_10_13, stg4_6); + tmp4 = _mm_madd_epi16(lo_10_13, stg4_7); + + tmp1 = _mm_add_epi32(tmp1, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp4 = _mm_add_epi32(tmp4, rounding); + + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); + + stp1_0 = _mm_unpacklo_epi64(tmp0, tmp0); + stp1_1 = _mm_unpackhi_epi64(tmp0, tmp0); + stp2_9 = _mm_packs_epi32(tmp1, tmp3); + stp2_10 = _mm_packs_epi32(tmp2, tmp4); + + stp2_6 = _mm_unpackhi_epi64(stp1_4, zero); + } + + // Stage5 and Stage6 + { + tmp0 = _mm_add_epi16(stp2_8, stp2_11); + tmp1 = _mm_sub_epi16(stp2_8, stp2_11); + tmp2 = _mm_add_epi16(stp2_9, stp2_10); + tmp3 = _mm_sub_epi16(stp2_9, stp2_10); + + stp1_9 = _mm_unpacklo_epi64(tmp2, zero); + stp1_10 = _mm_unpacklo_epi64(tmp3, zero); + stp1_8 = _mm_unpacklo_epi64(tmp0, zero); + stp1_11 = _mm_unpacklo_epi64(tmp1, zero); + + stp1_13 = _mm_unpackhi_epi64(tmp3, zero); + stp1_14 = _mm_unpackhi_epi64(tmp2, zero); + stp1_12 = _mm_unpackhi_epi64(tmp1, zero); + stp1_15 = _mm_unpackhi_epi64(tmp0, zero); + } + + // Stage6 + { + const __m128i lo_6_5 = _mm_add_epi16(stp2_6, stp1_4); + const __m128i lo_6_6 = _mm_sub_epi16(stp2_6, stp1_4); + const __m128i lo_10_13 = _mm_sub_epi16(stp1_13, stp1_10); + const __m128i lo_10_14 = _mm_add_epi16(stp1_13, stp1_10); + const __m128i lo_11_12 = _mm_sub_epi16(stp1_12, stp1_11); + const __m128i lo_11_13 = _mm_add_epi16(stp1_12, stp1_11); + + tmp1 = _mm_unpacklo_epi64(lo_6_5, lo_6_6); + tmp0 = _mm_unpacklo_epi64(lo_10_13, lo_10_14); + tmp4 = _mm_unpacklo_epi64(lo_11_12, lo_11_13); + + stp1_6 = _mm_mulhrs_epi16(tmp1, stg4_01); + tmp0 = _mm_mulhrs_epi16(tmp0, stg4_01); + tmp4 = _mm_mulhrs_epi16(tmp4, stg4_01); + + stp2_10 = _mm_unpacklo_epi64(tmp0, zero); + stp2_13 = _mm_unpackhi_epi64(tmp0, zero); + stp2_11 = _mm_unpacklo_epi64(tmp4, zero); + stp2_12 = _mm_unpackhi_epi64(tmp4, zero); + + tmp0 = _mm_add_epi16(stp1_0, stp1_4); + tmp1 = _mm_sub_epi16(stp1_0, stp1_4); + tmp2 = _mm_add_epi16(stp1_1, stp1_6); + tmp3 = _mm_sub_epi16(stp1_1, stp1_6); + + stp2_0 = _mm_unpackhi_epi64(tmp0, zero); + stp2_1 = _mm_unpacklo_epi64(tmp2, zero); + stp2_2 = _mm_unpackhi_epi64(tmp2, zero); + stp2_3 = _mm_unpacklo_epi64(tmp0, zero); + stp2_4 = _mm_unpacklo_epi64(tmp1, zero); + stp2_5 = _mm_unpackhi_epi64(tmp3, zero); + stp2_6 = _mm_unpacklo_epi64(tmp3, zero); + stp2_7 = _mm_unpackhi_epi64(tmp1, zero); + } + + // Stage7. Left 8x16 only. + l[0] = _mm_add_epi16(stp2_0, stp1_15); + l[1] = _mm_add_epi16(stp2_1, stp1_14); + l[2] = _mm_add_epi16(stp2_2, stp2_13); + l[3] = _mm_add_epi16(stp2_3, stp2_12); + l[4] = _mm_add_epi16(stp2_4, stp2_11); + l[5] = _mm_add_epi16(stp2_5, stp2_10); + l[6] = _mm_add_epi16(stp2_6, stp1_9); + l[7] = _mm_add_epi16(stp2_7, stp1_8); + l[8] = _mm_sub_epi16(stp2_7, stp1_8); + l[9] = _mm_sub_epi16(stp2_6, stp1_9); + l[10] = _mm_sub_epi16(stp2_5, stp2_10); + l[11] = _mm_sub_epi16(stp2_4, stp2_11); + l[12] = _mm_sub_epi16(stp2_3, stp2_12); + l[13] = _mm_sub_epi16(stp2_2, stp2_13); + l[14] = _mm_sub_epi16(stp2_1, stp1_14); + l[15] = _mm_sub_epi16(stp2_0, stp1_15); +} + +static void idct16_10_r2(__m128i *in) { + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + + const __m128i stg2_0 = dual_set_epi16(3212, 3212); + const __m128i stg2_1 = dual_set_epi16(32610, 32610); + const __m128i stg2_6 = dual_set_epi16(-9512, -9512); + const __m128i stg2_7 = dual_set_epi16(31358, 31358); + const __m128i stg3_0 = dual_set_epi16(6392, 6392); + const __m128i stg3_1 = dual_set_epi16(32138, 32138); + const __m128i stg4_01 = dual_set_epi16(23170, 23170); + + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + const __m128i stg4_7 = pair_set_epi16(-cospi_8_64, cospi_24_64); + + __m128i stp1_0, stp1_2, stp1_3, stp1_5, stp1_6, + stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, + stp1_8_0, stp1_12_0; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, + stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + /* Stage2 */ + { + stp1_8_0 = _mm_mulhrs_epi16(in[1], stg2_0); + stp1_15 = _mm_mulhrs_epi16(in[1], stg2_1); + stp1_11 = _mm_mulhrs_epi16(in[3], stg2_6); + stp1_12_0 = _mm_mulhrs_epi16(in[3], stg2_7); + } + + /* Stage3 */ + { + stp2_4 = _mm_mulhrs_epi16(in[2], stg3_0); + stp2_7 = _mm_mulhrs_epi16(in[2], stg3_1); + + stp1_9 = stp1_8_0; + stp1_10 = stp1_11; + + stp1_13 = stp1_12_0; + stp1_14 = stp1_15; + } + + /* Stage4 */ + { + const __m128i lo_9_14 = _mm_unpacklo_epi16(stp1_9, stp1_14); + const __m128i hi_9_14 = _mm_unpackhi_epi16(stp1_9, stp1_14); + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); + const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); + + stp1_0 = _mm_mulhrs_epi16(in[0], stg4_01); + + stp2_5 = stp2_4; + stp2_6 = stp2_7; + + + tmp0 = _mm_madd_epi16(lo_9_14, stg4_4); + tmp1 = _mm_madd_epi16(hi_9_14, stg4_4); + tmp2 = _mm_madd_epi16(lo_9_14, stg4_5); + tmp3 = _mm_madd_epi16(hi_9_14, stg4_5); + tmp4 = _mm_madd_epi16(lo_10_13, stg4_6); + tmp5 = _mm_madd_epi16(hi_10_13, stg4_6); + tmp6 = _mm_madd_epi16(lo_10_13, stg4_7); + tmp7 = _mm_madd_epi16(hi_10_13, stg4_7); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp1 = _mm_add_epi32(tmp1, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + tmp4 = _mm_add_epi32(tmp4, rounding); + tmp5 = _mm_add_epi32(tmp5, rounding); + tmp6 = _mm_add_epi32(tmp6, rounding); + tmp7 = _mm_add_epi32(tmp7, rounding); + + tmp0 = _mm_srai_epi32(tmp0, 14); + tmp1 = _mm_srai_epi32(tmp1, 14); + tmp2 = _mm_srai_epi32(tmp2, 14); + tmp3 = _mm_srai_epi32(tmp3, 14); + tmp4 = _mm_srai_epi32(tmp4, 14); + tmp5 = _mm_srai_epi32(tmp5, 14); + tmp6 = _mm_srai_epi32(tmp6, 14); + tmp7 = _mm_srai_epi32(tmp7, 14); + + stp2_9 = _mm_packs_epi32(tmp0, tmp1); + stp2_14 = _mm_packs_epi32(tmp2, tmp3); + stp2_10 = _mm_packs_epi32(tmp4, tmp5); + stp2_13 = _mm_packs_epi32(tmp6, tmp7); + } + + /* Stage5 */ + { + stp1_2 = stp1_0; + stp1_3 = stp1_0; + + tmp0 = _mm_sub_epi16(stp2_6, stp2_5); + tmp1 = _mm_add_epi16(stp2_6, stp2_5); + + stp1_5 = _mm_mulhrs_epi16(tmp0, stg4_01); + stp1_6 = _mm_mulhrs_epi16(tmp1, stg4_01); + + stp1_8 = _mm_add_epi16(stp1_8_0, stp1_11); + stp1_9 = _mm_add_epi16(stp2_9, stp2_10); + stp1_10 = _mm_sub_epi16(stp2_9, stp2_10); + stp1_11 = _mm_sub_epi16(stp1_8_0, stp1_11); + + stp1_12 = _mm_sub_epi16(stp1_15, stp1_12_0); + stp1_13 = _mm_sub_epi16(stp2_14, stp2_13); + stp1_14 = _mm_add_epi16(stp2_14, stp2_13); + stp1_15 = _mm_add_epi16(stp1_15, stp1_12_0); + } + + /* Stage6 */ + { + stp2_0 = _mm_add_epi16(stp1_0, stp2_7); + stp2_1 = _mm_add_epi16(stp1_0, stp1_6); + stp2_2 = _mm_add_epi16(stp1_2, stp1_5); + stp2_3 = _mm_add_epi16(stp1_3, stp2_4); + + tmp0 = _mm_sub_epi16(stp1_13, stp1_10); + tmp1 = _mm_add_epi16(stp1_13, stp1_10); + tmp2 = _mm_sub_epi16(stp1_12, stp1_11); + tmp3 = _mm_add_epi16(stp1_12, stp1_11); + + stp2_4 = _mm_sub_epi16(stp1_3, stp2_4); + stp2_5 = _mm_sub_epi16(stp1_2, stp1_5); + stp2_6 = _mm_sub_epi16(stp1_0, stp1_6); + stp2_7 = _mm_sub_epi16(stp1_0, stp2_7); + + stp2_10 = _mm_mulhrs_epi16(tmp0, stg4_01); + stp2_13 = _mm_mulhrs_epi16(tmp1, stg4_01); + stp2_11 = _mm_mulhrs_epi16(tmp2, stg4_01); + stp2_12 = _mm_mulhrs_epi16(tmp3, stg4_01); + } + + // Stage7 + in[0] = _mm_add_epi16(stp2_0, stp1_15); + in[1] = _mm_add_epi16(stp2_1, stp1_14); + in[2] = _mm_add_epi16(stp2_2, stp2_13); + in[3] = _mm_add_epi16(stp2_3, stp2_12); + in[4] = _mm_add_epi16(stp2_4, stp2_11); + in[5] = _mm_add_epi16(stp2_5, stp2_10); + in[6] = _mm_add_epi16(stp2_6, stp1_9); + in[7] = _mm_add_epi16(stp2_7, stp1_8); + in[8] = _mm_sub_epi16(stp2_7, stp1_8); + in[9] = _mm_sub_epi16(stp2_6, stp1_9); + in[10] = _mm_sub_epi16(stp2_5, stp2_10); + in[11] = _mm_sub_epi16(stp2_4, stp2_11); + in[12] = _mm_sub_epi16(stp2_3, stp2_12); + in[13] = _mm_sub_epi16(stp2_2, stp2_13); + in[14] = _mm_sub_epi16(stp2_1, stp1_14); + in[15] = _mm_sub_epi16(stp2_0, stp1_15); +} + +void vp9_idct16x16_10_add_ssse3(const int16_t *input, uint8_t *dest, + int stride) { + const __m128i final_rounding = _mm_set1_epi16(1<<5); + const __m128i zero = _mm_setzero_si128(); + __m128i in[16], l[16]; + + int i; + // First 1-D inverse DCT + // Load input data. + in[0] = _mm_load_si128((const __m128i *)input); + in[1] = _mm_load_si128((const __m128i *)(input + 8 * 2)); + in[2] = _mm_load_si128((const __m128i *)(input + 8 * 4)); + in[3] = _mm_load_si128((const __m128i *)(input + 8 * 6)); + + TRANSPOSE_8X4(in[0], in[1], in[2], in[3], in[0], in[1]); + + idct16_10_r1(in, l); + + // Second 1-D inverse transform, performed per 8x16 block + for (i = 0; i < 2; i++) { + array_transpose_4X8(l + 8*i, in); + + idct16_10_r2(in); + + // Final rounding and shift + in[0] = _mm_adds_epi16(in[0], final_rounding); + in[1] = _mm_adds_epi16(in[1], final_rounding); + in[2] = _mm_adds_epi16(in[2], final_rounding); + in[3] = _mm_adds_epi16(in[3], final_rounding); + in[4] = _mm_adds_epi16(in[4], final_rounding); + in[5] = _mm_adds_epi16(in[5], final_rounding); + in[6] = _mm_adds_epi16(in[6], final_rounding); + in[7] = _mm_adds_epi16(in[7], final_rounding); + in[8] = _mm_adds_epi16(in[8], final_rounding); + in[9] = _mm_adds_epi16(in[9], final_rounding); + in[10] = _mm_adds_epi16(in[10], final_rounding); + in[11] = _mm_adds_epi16(in[11], final_rounding); + in[12] = _mm_adds_epi16(in[12], final_rounding); + in[13] = _mm_adds_epi16(in[13], final_rounding); + in[14] = _mm_adds_epi16(in[14], final_rounding); + in[15] = _mm_adds_epi16(in[15], final_rounding); + + in[0] = _mm_srai_epi16(in[0], 6); + in[1] = _mm_srai_epi16(in[1], 6); + in[2] = _mm_srai_epi16(in[2], 6); + in[3] = _mm_srai_epi16(in[3], 6); + in[4] = _mm_srai_epi16(in[4], 6); + in[5] = _mm_srai_epi16(in[5], 6); + in[6] = _mm_srai_epi16(in[6], 6); + in[7] = _mm_srai_epi16(in[7], 6); + in[8] = _mm_srai_epi16(in[8], 6); + in[9] = _mm_srai_epi16(in[9], 6); + in[10] = _mm_srai_epi16(in[10], 6); + in[11] = _mm_srai_epi16(in[11], 6); + in[12] = _mm_srai_epi16(in[12], 6); + in[13] = _mm_srai_epi16(in[13], 6); + in[14] = _mm_srai_epi16(in[14], 6); + in[15] = _mm_srai_epi16(in[15], 6); + + RECON_AND_STORE(dest, in[0]); + RECON_AND_STORE(dest, in[1]); + RECON_AND_STORE(dest, in[2]); + RECON_AND_STORE(dest, in[3]); + RECON_AND_STORE(dest, in[4]); + RECON_AND_STORE(dest, in[5]); + RECON_AND_STORE(dest, in[6]); + RECON_AND_STORE(dest, in[7]); + RECON_AND_STORE(dest, in[8]); + RECON_AND_STORE(dest, in[9]); + RECON_AND_STORE(dest, in[10]); + RECON_AND_STORE(dest, in[11]); + RECON_AND_STORE(dest, in[12]); + RECON_AND_STORE(dest, in[13]); + RECON_AND_STORE(dest, in[14]); + RECON_AND_STORE(dest, in[15]); + + dest += 8 - (stride * 16); + } +} |