diff options
Diffstat (limited to 'vpx_dsp/x86/inv_txfm_ssse3.c')
| -rw-r--r-- | vpx_dsp/x86/inv_txfm_ssse3.c | 627 |
1 files changed, 234 insertions, 393 deletions
diff --git a/vpx_dsp/x86/inv_txfm_ssse3.c b/vpx_dsp/x86/inv_txfm_ssse3.c index d4c3c7d30..2f4e5ab85 100644 --- a/vpx_dsp/x86/inv_txfm_ssse3.c +++ b/vpx_dsp/x86/inv_txfm_ssse3.c @@ -1029,421 +1029,262 @@ void vpx_idct32x32_135_add_ssse3(const tran_low_t *input, uint8_t *dest, recon_and_store(col0, col1, dest + 16, stride); } -// For each 8x32 block __m128i in[32], output __m128i in[32] -static void idct32_8x32(const __m128i *in, __m128i *out) { - const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); - // idct constants for each stage - const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64); - const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64); - const __m128i stg1_2 = pair_set_epi16(cospi_15_64, -cospi_17_64); - const __m128i stg1_3 = pair_set_epi16(cospi_17_64, cospi_15_64); - const __m128i stg1_4 = pair_set_epi16(cospi_23_64, -cospi_9_64); - const __m128i stg1_5 = pair_set_epi16(cospi_9_64, cospi_23_64); - const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64); - const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64); - const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64); - const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64); - const __m128i stg1_10 = pair_set_epi16(cospi_11_64, -cospi_21_64); - const __m128i stg1_11 = pair_set_epi16(cospi_21_64, cospi_11_64); - const __m128i stg1_12 = pair_set_epi16(cospi_19_64, -cospi_13_64); - const __m128i stg1_13 = pair_set_epi16(cospi_13_64, cospi_19_64); - const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64); - const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64); - - const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64); - const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64); - const __m128i stg2_2 = pair_set_epi16(cospi_14_64, -cospi_18_64); - const __m128i stg2_3 = pair_set_epi16(cospi_18_64, cospi_14_64); - const __m128i stg2_4 = pair_set_epi16(cospi_22_64, -cospi_10_64); - const __m128i stg2_5 = pair_set_epi16(cospi_10_64, cospi_22_64); - const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64); - const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64); - - const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); - const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64); - const __m128i stg3_2 = pair_set_epi16(cospi_12_64, -cospi_20_64); - const __m128i stg3_3 = pair_set_epi16(cospi_20_64, cospi_12_64); - const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); - const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); - const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); - const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); - const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); - const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); +// For each 8x32 block __m128i in[32], +// Input with index, 2, 6, 10, 14, 18, 22, 26, 30 +// output pixels: 8-15 in __m128i in[32] +static void idct32_full_8x32_quarter_2(const __m128i *in /*in[32]*/, + __m128i *out /*out[16]*/) { + __m128i u8, u9, u10, u11, u12, u13, u14, u15; // stp2_ + __m128i v8, v9, v10, v11, v12, v13, v14, v15; // stp1_ - const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); - const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); - const __m128i stg4_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); - const __m128i stg4_3 = pair_set_epi16(cospi_8_64, cospi_24_64); - const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); - const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); - const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + { + const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i stg2_2 = pair_set_epi16(cospi_14_64, -cospi_18_64); + const __m128i stg2_3 = pair_set_epi16(cospi_18_64, cospi_14_64); + butterfly(&in[2], &in[30], &stg2_0, &stg2_1, &u8, &u15); + butterfly(&in[18], &in[14], &stg2_2, &stg2_3, &u9, &u14); + } - const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); - __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7, - stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, - stp1_16, stp1_17, stp1_18, stp1_19, stp1_20, stp1_21, stp1_22, stp1_23, - stp1_24, stp1_25, stp1_26, stp1_27, stp1_28, stp1_29, stp1_30, stp1_31; - __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, - stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15, - stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23, - stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + v8 = _mm_add_epi16(u8, u9); + v9 = _mm_sub_epi16(u8, u9); + v14 = _mm_sub_epi16(u15, u14); + v15 = _mm_add_epi16(u15, u14); - /* Stage1 */ { - const __m128i lo_1_31 = _mm_unpacklo_epi16(in[1], in[31]); - const __m128i hi_1_31 = _mm_unpackhi_epi16(in[1], in[31]); - const __m128i lo_17_15 = _mm_unpacklo_epi16(in[17], in[15]); - const __m128i hi_17_15 = _mm_unpackhi_epi16(in[17], in[15]); - - const __m128i lo_9_23 = _mm_unpacklo_epi16(in[9], in[23]); - const __m128i hi_9_23 = _mm_unpackhi_epi16(in[9], in[23]); - const __m128i lo_25_7 = _mm_unpacklo_epi16(in[25], in[7]); - const __m128i hi_25_7 = _mm_unpackhi_epi16(in[25], in[7]); - - const __m128i lo_5_27 = _mm_unpacklo_epi16(in[5], in[27]); - const __m128i hi_5_27 = _mm_unpackhi_epi16(in[5], in[27]); - const __m128i lo_21_11 = _mm_unpacklo_epi16(in[21], in[11]); - const __m128i hi_21_11 = _mm_unpackhi_epi16(in[21], in[11]); - - const __m128i lo_13_19 = _mm_unpacklo_epi16(in[13], in[19]); - const __m128i hi_13_19 = _mm_unpackhi_epi16(in[13], in[19]); - const __m128i lo_29_3 = _mm_unpacklo_epi16(in[29], in[3]); - const __m128i hi_29_3 = _mm_unpackhi_epi16(in[29], in[3]); - - MULTIPLICATION_AND_ADD(lo_1_31, hi_1_31, lo_17_15, hi_17_15, stg1_0, stg1_1, - stg1_2, stg1_3, stp1_16, stp1_31, stp1_17, stp1_30) - MULTIPLICATION_AND_ADD(lo_9_23, hi_9_23, lo_25_7, hi_25_7, stg1_4, stg1_5, - stg1_6, stg1_7, stp1_18, stp1_29, stp1_19, stp1_28) - MULTIPLICATION_AND_ADD(lo_5_27, hi_5_27, lo_21_11, hi_21_11, stg1_8, stg1_9, - stg1_10, stg1_11, stp1_20, stp1_27, stp1_21, stp1_26) - MULTIPLICATION_AND_ADD(lo_13_19, hi_13_19, lo_29_3, hi_29_3, stg1_12, - stg1_13, stg1_14, stg1_15, stp1_22, stp1_25, stp1_23, - stp1_24) + const __m128i stg2_4 = pair_set_epi16(cospi_22_64, -cospi_10_64); + const __m128i stg2_5 = pair_set_epi16(cospi_10_64, cospi_22_64); + const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64); + butterfly(&in[10], &in[22], &stg2_4, &stg2_5, &u10, &u13); + butterfly(&in[26], &in[6], &stg2_6, &stg2_7, &u11, &u12); } - /* Stage2 */ + v10 = _mm_sub_epi16(u11, u10); + v11 = _mm_add_epi16(u11, u10); + v12 = _mm_add_epi16(u12, u13); + v13 = _mm_sub_epi16(u12, u13); + { - const __m128i lo_2_30 = _mm_unpacklo_epi16(in[2], in[30]); - const __m128i hi_2_30 = _mm_unpackhi_epi16(in[2], in[30]); - const __m128i lo_18_14 = _mm_unpacklo_epi16(in[18], in[14]); - const __m128i hi_18_14 = _mm_unpackhi_epi16(in[18], in[14]); - - const __m128i lo_10_22 = _mm_unpacklo_epi16(in[10], in[22]); - const __m128i hi_10_22 = _mm_unpackhi_epi16(in[10], in[22]); - const __m128i lo_26_6 = _mm_unpacklo_epi16(in[26], in[6]); - const __m128i hi_26_6 = _mm_unpackhi_epi16(in[26], in[6]); - - MULTIPLICATION_AND_ADD(lo_2_30, hi_2_30, lo_18_14, hi_18_14, stg2_0, stg2_1, - stg2_2, stg2_3, stp2_8, stp2_15, stp2_9, stp2_14) - MULTIPLICATION_AND_ADD(lo_10_22, hi_10_22, lo_26_6, hi_26_6, stg2_4, stg2_5, - stg2_6, stg2_7, stp2_10, stp2_13, stp2_11, stp2_12) - - stp2_16 = _mm_add_epi16(stp1_16, stp1_17); - stp2_17 = _mm_sub_epi16(stp1_16, stp1_17); - stp2_18 = _mm_sub_epi16(stp1_19, stp1_18); - stp2_19 = _mm_add_epi16(stp1_19, stp1_18); - - stp2_20 = _mm_add_epi16(stp1_20, stp1_21); - stp2_21 = _mm_sub_epi16(stp1_20, stp1_21); - stp2_22 = _mm_sub_epi16(stp1_23, stp1_22); - stp2_23 = _mm_add_epi16(stp1_23, stp1_22); - - stp2_24 = _mm_add_epi16(stp1_24, stp1_25); - stp2_25 = _mm_sub_epi16(stp1_24, stp1_25); - stp2_26 = _mm_sub_epi16(stp1_27, stp1_26); - stp2_27 = _mm_add_epi16(stp1_27, stp1_26); - - stp2_28 = _mm_add_epi16(stp1_28, stp1_29); - stp2_29 = _mm_sub_epi16(stp1_28, stp1_29); - stp2_30 = _mm_sub_epi16(stp1_31, stp1_30); - stp2_31 = _mm_add_epi16(stp1_31, stp1_30); + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + butterfly_self(&v9, &v14, stg4_4, stg4_5); + butterfly_self(&v10, &v13, stg4_6, stg4_4); } - /* Stage3 */ + out[0] = _mm_add_epi16(v8, v11); + out[1] = _mm_add_epi16(v9, v10); + out[6] = _mm_add_epi16(v14, v13); + out[7] = _mm_add_epi16(v15, v12); + + out[2] = _mm_sub_epi16(v9, v10); + out[3] = _mm_sub_epi16(v8, v11); + out[4] = _mm_sub_epi16(v15, v12); + out[5] = _mm_sub_epi16(v14, v13); + { - const __m128i lo_4_28 = _mm_unpacklo_epi16(in[4], in[28]); - const __m128i hi_4_28 = _mm_unpackhi_epi16(in[4], in[28]); - const __m128i lo_20_12 = _mm_unpacklo_epi16(in[20], in[12]); - const __m128i hi_20_12 = _mm_unpackhi_epi16(in[20], in[12]); - - const __m128i lo_17_30 = _mm_unpacklo_epi16(stp2_17, stp2_30); - const __m128i hi_17_30 = _mm_unpackhi_epi16(stp2_17, stp2_30); - const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); - const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); - - const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); - const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); - const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); - const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); - - MULTIPLICATION_AND_ADD(lo_4_28, hi_4_28, lo_20_12, hi_20_12, stg3_0, stg3_1, - stg3_2, stg3_3, stp1_4, stp1_7, stp1_5, stp1_6) - - stp1_8 = _mm_add_epi16(stp2_8, stp2_9); - stp1_9 = _mm_sub_epi16(stp2_8, stp2_9); - stp1_10 = _mm_sub_epi16(stp2_11, stp2_10); - stp1_11 = _mm_add_epi16(stp2_11, stp2_10); - stp1_12 = _mm_add_epi16(stp2_12, stp2_13); - stp1_13 = _mm_sub_epi16(stp2_12, stp2_13); - stp1_14 = _mm_sub_epi16(stp2_15, stp2_14); - stp1_15 = _mm_add_epi16(stp2_15, stp2_14); - - MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4, - stg3_5, stg3_6, stg3_4, stp1_17, stp1_30, stp1_18, - stp1_29) - MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8, - stg3_9, stg3_10, stg3_8, stp1_21, stp1_26, stp1_22, - stp1_25) - - stp1_16 = stp2_16; - stp1_31 = stp2_31; - stp1_19 = stp2_19; - stp1_20 = stp2_20; - stp1_23 = stp2_23; - stp1_24 = stp2_24; - stp1_27 = stp2_27; - stp1_28 = stp2_28; + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + butterfly_self(&out[2], &out[5], stg6_0, stg4_0); + butterfly_self(&out[3], &out[4], stg6_0, stg4_0); } +} + +// For each 8x32 block __m128i in[32], +// Input with index, 0, 4, 8, 12, 16, 20, 24, 28 +// output pixels: 0-7 in __m128i in[32] +static void idct32_full_8x32_quarter_1(const __m128i *in /*in[32]*/, + __m128i *out /*out[8]*/) { + __m128i u0, u1, u2, u3, u4, u5, u6, u7; // stp1_ + __m128i v0, v1, v2, v3, v4, v5, v6, v7; // stp2_ - /* Stage4 */ { - const __m128i lo_0_16 = _mm_unpacklo_epi16(in[0], in[16]); - const __m128i hi_0_16 = _mm_unpackhi_epi16(in[0], in[16]); - const __m128i lo_8_24 = _mm_unpacklo_epi16(in[8], in[24]); - const __m128i hi_8_24 = _mm_unpackhi_epi16(in[8], in[24]); - - const __m128i lo_9_14 = _mm_unpacklo_epi16(stp1_9, stp1_14); - const __m128i hi_9_14 = _mm_unpackhi_epi16(stp1_9, stp1_14); - const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); - const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); - - MULTIPLICATION_AND_ADD(lo_0_16, hi_0_16, lo_8_24, hi_8_24, stg4_0, stg4_1, - stg4_2, stg4_3, stp2_0, stp2_1, stp2_2, stp2_3) - - stp2_4 = _mm_add_epi16(stp1_4, stp1_5); - stp2_5 = _mm_sub_epi16(stp1_4, stp1_5); - stp2_6 = _mm_sub_epi16(stp1_7, stp1_6); - stp2_7 = _mm_add_epi16(stp1_7, stp1_6); - - MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5, - stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13) - - stp2_8 = stp1_8; - stp2_15 = stp1_15; - stp2_11 = stp1_11; - stp2_12 = stp1_12; - - stp2_16 = _mm_add_epi16(stp1_16, stp1_19); - stp2_17 = _mm_add_epi16(stp1_17, stp1_18); - stp2_18 = _mm_sub_epi16(stp1_17, stp1_18); - stp2_19 = _mm_sub_epi16(stp1_16, stp1_19); - stp2_20 = _mm_sub_epi16(stp1_23, stp1_20); - stp2_21 = _mm_sub_epi16(stp1_22, stp1_21); - stp2_22 = _mm_add_epi16(stp1_22, stp1_21); - stp2_23 = _mm_add_epi16(stp1_23, stp1_20); - - stp2_24 = _mm_add_epi16(stp1_24, stp1_27); - stp2_25 = _mm_add_epi16(stp1_25, stp1_26); - stp2_26 = _mm_sub_epi16(stp1_25, stp1_26); - stp2_27 = _mm_sub_epi16(stp1_24, stp1_27); - stp2_28 = _mm_sub_epi16(stp1_31, stp1_28); - stp2_29 = _mm_sub_epi16(stp1_30, stp1_29); - stp2_30 = _mm_add_epi16(stp1_29, stp1_30); - stp2_31 = _mm_add_epi16(stp1_28, stp1_31); + const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg3_2 = pair_set_epi16(cospi_12_64, -cospi_20_64); + const __m128i stg3_3 = pair_set_epi16(cospi_20_64, cospi_12_64); + butterfly(&in[4], &in[28], &stg3_0, &stg3_1, &u4, &u7); + butterfly(&in[20], &in[12], &stg3_2, &stg3_3, &u5, &u6); } - /* Stage5 */ + v4 = _mm_add_epi16(u4, u5); + v5 = _mm_sub_epi16(u4, u5); + v6 = _mm_sub_epi16(u7, u6); + v7 = _mm_add_epi16(u7, u6); + { - const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp2_5); - const __m128i hi_6_5 = _mm_unpackhi_epi16(stp2_6, stp2_5); - const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); - const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); - - const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28); - const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28); - const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); - const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); - - const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); - const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); - - stp1_0 = _mm_add_epi16(stp2_0, stp2_3); - stp1_1 = _mm_add_epi16(stp2_1, stp2_2); - stp1_2 = _mm_sub_epi16(stp2_1, stp2_2); - stp1_3 = _mm_sub_epi16(stp2_0, stp2_3); - - tmp0 = _mm_madd_epi16(lo_6_5, stg4_1); - tmp1 = _mm_madd_epi16(hi_6_5, stg4_1); - tmp2 = _mm_madd_epi16(lo_6_5, stg4_0); - tmp3 = _mm_madd_epi16(hi_6_5, stg4_0); - - tmp0 = _mm_add_epi32(tmp0, rounding); - tmp1 = _mm_add_epi32(tmp1, rounding); - tmp2 = _mm_add_epi32(tmp2, rounding); - tmp3 = _mm_add_epi32(tmp3, rounding); - - tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); - tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); - tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); - tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); - - stp1_5 = _mm_packs_epi32(tmp0, tmp1); - stp1_6 = _mm_packs_epi32(tmp2, tmp3); - - stp1_4 = stp2_4; - stp1_7 = stp2_7; - - stp1_8 = _mm_add_epi16(stp2_8, stp2_11); - stp1_9 = _mm_add_epi16(stp2_9, stp2_10); - stp1_10 = _mm_sub_epi16(stp2_9, stp2_10); - stp1_11 = _mm_sub_epi16(stp2_8, stp2_11); - stp1_12 = _mm_sub_epi16(stp2_15, stp2_12); - stp1_13 = _mm_sub_epi16(stp2_14, stp2_13); - stp1_14 = _mm_add_epi16(stp2_14, stp2_13); - stp1_15 = _mm_add_epi16(stp2_15, stp2_12); - - stp1_16 = stp2_16; - stp1_17 = stp2_17; - - MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4, - stg4_5, stg4_4, stg4_5, stp1_18, stp1_29, stp1_19, - stp1_28) - MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6, - stg4_4, stg4_6, stg4_4, stp1_20, stp1_27, stp1_21, - stp1_26) - - stp1_22 = stp2_22; - stp1_23 = stp2_23; - stp1_24 = stp2_24; - stp1_25 = stp2_25; - stp1_30 = stp2_30; - stp1_31 = stp2_31; + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg4_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i stg4_3 = pair_set_epi16(cospi_8_64, cospi_24_64); + butterfly(&v6, &v5, &stg4_1, &stg4_0, &v5, &v6); + + butterfly(&in[0], &in[16], &stg4_0, &stg4_1, &u0, &u1); + butterfly(&in[8], &in[24], &stg4_2, &stg4_3, &u2, &u3); } - /* Stage6 */ + v0 = _mm_add_epi16(u0, u3); + v1 = _mm_add_epi16(u1, u2); + v2 = _mm_sub_epi16(u1, u2); + v3 = _mm_sub_epi16(u0, u3); + + out[0] = _mm_add_epi16(v0, v7); + out[1] = _mm_add_epi16(v1, v6); + out[2] = _mm_add_epi16(v2, v5); + out[3] = _mm_add_epi16(v3, v4); + out[4] = _mm_sub_epi16(v3, v4); + out[5] = _mm_sub_epi16(v2, v5); + out[6] = _mm_sub_epi16(v1, v6); + out[7] = _mm_sub_epi16(v0, v7); +} + +// For each 8x32 block __m128i in[32], +// Input with odd index, +// 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +// output pixels: 16-23, 24-31 in __m128i in[32] +// We avoid hide an offset, 16, inside this function. So we output 0-15 into +// array out[16] +static void idct32_full_8x32_quarter_3_4(const __m128i *in /*in[32]*/, + __m128i *out /*out[16]*/) { + __m128i v16, v17, v18, v19, v20, v21, v22, v23; + __m128i v24, v25, v26, v27, v28, v29, v30, v31; + __m128i u16, u17, u18, u19, u20, u21, u22, u23; + __m128i u24, u25, u26, u27, u28, u29, u30, u31; + { - const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); - const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); - const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12); - const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1_11, stp1_12); - - stp2_0 = _mm_add_epi16(stp1_0, stp1_7); - stp2_1 = _mm_add_epi16(stp1_1, stp1_6); - stp2_2 = _mm_add_epi16(stp1_2, stp1_5); - stp2_3 = _mm_add_epi16(stp1_3, stp1_4); - stp2_4 = _mm_sub_epi16(stp1_3, stp1_4); - stp2_5 = _mm_sub_epi16(stp1_2, stp1_5); - stp2_6 = _mm_sub_epi16(stp1_1, stp1_6); - stp2_7 = _mm_sub_epi16(stp1_0, stp1_7); - - stp2_8 = stp1_8; - stp2_9 = stp1_9; - stp2_14 = stp1_14; - stp2_15 = stp1_15; - - MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0, - stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11, - stp2_12) - - stp2_16 = _mm_add_epi16(stp1_16, stp1_23); - stp2_17 = _mm_add_epi16(stp1_17, stp1_22); - stp2_18 = _mm_add_epi16(stp1_18, stp1_21); - stp2_19 = _mm_add_epi16(stp1_19, stp1_20); - stp2_20 = _mm_sub_epi16(stp1_19, stp1_20); - stp2_21 = _mm_sub_epi16(stp1_18, stp1_21); - stp2_22 = _mm_sub_epi16(stp1_17, stp1_22); - stp2_23 = _mm_sub_epi16(stp1_16, stp1_23); - - stp2_24 = _mm_sub_epi16(stp1_31, stp1_24); - stp2_25 = _mm_sub_epi16(stp1_30, stp1_25); - stp2_26 = _mm_sub_epi16(stp1_29, stp1_26); - stp2_27 = _mm_sub_epi16(stp1_28, stp1_27); - stp2_28 = _mm_add_epi16(stp1_27, stp1_28); - stp2_29 = _mm_add_epi16(stp1_26, stp1_29); - stp2_30 = _mm_add_epi16(stp1_25, stp1_30); - stp2_31 = _mm_add_epi16(stp1_24, stp1_31); + const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64); + const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64); + const __m128i stg1_2 = pair_set_epi16(cospi_15_64, -cospi_17_64); + const __m128i stg1_3 = pair_set_epi16(cospi_17_64, cospi_15_64); + const __m128i stg1_4 = pair_set_epi16(cospi_23_64, -cospi_9_64); + const __m128i stg1_5 = pair_set_epi16(cospi_9_64, cospi_23_64); + const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64); + const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64); + const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64); + const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64); + const __m128i stg1_10 = pair_set_epi16(cospi_11_64, -cospi_21_64); + const __m128i stg1_11 = pair_set_epi16(cospi_21_64, cospi_11_64); + const __m128i stg1_12 = pair_set_epi16(cospi_19_64, -cospi_13_64); + const __m128i stg1_13 = pair_set_epi16(cospi_13_64, cospi_19_64); + const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64); + const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64); + butterfly(&in[1], &in[31], &stg1_0, &stg1_1, &u16, &u31); + butterfly(&in[17], &in[15], &stg1_2, &stg1_3, &u17, &u30); + butterfly(&in[9], &in[23], &stg1_4, &stg1_5, &u18, &u29); + butterfly(&in[25], &in[7], &stg1_6, &stg1_7, &u19, &u28); + + butterfly(&in[5], &in[27], &stg1_8, &stg1_9, &u20, &u27); + butterfly(&in[21], &in[11], &stg1_10, &stg1_11, &u21, &u26); + + butterfly(&in[13], &in[19], &stg1_12, &stg1_13, &u22, &u25); + butterfly(&in[29], &in[3], &stg1_14, &stg1_15, &u23, &u24); } - /* Stage7 */ + v16 = _mm_add_epi16(u16, u17); + v17 = _mm_sub_epi16(u16, u17); + v18 = _mm_sub_epi16(u19, u18); + v19 = _mm_add_epi16(u19, u18); + + v20 = _mm_add_epi16(u20, u21); + v21 = _mm_sub_epi16(u20, u21); + v22 = _mm_sub_epi16(u23, u22); + v23 = _mm_add_epi16(u23, u22); + + v24 = _mm_add_epi16(u24, u25); + v25 = _mm_sub_epi16(u24, u25); + v26 = _mm_sub_epi16(u27, u26); + v27 = _mm_add_epi16(u27, u26); + + v28 = _mm_add_epi16(u28, u29); + v29 = _mm_sub_epi16(u28, u29); + v30 = _mm_sub_epi16(u31, u30); + v31 = _mm_add_epi16(u31, u30); + + { + const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); + const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); + butterfly_self(&v17, &v30, stg3_4, stg3_5); + butterfly_self(&v18, &v29, stg3_6, stg3_4); + butterfly_self(&v21, &v26, stg3_8, stg3_9); + butterfly_self(&v22, &v25, stg3_10, stg3_8); + } + + u16 = _mm_add_epi16(v16, v19); + u17 = _mm_add_epi16(v17, v18); + u18 = _mm_sub_epi16(v17, v18); + u19 = _mm_sub_epi16(v16, v19); + u20 = _mm_sub_epi16(v23, v20); + u21 = _mm_sub_epi16(v22, v21); + u22 = _mm_add_epi16(v22, v21); + u23 = _mm_add_epi16(v23, v20); + + u24 = _mm_add_epi16(v24, v27); + u25 = _mm_add_epi16(v25, v26); + u26 = _mm_sub_epi16(v25, v26); + u27 = _mm_sub_epi16(v24, v27); + + u28 = _mm_sub_epi16(v31, v28); + u29 = _mm_sub_epi16(v30, v29); + u30 = _mm_add_epi16(v29, v30); + u31 = _mm_add_epi16(v28, v31); + { - const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); - const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); - const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); - const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); - - const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); - const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); - const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24); - const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24); - - stp1_0 = _mm_add_epi16(stp2_0, stp2_15); - stp1_1 = _mm_add_epi16(stp2_1, stp2_14); - stp1_2 = _mm_add_epi16(stp2_2, stp2_13); - stp1_3 = _mm_add_epi16(stp2_3, stp2_12); - stp1_4 = _mm_add_epi16(stp2_4, stp2_11); - stp1_5 = _mm_add_epi16(stp2_5, stp2_10); - stp1_6 = _mm_add_epi16(stp2_6, stp2_9); - stp1_7 = _mm_add_epi16(stp2_7, stp2_8); - stp1_8 = _mm_sub_epi16(stp2_7, stp2_8); - stp1_9 = _mm_sub_epi16(stp2_6, stp2_9); - stp1_10 = _mm_sub_epi16(stp2_5, stp2_10); - stp1_11 = _mm_sub_epi16(stp2_4, stp2_11); - stp1_12 = _mm_sub_epi16(stp2_3, stp2_12); - stp1_13 = _mm_sub_epi16(stp2_2, stp2_13); - stp1_14 = _mm_sub_epi16(stp2_1, stp2_14); - stp1_15 = _mm_sub_epi16(stp2_0, stp2_15); - - stp1_16 = stp2_16; - stp1_17 = stp2_17; - stp1_18 = stp2_18; - stp1_19 = stp2_19; - - MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0, - stg4_0, stg6_0, stg4_0, stp1_20, stp1_27, stp1_21, - stp1_26) - MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0, - stg4_0, stg6_0, stg4_0, stp1_22, stp1_25, stp1_23, - stp1_24) - - stp1_28 = stp2_28; - stp1_29 = stp2_29; - stp1_30 = stp2_30; - stp1_31 = stp2_31; + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + butterfly_self(&u18, &u29, stg4_4, stg4_5); + butterfly_self(&u19, &u28, stg4_4, stg4_5); + butterfly_self(&u20, &u27, stg4_6, stg4_4); + butterfly_self(&u21, &u26, stg4_6, stg4_4); } - out[0] = _mm_add_epi16(stp1_0, stp1_31); - out[1] = _mm_add_epi16(stp1_1, stp1_30); - out[2] = _mm_add_epi16(stp1_2, stp1_29); - out[3] = _mm_add_epi16(stp1_3, stp1_28); - out[4] = _mm_add_epi16(stp1_4, stp1_27); - out[5] = _mm_add_epi16(stp1_5, stp1_26); - out[6] = _mm_add_epi16(stp1_6, stp1_25); - out[7] = _mm_add_epi16(stp1_7, stp1_24); - out[8] = _mm_add_epi16(stp1_8, stp1_23); - out[9] = _mm_add_epi16(stp1_9, stp1_22); - out[10] = _mm_add_epi16(stp1_10, stp1_21); - out[11] = _mm_add_epi16(stp1_11, stp1_20); - out[12] = _mm_add_epi16(stp1_12, stp1_19); - out[13] = _mm_add_epi16(stp1_13, stp1_18); - out[14] = _mm_add_epi16(stp1_14, stp1_17); - out[15] = _mm_add_epi16(stp1_15, stp1_16); - out[16] = _mm_sub_epi16(stp1_15, stp1_16); - out[17] = _mm_sub_epi16(stp1_14, stp1_17); - out[18] = _mm_sub_epi16(stp1_13, stp1_18); - out[19] = _mm_sub_epi16(stp1_12, stp1_19); - out[20] = _mm_sub_epi16(stp1_11, stp1_20); - out[21] = _mm_sub_epi16(stp1_10, stp1_21); - out[22] = _mm_sub_epi16(stp1_9, stp1_22); - out[23] = _mm_sub_epi16(stp1_8, stp1_23); - out[24] = _mm_sub_epi16(stp1_7, stp1_24); - out[25] = _mm_sub_epi16(stp1_6, stp1_25); - out[26] = _mm_sub_epi16(stp1_5, stp1_26); - out[27] = _mm_sub_epi16(stp1_4, stp1_27); - out[28] = _mm_sub_epi16(stp1_3, stp1_28); - out[29] = _mm_sub_epi16(stp1_2, stp1_29); - out[30] = _mm_sub_epi16(stp1_1, stp1_30); - out[31] = _mm_sub_epi16(stp1_0, stp1_31); + out[0] = _mm_add_epi16(u16, u23); + out[1] = _mm_add_epi16(u17, u22); + out[2] = _mm_add_epi16(u18, u21); + out[3] = _mm_add_epi16(u19, u20); + out[4] = _mm_sub_epi16(u19, u20); + out[5] = _mm_sub_epi16(u18, u21); + out[6] = _mm_sub_epi16(u17, u22); + out[7] = _mm_sub_epi16(u16, u23); + + out[8] = _mm_sub_epi16(u31, u24); + out[9] = _mm_sub_epi16(u30, u25); + out[10] = _mm_sub_epi16(u29, u26); + out[11] = _mm_sub_epi16(u28, u27); + out[12] = _mm_add_epi16(u27, u28); + out[13] = _mm_add_epi16(u26, u29); + out[14] = _mm_add_epi16(u25, u30); + out[15] = _mm_add_epi16(u24, u31); + + { + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + butterfly_self(&out[4], &out[11], stg6_0, stg4_0); + butterfly_self(&out[5], &out[10], stg6_0, stg4_0); + butterfly_self(&out[6], &out[9], stg6_0, stg4_0); + butterfly_self(&out[7], &out[8], stg6_0, stg4_0); + } +} + +static void idct32_full_8x32_quarter_1_2(const __m128i *in /*in[32]*/, + __m128i *out /*out[32]*/) { + __m128i temp[16]; + idct32_full_8x32_quarter_1(in, temp); + idct32_full_8x32_quarter_2(in, &temp[8]); + add_sub_butterfly(temp, out, 16); +} + +static void idct32_full_8x32(const __m128i *in /*in[32]*/, + __m128i *out /*out[32]*/) { + __m128i temp[32]; + idct32_full_8x32_quarter_1_2(in, temp); + idct32_full_8x32_quarter_3_4(in, &temp[16]); + add_sub_butterfly(temp, out, 32); } static void load_buffer_8x32(const tran_low_t *input, __m128i *in) { @@ -1473,7 +1314,7 @@ void vpx_idct32x32_1024_add_ssse3(const tran_low_t *input, uint8_t *dest, array_transpose_8x8(in + 16, in + 16); array_transpose_8x8(in + 24, in + 24); - idct32_8x32(in, col + (i << 5)); + idct32_full_8x32(in, col + (i << 5)); } // columns @@ -1485,7 +1326,7 @@ void vpx_idct32x32_1024_add_ssse3(const tran_low_t *input, uint8_t *dest, array_transpose_8x8(col + j + 64, in + 16); array_transpose_8x8(col + j + 96, in + 24); - idct32_8x32(in, in); + idct32_full_8x32(in, in); store_buffer_8x32(in, dest, stride); dest += 8; } |