From f357335c38c82dc6e4eb27a587dd13d42b0d45a2 Mon Sep 17 00:00:00 2001 From: Linfeng Zhang Date: Tue, 19 Sep 2017 16:14:56 -0700 Subject: Refactor convolve NEON code Rename a couple of hbd static functions. Move the position of NEON function convolve8_4(). Change-Id: Idfac00edf2e99cdd8e0a73b9f895402f60be6349 --- vpx_dsp/arm/highbd_vpx_convolve8_neon.c | 136 ++++++++++++++++++-------------- vpx_dsp/arm/vpx_convolve8_neon.c | 22 ------ vpx_dsp/arm/vpx_convolve8_neon.h | 22 ++++++ 3 files changed, 99 insertions(+), 81 deletions(-) (limited to 'vpx_dsp/arm') diff --git a/vpx_dsp/arm/highbd_vpx_convolve8_neon.c b/vpx_dsp/arm/highbd_vpx_convolve8_neon.c index da02488da..c46c01631 100644 --- a/vpx_dsp/arm/highbd_vpx_convolve8_neon.c +++ b/vpx_dsp/arm/highbd_vpx_convolve8_neon.c @@ -85,11 +85,10 @@ static INLINE void store_8x8(uint16_t *s, const ptrdiff_t p, vst1q_u16(s, s7); } -static INLINE int32x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1, - const int16x4_t s2, const int16x4_t s3, - const int16x4_t s4, const int16x4_t s5, - const int16x4_t s6, const int16x4_t s7, - const int16x8_t filters) { +static INLINE int32x4_t highbd_convolve8_4( + const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, + const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, + const int16x4_t s6, const int16x4_t s7, const int16x8_t filters) { const int16x4_t filters_lo = vget_low_s16(filters); const int16x4_t filters_hi = vget_high_s16(filters); int32x4_t sum; @@ -105,12 +104,11 @@ static INLINE int32x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1, return sum; } -static INLINE uint16x8_t convolve8_8(const int16x8_t s0, const int16x8_t s1, - const int16x8_t s2, const int16x8_t s3, - const int16x8_t s4, const int16x8_t s5, - const int16x8_t s6, const int16x8_t s7, - const int16x8_t filters, - const uint16x8_t max) { +static INLINE uint16x8_t +highbd_convolve8_8(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, + const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, + const int16x8_t s6, const int16x8_t s7, + const int16x8_t filters, const uint16x8_t max) { const int16x4_t filters_lo = vget_low_s16(filters); const int16x4_t filters_hi = vget_high_s16(filters); int32x4_t sum0, sum1; @@ -183,10 +181,10 @@ void vpx_highbd_convolve8_horiz_neon(const uint16_t *src, ptrdiff_t src_stride, load_4x4((const int16_t *)src, src_stride, &s7, &s8, &s9, &s10); transpose_s16_4x4d(&s7, &s8, &s9, &s10); - d0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters); - d1 = convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters); - d2 = convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters); - d3 = convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters); + d0 = highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters); + d1 = highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters); + d2 = highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters); + d3 = highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters); d01 = vcombine_u16(vqrshrun_n_s32(d0, 7), vqrshrun_n_s32(d1, 7)); d23 = vcombine_u16(vqrshrun_n_s32(d2, 7), vqrshrun_n_s32(d3, 7)); @@ -242,10 +240,11 @@ void vpx_highbd_convolve8_horiz_neon(const uint16_t *src, ptrdiff_t src_stride, __builtin_prefetch(src + 5 * src_stride); __builtin_prefetch(src + 6 * src_stride); __builtin_prefetch(src + 7 * src_stride); - d0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); - d1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); - d2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); - d3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); + d0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); + d1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); + d2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); + d3 = + highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); transpose_u16_8x4(&d0, &d1, &d2, &d3); vst1_u16(dst, vget_low_u16(d0)); @@ -303,14 +302,22 @@ void vpx_highbd_convolve8_horiz_neon(const uint16_t *src, ptrdiff_t src_stride, &s12, &s13, &s14); transpose_s16_8x8(&s7, &s8, &s9, &s10, &s11, &s12, &s13, &s14); - d0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); - d1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); - d2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); - d3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); - d4 = convolve8_8(s4, s5, s6, s7, s8, s9, s10, s11, filters, max); - d5 = convolve8_8(s5, s6, s7, s8, s9, s10, s11, s12, filters, max); - d6 = convolve8_8(s6, s7, s8, s9, s10, s11, s12, s13, filters, max); - d7 = convolve8_8(s7, s8, s9, s10, s11, s12, s13, s14, filters, max); + d0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, + max); + d1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, + max); + d2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, + max); + d3 = highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, + max); + d4 = highbd_convolve8_8(s4, s5, s6, s7, s8, s9, s10, s11, filters, + max); + d5 = highbd_convolve8_8(s5, s6, s7, s8, s9, s10, s11, s12, filters, + max); + d6 = highbd_convolve8_8(s6, s7, s8, s9, s10, s11, s12, s13, filters, + max); + d7 = highbd_convolve8_8(s7, s8, s9, s10, s11, s12, s13, s14, + filters, max); transpose_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7); store_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7); @@ -383,10 +390,10 @@ void vpx_highbd_convolve8_avg_horiz_neon(const uint16_t *src, load_4x4((const int16_t *)src, src_stride, &s7, &s8, &s9, &s10); transpose_s16_4x4d(&s7, &s8, &s9, &s10); - d0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters); - d1 = convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters); - d2 = convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters); - d3 = convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters); + d0 = highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters); + d1 = highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters); + d2 = highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters); + d3 = highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters); t01 = vcombine_u16(vqrshrun_n_s32(d0, 7), vqrshrun_n_s32(d1, 7)); t23 = vcombine_u16(vqrshrun_n_s32(d2, 7), vqrshrun_n_s32(d3, 7)); @@ -449,10 +456,11 @@ void vpx_highbd_convolve8_avg_horiz_neon(const uint16_t *src, __builtin_prefetch(src + 5 * src_stride); __builtin_prefetch(src + 6 * src_stride); __builtin_prefetch(src + 7 * src_stride); - t0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); - t1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); - t2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); - t3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); + t0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); + t1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); + t2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); + t3 = + highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); transpose_u16_8x4(&t0, &t1, &t2, &t3); d0 = vcombine_u16(vld1_u16(dst + 0 * dst_stride), @@ -523,14 +531,22 @@ void vpx_highbd_convolve8_avg_horiz_neon(const uint16_t *src, &s12, &s13, &s14); transpose_s16_8x8(&s7, &s8, &s9, &s10, &s11, &s12, &s13, &s14); - d0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); - d1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); - d2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); - d3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); - d4 = convolve8_8(s4, s5, s6, s7, s8, s9, s10, s11, filters, max); - d5 = convolve8_8(s5, s6, s7, s8, s9, s10, s11, s12, filters, max); - d6 = convolve8_8(s6, s7, s8, s9, s10, s11, s12, s13, filters, max); - d7 = convolve8_8(s7, s8, s9, s10, s11, s12, s13, s14, filters, max); + d0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, + max); + d1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, + max); + d2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, + max); + d3 = highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, + max); + d4 = highbd_convolve8_8(s4, s5, s6, s7, s8, s9, s10, s11, filters, + max); + d5 = highbd_convolve8_8(s5, s6, s7, s8, s9, s10, s11, s12, filters, + max); + d6 = highbd_convolve8_8(s6, s7, s8, s9, s10, s11, s12, s13, filters, + max); + d7 = highbd_convolve8_8(s7, s8, s9, s10, s11, s12, s13, s14, + filters, max); transpose_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7); @@ -620,10 +636,10 @@ void vpx_highbd_convolve8_vert_neon(const uint16_t *src, ptrdiff_t src_stride, __builtin_prefetch(src + 1 * src_stride); __builtin_prefetch(src + 2 * src_stride); __builtin_prefetch(src + 3 * src_stride); - d0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters); - d1 = convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters); - d2 = convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters); - d3 = convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters); + d0 = highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters); + d1 = highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters); + d2 = highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters); + d3 = highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters); d01 = vcombine_u16(vqrshrun_n_s32(d0, 7), vqrshrun_n_s32(d1, 7)); d23 = vcombine_u16(vqrshrun_n_s32(d2, 7), vqrshrun_n_s32(d3, 7)); @@ -698,10 +714,11 @@ void vpx_highbd_convolve8_vert_neon(const uint16_t *src, ptrdiff_t src_stride, __builtin_prefetch(s + 1 * src_stride); __builtin_prefetch(s + 2 * src_stride); __builtin_prefetch(s + 3 * src_stride); - d0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); - d1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); - d2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); - d3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); + d0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); + d1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); + d2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); + d3 = + highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); vst1q_u16(d, d0); d += dst_stride; @@ -786,10 +803,10 @@ void vpx_highbd_convolve8_avg_vert_neon(const uint16_t *src, __builtin_prefetch(src + 1 * src_stride); __builtin_prefetch(src + 2 * src_stride); __builtin_prefetch(src + 3 * src_stride); - d0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters); - d1 = convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters); - d2 = convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters); - d3 = convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters); + d0 = highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters); + d1 = highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters); + d2 = highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters); + d3 = highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters); t01 = vcombine_u16(vqrshrun_n_s32(d0, 7), vqrshrun_n_s32(d1, 7)); t23 = vcombine_u16(vqrshrun_n_s32(d2, 7), vqrshrun_n_s32(d3, 7)); @@ -872,10 +889,11 @@ void vpx_highbd_convolve8_avg_vert_neon(const uint16_t *src, __builtin_prefetch(s + 1 * src_stride); __builtin_prefetch(s + 2 * src_stride); __builtin_prefetch(s + 3 * src_stride); - t0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); - t1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); - t2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); - t3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); + t0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max); + t1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max); + t2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max); + t3 = + highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max); d0 = vld1q_u16(d + 0 * dst_stride); d1 = vld1q_u16(d + 1 * dst_stride); diff --git a/vpx_dsp/arm/vpx_convolve8_neon.c b/vpx_dsp/arm/vpx_convolve8_neon.c index 1325569fc..08ae17dba 100644 --- a/vpx_dsp/arm/vpx_convolve8_neon.c +++ b/vpx_dsp/arm/vpx_convolve8_neon.c @@ -52,28 +52,6 @@ static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p, vst1_u8(s, s7); } -static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1, - const int16x4_t s2, const int16x4_t s3, - const int16x4_t s4, const int16x4_t s5, - const int16x4_t s6, const int16x4_t s7, - const int16x8_t filters, - const int16x4_t filter3, - const int16x4_t filter4) { - const int16x4_t filters_lo = vget_low_s16(filters); - const int16x4_t filters_hi = vget_high_s16(filters); - int16x4_t sum; - - sum = vmul_lane_s16(s0, filters_lo, 0); - sum = vmla_lane_s16(sum, s1, filters_lo, 1); - sum = vmla_lane_s16(sum, s2, filters_lo, 2); - sum = vmla_lane_s16(sum, s5, filters_hi, 1); - sum = vmla_lane_s16(sum, s6, filters_hi, 2); - sum = vmla_lane_s16(sum, s7, filters_hi, 3); - sum = vqadd_s16(sum, vmul_s16(s3, filter3)); - sum = vqadd_s16(sum, vmul_s16(s4, filter4)); - return sum; -} - void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, diff --git a/vpx_dsp/arm/vpx_convolve8_neon.h b/vpx_dsp/arm/vpx_convolve8_neon.h index 0f3b2c970..bd20a263c 100644 --- a/vpx_dsp/arm/vpx_convolve8_neon.h +++ b/vpx_dsp/arm/vpx_convolve8_neon.h @@ -47,6 +47,28 @@ static INLINE void load_u8_8x8(const uint8_t *s, const ptrdiff_t p, *s7 = vld1_u8(s); } +static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1, + const int16x4_t s2, const int16x4_t s3, + const int16x4_t s4, const int16x4_t s5, + const int16x4_t s6, const int16x4_t s7, + const int16x8_t filters, + const int16x4_t filter3, + const int16x4_t filter4) { + const int16x4_t filters_lo = vget_low_s16(filters); + const int16x4_t filters_hi = vget_high_s16(filters); + int16x4_t sum; + + sum = vmul_lane_s16(s0, filters_lo, 0); + sum = vmla_lane_s16(sum, s1, filters_lo, 1); + sum = vmla_lane_s16(sum, s2, filters_lo, 2); + sum = vmla_lane_s16(sum, s5, filters_hi, 1); + sum = vmla_lane_s16(sum, s6, filters_hi, 2); + sum = vmla_lane_s16(sum, s7, filters_hi, 3); + sum = vqadd_s16(sum, vmul_s16(s3, filter3)); + sum = vqadd_s16(sum, vmul_s16(s4, filter4)); + return sum; +} + static INLINE uint8x8_t convolve8_8(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, -- cgit v1.2.3