partial fdct neon: maintain neon registers

Finish the calulations in neon registers. This avoids a potentially
expensive move from neon to gp and allows at least clang to store
directly to memory.

BUG=webm:1424

Change-Id: Idef25eec95f7610947167818e9194bde8b00d282

1 files changed, 28 insertions, 11 deletions

diff --git a/vpx_dsp/arm/fdct_partial_neon.c b/vpx_dsp/arm/fdct_partial_neon.c
index 3c4b292d7..51e69d026 100644
--- a/vpx_dsp/arm/fdct_partial_neon.c
+++ b/vpx_dsp/arm/fdct_partial_neon.c
@@ -14,18 +14,26 @@
 #include "./vpx_config.h"
 #include "vpx_dsp/arm/mem_neon.h"
 
-static INLINE int32_t sum_int16x8(const int16x8_t a) {
+static INLINE int32x2_t sum_int16x8(const int16x8_t a) {
   const int32x4_t b = vpaddlq_s16(a);
   const int64x2_t c = vpaddlq_s32(b);
-  const int32x2_t d = vadd_s32(vreinterpret_s32_s64(vget_low_s64(c)),
-                               vreinterpret_s32_s64(vget_high_s64(c)));
-  return vget_lane_s32(d, 0);
+  return vadd_s32(vreinterpret_s32_s64(vget_low_s64(c)),
+                  vreinterpret_s32_s64(vget_high_s64(c)));
+}
+
+static INLINE tran_low_t get_lane(const int32x2_t a) {
+#if CONFIG_VP9_HIGHBITDEPTH
+  return vget_lane_s32(a, 0);
+#else
+  return vget_lane_s16(vreinterpret_s16_s32(a), 0);
+#endif  // CONFIG_VP9_HIGHBITDETPH
 }
 
 void vpx_fdct4x4_1_neon(const int16_t *input, tran_low_t *output, int stride) {
   int16x4_t a0, a1, a2, a3;
   int16x8_t b0, b1;
   int16x8_t c;
+  int32x2_t d;
 
   a0 = vld1_s16(input);
   input += stride;
@@ -40,19 +48,22 @@ void vpx_fdct4x4_1_neon(const int16_t *input, tran_low_t *output, int stride) {
 
   c = vaddq_s16(b0, b1);
 
-  output[0] = (tran_low_t)(sum_int16x8(c) << 1);
+  d = sum_int16x8(c);
+
+  output[0] = get_lane(vshl_n_s32(d, 1));
   output[1] = 0;
 }
 
 void vpx_fdct8x8_1_neon(const int16_t *input, tran_low_t *output, int stride) {
   int r;
   int16x8_t sum = vld1q_s16(&input[0]);
+
   for (r = 1; r < 8; ++r) {
     const int16x8_t input_00 = vld1q_s16(&input[r * stride]);
     sum = vaddq_s16(sum, input_00);
   }
 
-  output[0] = (tran_low_t)sum_int16x8(sum);
+  output[0] = get_lane(sum_int16x8(sum));
   output[1] = 0;
 }
 
@@ -61,7 +72,9 @@ void vpx_fdct16x16_1_neon(const int16_t *input, tran_low_t *output,
   int r;
   int16x8_t left = vld1q_s16(input);
   int16x8_t right = vld1q_s16(input + 8);
+  int32x2_t sum;
   input += stride;
+
   for (r = 1; r < 16; ++r) {
     const int16x8_t a = vld1q_s16(input);
     const int16x8_t b = vld1q_s16(input + 8);
@@ -70,7 +83,9 @@ void vpx_fdct16x16_1_neon(const int16_t *input, tran_low_t *output,
     right = vaddq_s16(right, b);
   }
 
-  output[0] = (tran_low_t)((sum_int16x8(left) + sum_int16x8(right)) >> 1);
+  sum = vadd_s32(sum_int16x8(left), sum_int16x8(right));
+
+  output[0] = get_lane(vshr_n_s32(sum, 1));
   output[1] = 0;
 }
 
@@ -81,7 +96,9 @@ void vpx_fdct32x32_1_neon(const int16_t *input, tran_low_t *output,
   int16x8_t a1 = vld1q_s16(input + 8);
   int16x8_t a2 = vld1q_s16(input + 16);
   int16x8_t a3 = vld1q_s16(input + 24);
+  int32x2_t sum;
   input += stride;
+
   for (r = 1; r < 32; ++r) {
     const int16x8_t b0 = vld1q_s16(input);
     const int16x8_t b1 = vld1q_s16(input + 8);
@@ -94,9 +111,9 @@ void vpx_fdct32x32_1_neon(const int16_t *input, tran_low_t *output,
     a3 = vaddq_s16(a3, b3);
   }
 
-  // TODO(johannkoenig): sum and shift the values in neon registers.
-  output[0] = (tran_low_t)(
-      (sum_int16x8(a0) + sum_int16x8(a1) + sum_int16x8(a2) + sum_int16x8(a3)) >>
-      3);
+  sum = vadd_s32(sum_int16x8(a0), sum_int16x8(a1));
+  sum = vadd_s32(sum, sum_int16x8(a2));
+  sum = vadd_s32(sum, sum_int16x8(a3));
+  output[0] = get_lane(vshr_n_s32(sum, 3));
   output[1] = 0;
 }