neon: vpx_quantize_b

With skip block or coeff < zbin it is about twice as fast as C.

If most coeff values are > zbin it is about 10-15x as fast as C.

BUG=webm:1426

Change-Id: I5d3c007b014a372d5ef0882b39bb48983b4131c7

3 files changed, 157 insertions, 1 deletions

diff --git a/vpx_dsp/arm/quantize_neon.c b/vpx_dsp/arm/quantize_neon.c
new file mode 100644
index 000000000..8bed05703
--- /dev/null
+++ b/vpx_dsp/arm/quantize_neon.c
@@ -0,0 +1,155 @@
+/*
+ *  Copyright (c) 2017 The WebM project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+
+#include "./vpx_dsp_rtcd.h"
+#include "vpx_dsp/arm/mem_neon.h"
+
+void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+                         int skip_block, const int16_t *zbin_ptr,
+                         const int16_t *round_ptr, const int16_t *quant_ptr,
+                         const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+                         tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
+                         uint16_t *eob_ptr, const int16_t *scan_ptr,
+                         const int16_t *iscan_ptr) {
+  const int16x8_t zero = vdupq_n_s16(0);
+  const int16x8_t one = vdupq_n_s16(1);
+  const int16x8_t neg_one = vdupq_n_s16(-1);
+  uint16x8_t eob_max;
+  (void)scan_ptr;
+
+  if (skip_block) {
+    do {
+      store_s16q_to_tran_low(qcoeff_ptr, zero);
+      store_s16q_to_tran_low(dqcoeff_ptr, zero);
+      qcoeff_ptr += 8;
+      dqcoeff_ptr += 8;
+      n_coeffs -= 8;
+    } while (n_coeffs > 0);
+
+    *eob_ptr = 0;
+    return;
+  }
+
+  // Process first 8 values which include a dc component.
+  {
+    // Only the first element of each vector is DC.
+    const int16x8_t zbin = vld1q_s16(zbin_ptr);
+    const int16x8_t round = vld1q_s16(round_ptr);
+    const int16x8_t quant = vld1q_s16(quant_ptr);
+    const int16x8_t quant_shift = vld1q_s16(quant_shift_ptr);
+    const int16x8_t dequant = vld1q_s16(dequant_ptr);
+    // Add one because the eob does not index from 0.
+    const uint16x8_t iscan =
+        vreinterpretq_u16_s16(vaddq_s16(vld1q_s16(iscan_ptr), one));
+
+    const int16x8_t coeff = load_tran_low_to_s16q(coeff_ptr);
+    const int16x8_t coeff_sign = vshrq_n_s16(coeff, 15);
+    const int16x8_t coeff_abs = vabsq_s16(coeff);
+
+    const int16x8_t zbin_mask =
+        vreinterpretq_s16_u16(vcgeq_s16(coeff_abs, zbin));
+
+    const int16x8_t rounded = vqaddq_s16(coeff_abs, round);
+
+    // (round * quant * 2) >> 16 >> 1 == (round * quant) >> 16
+    int16x8_t qcoeff = vshrq_n_s16(vqdmulhq_s16(rounded, quant), 1);
+
+    qcoeff = vaddq_s16(qcoeff, rounded);
+
+    // (qcoeff * quant_shift * 2) >> 16 >> 1 == (qcoeff * quant_shift) >> 16
+    qcoeff = vshrq_n_s16(vqdmulhq_s16(qcoeff, quant_shift), 1);
+
+    // Restore the sign bit.
+    qcoeff = veorq_s16(qcoeff, coeff_sign);
+    qcoeff = vsubq_s16(qcoeff, coeff_sign);
+
+    qcoeff = vandq_s16(qcoeff, zbin_mask);
+
+    // Set non-zero elements to -1 and use that to extract values for eob.
+    eob_max = vandq_u16(vtstq_s16(qcoeff, neg_one), iscan);
+
+    coeff_ptr += 8;
+    iscan_ptr += 8;
+
+    store_s16q_to_tran_low(qcoeff_ptr, qcoeff);
+    qcoeff_ptr += 8;
+
+    qcoeff = vmulq_s16(qcoeff, dequant);
+
+    store_s16q_to_tran_low(dqcoeff_ptr, qcoeff);
+    dqcoeff_ptr += 8;
+  }
+
+  n_coeffs -= 8;
+
+  {
+    const int16x8_t zbin = vdupq_n_s16(zbin_ptr[1]);
+    const int16x8_t round = vdupq_n_s16(round_ptr[1]);
+    const int16x8_t quant = vdupq_n_s16(quant_ptr[1]);
+    const int16x8_t quant_shift = vdupq_n_s16(quant_shift_ptr[1]);
+    const int16x8_t dequant = vdupq_n_s16(dequant_ptr[1]);
+
+    do {
+      // Add one because the eob is not it's index.
+      const uint16x8_t iscan =
+          vreinterpretq_u16_s16(vaddq_s16(vld1q_s16(iscan_ptr), one));
+
+      const int16x8_t coeff = load_tran_low_to_s16q(coeff_ptr);
+      const int16x8_t coeff_sign = vshrq_n_s16(coeff, 15);
+      const int16x8_t coeff_abs = vabsq_s16(coeff);
+
+      const int16x8_t zbin_mask =
+          vreinterpretq_s16_u16(vcgeq_s16(coeff_abs, zbin));
+
+      const int16x8_t rounded = vqaddq_s16(coeff_abs, round);
+
+      // (round * quant * 2) >> 16 >> 1 == (round * quant) >> 16
+      int16x8_t qcoeff = vshrq_n_s16(vqdmulhq_s16(rounded, quant), 1);
+
+      qcoeff = vaddq_s16(qcoeff, rounded);
+
+      // (qcoeff * quant_shift * 2) >> 16 >> 1 == (qcoeff * quant_shift) >> 16
+      qcoeff = vshrq_n_s16(vqdmulhq_s16(qcoeff, quant_shift), 1);
+
+      // Restore the sign bit.
+      qcoeff = veorq_s16(qcoeff, coeff_sign);
+      qcoeff = vsubq_s16(qcoeff, coeff_sign);
+
+      qcoeff = vandq_s16(qcoeff, zbin_mask);
+
+      // Set non-zero elements to -1 and use that to extract values for eob.
+      eob_max =
+          vmaxq_u16(eob_max, vandq_u16(vtstq_s16(qcoeff, neg_one), iscan));
+
+      coeff_ptr += 8;
+      iscan_ptr += 8;
+
+      store_s16q_to_tran_low(qcoeff_ptr, qcoeff);
+      qcoeff_ptr += 8;
+
+      qcoeff = vmulq_s16(qcoeff, dequant);
+
+      store_s16q_to_tran_low(dqcoeff_ptr, qcoeff);
+      dqcoeff_ptr += 8;
+
+      n_coeffs -= 8;
+    } while (n_coeffs > 0);
+  }
+
+  {
+    const uint16x4_t eob_max_0 =
+        vmax_u16(vget_low_u16(eob_max), vget_high_u16(eob_max));
+    const uint16x4_t eob_max_1 = vpmax_u16(eob_max_0, eob_max_0);
+    const uint16x4_t eob_max_2 = vpmax_u16(eob_max_1, eob_max_1);
+    vst1_lane_u16(eob_ptr, eob_max_2, 0);
+  }
+}
diff --git a/vpx_dsp/vpx_dsp.mk b/vpx_dsp/vpx_dsp.mk
index 4dc1dbadd..ff61685a4 100644
--- a/vpx_dsp/vpx_dsp.mk
+++ b/vpx_dsp/vpx_dsp.mk
@@ -274,6 +274,7 @@ DSP_SRCS-yes            += quantize.c
 DSP_SRCS-yes            += quantize.h
 
 DSP_SRCS-$(HAVE_SSE2)   += x86/quantize_sse2.c
+DSP_SRCS-$(HAVE_NEON)   += arm/quantize_neon.c
 ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
 DSP_SRCS-$(HAVE_SSE2)   += x86/highbd_quantize_intrin_sse2.c
 endif
diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl
index 358d16914..275ad7a83 100644
--- a/vpx_dsp/vpx_dsp_rtcd_defs.pl
+++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl
@@ -673,7 +673,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
 #
 if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") {
   add_proto qw/void vpx_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
-  specialize qw/vpx_quantize_b sse2/, "$ssse3_x86_64", "$avx_x86_64";
+  specialize qw/vpx_quantize_b neon sse2/, "$ssse3_x86_64", "$avx_x86_64";
 
   add_proto qw/void vpx_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
   specialize qw/vpx_quantize_b_32x32/, "$ssse3_x86_64", "$avx_x86_64";