/*
 *  Copyright (c) 2017 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <arm_neon.h>

#include "./vpx_config.h"
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/txfm_common.h"
#include "vpx_dsp/arm/mem_neon.h"
#include "vpx_dsp/arm/transpose_neon.h"
#include "vpx_dsp/arm/fdct_neon.h"
#include "vpx_dsp/arm/fdct32x32_neon.h"

// Most gcc 4.9 distributions outside of Android do not generate correct code
// for this function.
#if !defined(__clang__) && !defined(__ANDROID__) && defined(__GNUC__) && \
    __GNUC__ == 4 && __GNUC_MINOR__ <= 9

void vpx_fdct32x32_neon(const int16_t *input, tran_low_t *output, int stride) {
  vpx_fdct32x32_c(input, output, stride);
}

void vpx_fdct32x32_rd_neon(const int16_t *input, tran_low_t *output,
                           int stride) {
  vpx_fdct32x32_rd_c(input, output, stride);
}

#else

void vpx_fdct32x32_neon(const int16_t *input, tran_low_t *output, int stride) {
  int16x8_t temp0[32];
  int16x8_t temp1[32];
  int16x8_t temp2[32];
  int16x8_t temp3[32];
  int16x8_t temp4[32];
  int16x8_t temp5[32];

  // Process in 8x32 columns.
  load_cross(input, stride, temp0);
  scale_input(temp0, temp5);
  dct_body_first_pass(temp5, temp1);

  load_cross(input + 8, stride, temp0);
  scale_input(temp0, temp5);
  dct_body_first_pass(temp5, temp2);

  load_cross(input + 16, stride, temp0);
  scale_input(temp0, temp5);
  dct_body_first_pass(temp5, temp3);

  load_cross(input + 24, stride, temp0);
  scale_input(temp0, temp5);
  dct_body_first_pass(temp5, temp4);

  // Generate the top row by munging the first set of 8 from each one together.
  transpose_s16_8x8_new(&temp1[0], &temp0[0]);
  transpose_s16_8x8_new(&temp2[0], &temp0[8]);
  transpose_s16_8x8_new(&temp3[0], &temp0[16]);
  transpose_s16_8x8_new(&temp4[0], &temp0[24]);

  dct_body_second_pass(temp0, temp5);

  transpose_s16_8x8(&temp5[0], &temp5[1], &temp5[2], &temp5[3], &temp5[4],
                    &temp5[5], &temp5[6], &temp5[7]);
  transpose_s16_8x8(&temp5[8], &temp5[9], &temp5[10], &temp5[11], &temp5[12],
                    &temp5[13], &temp5[14], &temp5[15]);
  transpose_s16_8x8(&temp5[16], &temp5[17], &temp5[18], &temp5[19], &temp5[20],
                    &temp5[21], &temp5[22], &temp5[23]);
  transpose_s16_8x8(&temp5[24], &temp5[25], &temp5[26], &temp5[27], &temp5[28],
                    &temp5[29], &temp5[30], &temp5[31]);
  store(output, temp5);

  // Second row of 8x32.
  transpose_s16_8x8_new(&temp1[8], &temp0[0]);
  transpose_s16_8x8_new(&temp2[8], &temp0[8]);
  transpose_s16_8x8_new(&temp3[8], &temp0[16]);
  transpose_s16_8x8_new(&temp4[8], &temp0[24]);

  dct_body_second_pass(temp0, temp5);

  transpose_s16_8x8(&temp5[0], &temp5[1], &temp5[2], &temp5[3], &temp5[4],
                    &temp5[5], &temp5[6], &temp5[7]);
  transpose_s16_8x8(&temp5[8], &temp5[9], &temp5[10], &temp5[11], &temp5[12],
                    &temp5[13], &temp5[14], &temp5[15]);
  transpose_s16_8x8(&temp5[16], &temp5[17], &temp5[18], &temp5[19], &temp5[20],
                    &temp5[21], &temp5[22], &temp5[23]);
  transpose_s16_8x8(&temp5[24], &temp5[25], &temp5[26], &temp5[27], &temp5[28],
                    &temp5[29], &temp5[30], &temp5[31]);
  store(output + 8 * 32, temp5);

  // Third row of 8x32
  transpose_s16_8x8_new(&temp1[16], &temp0[0]);
  transpose_s16_8x8_new(&temp2[16], &temp0[8]);
  transpose_s16_8x8_new(&temp3[16], &temp0[16]);
  transpose_s16_8x8_new(&temp4[16], &temp0[24]);

  dct_body_second_pass(temp0, temp5);

  transpose_s16_8x8(&temp5[0], &temp5[1], &temp5[2], &temp5[3], &temp5[4],
                    &temp5[5], &temp5[6], &temp5[7]);
  transpose_s16_8x8(&temp5[8], &temp5[9], &temp5[10], &temp5[11], &temp5[12],
                    &temp5[13], &temp5[14], &temp5[15]);
  transpose_s16_8x8(&temp5[16], &temp5[17], &temp5[18], &temp5[19], &temp5[20],
                    &temp5[21], &temp5[22], &temp5[23]);
  transpose_s16_8x8(&temp5[24], &temp5[25], &temp5[26], &temp5[27], &temp5[28],
                    &temp5[29], &temp5[30], &temp5[31]);
  store(output + 16 * 32, temp5);

  // Final row of 8x32.
  transpose_s16_8x8_new(&temp1[24], &temp0[0]);
  transpose_s16_8x8_new(&temp2[24], &temp0[8]);
  transpose_s16_8x8_new(&temp3[24], &temp0[16]);
  transpose_s16_8x8_new(&temp4[24], &temp0[24]);

  dct_body_second_pass(temp0, temp5);

  transpose_s16_8x8(&temp5[0], &temp5[1], &temp5[2], &temp5[3], &temp5[4],
                    &temp5[5], &temp5[6], &temp5[7]);
  transpose_s16_8x8(&temp5[8], &temp5[9], &temp5[10], &temp5[11], &temp5[12],
                    &temp5[13], &temp5[14], &temp5[15]);
  transpose_s16_8x8(&temp5[16], &temp5[17], &temp5[18], &temp5[19], &temp5[20],
                    &temp5[21], &temp5[22], &temp5[23]);
  transpose_s16_8x8(&temp5[24], &temp5[25], &temp5[26], &temp5[27], &temp5[28],
                    &temp5[29], &temp5[30], &temp5[31]);
  store(output + 24 * 32, temp5);
}

void vpx_fdct32x32_rd_neon(const int16_t *input, tran_low_t *output,
                           int stride) {
  int16x8_t temp0[32];
  int16x8_t temp1[32];
  int16x8_t temp2[32];
  int16x8_t temp3[32];
  int16x8_t temp4[32];
  int16x8_t temp5[32];

  // Process in 8x32 columns.
  load_cross(input, stride, temp0);
  scale_input(temp0, temp5);
  dct_body_first_pass(temp5, temp1);

  load_cross(input + 8, stride, temp0);
  scale_input(temp0, temp5);
  dct_body_first_pass(temp5, temp2);

  load_cross(input + 16, stride, temp0);
  scale_input(temp0, temp5);
  dct_body_first_pass(temp5, temp3);

  load_cross(input + 24, stride, temp0);
  scale_input(temp0, temp5);
  dct_body_first_pass(temp5, temp4);

  // Generate the top row by munging the first set of 8 from each one together.
  transpose_s16_8x8_new(&temp1[0], &temp0[0]);
  transpose_s16_8x8_new(&temp2[0], &temp0[8]);
  transpose_s16_8x8_new(&temp3[0], &temp0[16]);
  transpose_s16_8x8_new(&temp4[0], &temp0[24]);

  dct_body_second_pass_rd(temp0, temp5);

  transpose_s16_8x8(&temp5[0], &temp5[1], &temp5[2], &temp5[3], &temp5[4],
                    &temp5[5], &temp5[6], &temp5[7]);
  transpose_s16_8x8(&temp5[8], &temp5[9], &temp5[10], &temp5[11], &temp5[12],
                    &temp5[13], &temp5[14], &temp5[15]);
  transpose_s16_8x8(&temp5[16], &temp5[17], &temp5[18], &temp5[19], &temp5[20],
                    &temp5[21], &temp5[22], &temp5[23]);
  transpose_s16_8x8(&temp5[24], &temp5[25], &temp5[26], &temp5[27], &temp5[28],
                    &temp5[29], &temp5[30], &temp5[31]);
  store(output, temp5);

  // Second row of 8x32.
  transpose_s16_8x8_new(&temp1[8], &temp0[0]);
  transpose_s16_8x8_new(&temp2[8], &temp0[8]);
  transpose_s16_8x8_new(&temp3[8], &temp0[16]);
  transpose_s16_8x8_new(&temp4[8], &temp0[24]);

  dct_body_second_pass_rd(temp0, temp5);

  transpose_s16_8x8(&temp5[0], &temp5[1], &temp5[2], &temp5[3], &temp5[4],
                    &temp5[5], &temp5[6], &temp5[7]);
  transpose_s16_8x8(&temp5[8], &temp5[9], &temp5[10], &temp5[11], &temp5[12],
                    &temp5[13], &temp5[14], &temp5[15]);
  transpose_s16_8x8(&temp5[16], &temp5[17], &temp5[18], &temp5[19], &temp5[20],
                    &temp5[21], &temp5[22], &temp5[23]);
  transpose_s16_8x8(&temp5[24], &temp5[25], &temp5[26], &temp5[27], &temp5[28],
                    &temp5[29], &temp5[30], &temp5[31]);
  store(output + 8 * 32, temp5);

  // Third row of 8x32
  transpose_s16_8x8_new(&temp1[16], &temp0[0]);
  transpose_s16_8x8_new(&temp2[16], &temp0[8]);
  transpose_s16_8x8_new(&temp3[16], &temp0[16]);
  transpose_s16_8x8_new(&temp4[16], &temp0[24]);

  dct_body_second_pass_rd(temp0, temp5);

  transpose_s16_8x8(&temp5[0], &temp5[1], &temp5[2], &temp5[3], &temp5[4],
                    &temp5[5], &temp5[6], &temp5[7]);
  transpose_s16_8x8(&temp5[8], &temp5[9], &temp5[10], &temp5[11], &temp5[12],
                    &temp5[13], &temp5[14], &temp5[15]);
  transpose_s16_8x8(&temp5[16], &temp5[17], &temp5[18], &temp5[19], &temp5[20],
                    &temp5[21], &temp5[22], &temp5[23]);
  transpose_s16_8x8(&temp5[24], &temp5[25], &temp5[26], &temp5[27], &temp5[28],
                    &temp5[29], &temp5[30], &temp5[31]);
  store(output + 16 * 32, temp5);

  // Final row of 8x32.
  transpose_s16_8x8_new(&temp1[24], &temp0[0]);
  transpose_s16_8x8_new(&temp2[24], &temp0[8]);
  transpose_s16_8x8_new(&temp3[24], &temp0[16]);
  transpose_s16_8x8_new(&temp4[24], &temp0[24]);

  dct_body_second_pass_rd(temp0, temp5);

  transpose_s16_8x8(&temp5[0], &temp5[1], &temp5[2], &temp5[3], &temp5[4],
                    &temp5[5], &temp5[6], &temp5[7]);
  transpose_s16_8x8(&temp5[8], &temp5[9], &temp5[10], &temp5[11], &temp5[12],
                    &temp5[13], &temp5[14], &temp5[15]);
  transpose_s16_8x8(&temp5[16], &temp5[17], &temp5[18], &temp5[19], &temp5[20],
                    &temp5[21], &temp5[22], &temp5[23]);
  transpose_s16_8x8(&temp5[24], &temp5[25], &temp5[26], &temp5[27], &temp5[28],
                    &temp5[29], &temp5[30], &temp5[31]);
  store(output + 24 * 32, temp5);
}

#if CONFIG_VP9_HIGHBITDEPTH

void vpx_highbd_fdct32x32_neon(const int16_t *input, tran_low_t *output,
                               int stride) {
  int16x8_t temp0[32];
  int32x4_t left1[32], left2[32], left3[32], left4[32], right1[32], right2[32],
      right3[32], right4[32];
  int32x4_t left5[32], right5[32], left6[32], right6[32], left7[32], right7[32],
      left8[32], right8[32];
  int32x4_t temp1[32], temp2[32];

  // Process in 8x32 columns.
  load_cross(input, stride, temp0);
  highbd_scale_input(temp0, left1, right1);
  highbd_dct8x32_body_first_pass(left1, right1);
  highbd_partial_sub_round_shift(left1, right1);

  load_cross(input + 8, stride, temp0);
  highbd_scale_input(temp0, left2, right2);
  highbd_dct8x32_body_first_pass(left2, right2);
  highbd_partial_sub_round_shift(left2, right2);

  load_cross(input + 16, stride, temp0);
  highbd_scale_input(temp0, left3, right3);
  highbd_dct8x32_body_first_pass(left3, right3);
  highbd_partial_sub_round_shift(left3, right3);

  load_cross(input + 24, stride, temp0);
  highbd_scale_input(temp0, left4, right4);
  highbd_dct8x32_body_first_pass(left4, right4);
  highbd_partial_sub_round_shift(left4, right4);

  // Generate the top row by munging the first set of 8 from each one together.
  transpose_s32_8x8_2(left1, right1, temp1, temp2);
  transpose_s32_8x8_2(left2, right2, temp1 + 8, temp2 + 8);
  transpose_s32_8x8_2(left3, right3, temp1 + 16, temp2 + 16);
  transpose_s32_8x8_2(left4, right4, temp1 + 24, temp2 + 24);

  highbd_cross_input(temp1, temp2, left5, right5);
  highbd_dct8x32_body_second_pass(left5, right5);
  highbd_partial_add_round_shift(left5, right5);

  // Second row of 8x32.
  transpose_s32_8x8_2(left1 + 8, right1 + 8, temp1, temp2);
  transpose_s32_8x8_2(left2 + 8, right2 + 8, temp1 + 8, temp2 + 8);
  transpose_s32_8x8_2(left3 + 8, right3 + 8, temp1 + 16, temp2 + 16);
  transpose_s32_8x8_2(left4 + 8, right4 + 8, temp1 + 24, temp2 + 24);

  highbd_cross_input(temp1, temp2, left6, right6);
  highbd_dct8x32_body_second_pass(left6, right6);
  highbd_partial_add_round_shift(left6, right6);

  // Third row of 8x32
  transpose_s32_8x8_2(left1 + 16, right1 + 16, temp1, temp2);
  transpose_s32_8x8_2(left2 + 16, right2 + 16, temp1 + 8, temp2 + 8);
  transpose_s32_8x8_2(left3 + 16, right3 + 16, temp1 + 16, temp2 + 16);
  transpose_s32_8x8_2(left4 + 16, right4 + 16, temp1 + 24, temp2 + 24);

  highbd_cross_input(temp1, temp2, left7, right7);
  highbd_dct8x32_body_second_pass(left7, right7);
  highbd_partial_add_round_shift(left7, right7);

  // Final row of 8x32.
  transpose_s32_8x8_2(left1 + 24, right1 + 24, temp1, temp2);
  transpose_s32_8x8_2(left2 + 24, right2 + 24, temp1 + 8, temp2 + 8);
  transpose_s32_8x8_2(left3 + 24, right3 + 24, temp1 + 16, temp2 + 16);
  transpose_s32_8x8_2(left4 + 24, right4 + 24, temp1 + 24, temp2 + 24);

  highbd_cross_input(temp1, temp2, left8, right8);
  highbd_dct8x32_body_second_pass(left8, right8);
  highbd_partial_add_round_shift(left8, right8);

  // Final transpose
  transpose_s32_8x8_2(left5, right5, left1, right1);
  transpose_s32_8x8_2(left5 + 8, right5 + 8, left2, right2);
  transpose_s32_8x8_2(left5 + 16, right5 + 16, left3, right3);
  transpose_s32_8x8_2(left5 + 24, right5 + 24, left4, right4);
  transpose_s32_8x8_2(left6, right6, left1 + 8, right1 + 8);
  transpose_s32_8x8_2(left6 + 8, right6 + 8, left2 + 8, right2 + 8);
  transpose_s32_8x8_2(left6 + 16, right6 + 16, left3 + 8, right3 + 8);
  transpose_s32_8x8_2(left6 + 24, right6 + 24, left4 + 8, right4 + 8);
  transpose_s32_8x8_2(left7, right7, left1 + 16, right1 + 16);
  transpose_s32_8x8_2(left7 + 8, right7 + 8, left2 + 16, right2 + 16);
  transpose_s32_8x8_2(left7 + 16, right7 + 16, left3 + 16, right3 + 16);
  transpose_s32_8x8_2(left7 + 24, right7 + 24, left4 + 16, right4 + 16);
  transpose_s32_8x8_2(left8, right8, left1 + 24, right1 + 24);
  transpose_s32_8x8_2(left8 + 8, right8 + 8, left2 + 24, right2 + 24);
  transpose_s32_8x8_2(left8 + 16, right8 + 16, left3 + 24, right3 + 24);
  transpose_s32_8x8_2(left8 + 24, right8 + 24, left4 + 24, right4 + 24);

  store32x32_s32(output, left1, right1, left2, right2, left3, right3, left4,
                 right4);
}

void vpx_highbd_fdct32x32_rd_neon(const int16_t *input, tran_low_t *output,
                                  int stride) {
  int16x8_t temp0[32];
  int32x4_t left1[32], left2[32], left3[32], left4[32], right1[32], right2[32],
      right3[32], right4[32];
  int32x4_t left5[32], right5[32], left6[32], right6[32], left7[32], right7[32],
      left8[32], right8[32];
  int32x4_t temp1[32], temp2[32];

  // Process in 8x32 columns.
  load_cross(input, stride, temp0);
  highbd_scale_input(temp0, left1, right1);
  highbd_dct8x32_body_first_pass(left1, right1);
  highbd_partial_sub_round_shift(left1, right1);

  load_cross(input + 8, stride, temp0);
  highbd_scale_input(temp0, left2, right2);
  highbd_dct8x32_body_first_pass(left2, right2);
  highbd_partial_sub_round_shift(left2, right2);

  load_cross(input + 16, stride, temp0);
  highbd_scale_input(temp0, left3, right3);
  highbd_dct8x32_body_first_pass(left3, right3);
  highbd_partial_sub_round_shift(left3, right3);

  load_cross(input + 24, stride, temp0);
  highbd_scale_input(temp0, left4, right4);
  highbd_dct8x32_body_first_pass(left4, right4);
  highbd_partial_sub_round_shift(left4, right4);

  // Generate the top row by munging the first set of 8 from each one together.
  transpose_s32_8x8_2(left1, right1, temp1, temp2);
  transpose_s32_8x8_2(left2, right2, temp1 + 8, temp2 + 8);
  transpose_s32_8x8_2(left3, right3, temp1 + 16, temp2 + 16);
  transpose_s32_8x8_2(left4, right4, temp1 + 24, temp2 + 24);

  highbd_cross_input(temp1, temp2, left5, right5);
  highbd_dct8x32_body_second_pass_rd(left5, right5);

  // Second row of 8x32.
  transpose_s32_8x8_2(left1 + 8, right1 + 8, temp1, temp2);
  transpose_s32_8x8_2(left2 + 8, right2 + 8, temp1 + 8, temp2 + 8);
  transpose_s32_8x8_2(left3 + 8, right3 + 8, temp1 + 16, temp2 + 16);
  transpose_s32_8x8_2(left4 + 8, right4 + 8, temp1 + 24, temp2 + 24);

  highbd_cross_input(temp1, temp2, left6, right6);
  highbd_dct8x32_body_second_pass_rd(left6, right6);

  // Third row of 8x32
  transpose_s32_8x8_2(left1 + 16, right1 + 16, temp1, temp2);
  transpose_s32_8x8_2(left2 + 16, right2 + 16, temp1 + 8, temp2 + 8);
  transpose_s32_8x8_2(left3 + 16, right3 + 16, temp1 + 16, temp2 + 16);
  transpose_s32_8x8_2(left4 + 16, right4 + 16, temp1 + 24, temp2 + 24);

  highbd_cross_input(temp1, temp2, left7, right7);
  highbd_dct8x32_body_second_pass_rd(left7, right7);

  // Final row of 8x32.
  transpose_s32_8x8_2(left1 + 24, right1 + 24, temp1, temp2);
  transpose_s32_8x8_2(left2 + 24, right2 + 24, temp1 + 8, temp2 + 8);
  transpose_s32_8x8_2(left3 + 24, right3 + 24, temp1 + 16, temp2 + 16);
  transpose_s32_8x8_2(left4 + 24, right4 + 24, temp1 + 24, temp2 + 24);

  highbd_cross_input(temp1, temp2, left8, right8);
  highbd_dct8x32_body_second_pass_rd(left8, right8);

  // Final transpose
  transpose_s32_8x8_2(left5, right5, left1, right1);
  transpose_s32_8x8_2(left5 + 8, right5 + 8, left2, right2);
  transpose_s32_8x8_2(left5 + 16, right5 + 16, left3, right3);
  transpose_s32_8x8_2(left5 + 24, right5 + 24, left4, right4);
  transpose_s32_8x8_2(left6, right6, left1 + 8, right1 + 8);
  transpose_s32_8x8_2(left6 + 8, right6 + 8, left2 + 8, right2 + 8);
  transpose_s32_8x8_2(left6 + 16, right6 + 16, left3 + 8, right3 + 8);
  transpose_s32_8x8_2(left6 + 24, right6 + 24, left4 + 8, right4 + 8);
  transpose_s32_8x8_2(left7, right7, left1 + 16, right1 + 16);
  transpose_s32_8x8_2(left7 + 8, right7 + 8, left2 + 16, right2 + 16);
  transpose_s32_8x8_2(left7 + 16, right7 + 16, left3 + 16, right3 + 16);
  transpose_s32_8x8_2(left7 + 24, right7 + 24, left4 + 16, right4 + 16);
  transpose_s32_8x8_2(left8, right8, left1 + 24, right1 + 24);
  transpose_s32_8x8_2(left8 + 8, right8 + 8, left2 + 24, right2 + 24);
  transpose_s32_8x8_2(left8 + 16, right8 + 16, left3 + 24, right3 + 24);
  transpose_s32_8x8_2(left8 + 24, right8 + 24, left4 + 24, right4 + 24);

  store32x32_s32(output, left1, right1, left2, right2, left3, right3, left4,
                 right4);
}

#endif  // CONFIG_VP9_HIGHBITDEPTH

#endif  // !defined(__clang__) && !defined(__ANDROID__) && defined(__GNUC__) &&
        // __GNUC__ == 4 && __GNUC_MINOR__ <= 9