(61d00a474) v0.9.7.1
This commit is contained in:
Regalis
@@ -0,0 +1,240 @@
|
||||
/*
|
||||
* Copyright (c) 2015 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "./vpx_config.h"
|
||||
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/sum_neon.h"
|
||||
|
||||
uint32_t vpx_avg_4x4_neon(const uint8_t *a, int a_stride) {
|
||||
const uint8x16_t b = load_unaligned_u8q(a, a_stride);
|
||||
const uint16x8_t c = vaddl_u8(vget_low_u8(b), vget_high_u8(b));
|
||||
const uint32x2_t d = horizontal_add_uint16x8(c);
|
||||
return vget_lane_u32(vrshr_n_u32(d, 4), 0);
|
||||
}
|
||||
|
||||
uint32_t vpx_avg_8x8_neon(const uint8_t *a, int a_stride) {
|
||||
int i;
|
||||
uint8x8_t b, c;
|
||||
uint16x8_t sum;
|
||||
uint32x2_t d;
|
||||
b = vld1_u8(a);
|
||||
a += a_stride;
|
||||
c = vld1_u8(a);
|
||||
a += a_stride;
|
||||
sum = vaddl_u8(b, c);
|
||||
|
||||
for (i = 0; i < 6; ++i) {
|
||||
const uint8x8_t d = vld1_u8(a);
|
||||
a += a_stride;
|
||||
sum = vaddw_u8(sum, d);
|
||||
}
|
||||
|
||||
d = horizontal_add_uint16x8(sum);
|
||||
|
||||
return vget_lane_u32(vrshr_n_u32(d, 6), 0);
|
||||
}
|
||||
|
||||
// coeff: 16 bits, dynamic range [-32640, 32640].
|
||||
// length: value range {16, 64, 256, 1024}.
|
||||
int vpx_satd_neon(const tran_low_t *coeff, int length) {
|
||||
const int16x4_t zero = vdup_n_s16(0);
|
||||
int32x4_t accum = vdupq_n_s32(0);
|
||||
|
||||
do {
|
||||
const int16x8_t src0 = load_tran_low_to_s16q(coeff);
|
||||
const int16x8_t src8 = load_tran_low_to_s16q(coeff + 8);
|
||||
accum = vabal_s16(accum, vget_low_s16(src0), zero);
|
||||
accum = vabal_s16(accum, vget_high_s16(src0), zero);
|
||||
accum = vabal_s16(accum, vget_low_s16(src8), zero);
|
||||
accum = vabal_s16(accum, vget_high_s16(src8), zero);
|
||||
length -= 16;
|
||||
coeff += 16;
|
||||
} while (length != 0);
|
||||
|
||||
{
|
||||
// satd: 26 bits, dynamic range [-32640 * 1024, 32640 * 1024]
|
||||
const int64x2_t s0 = vpaddlq_s32(accum); // cascading summation of 'accum'.
|
||||
const int32x2_t s1 = vadd_s32(vreinterpret_s32_s64(vget_low_s64(s0)),
|
||||
vreinterpret_s32_s64(vget_high_s64(s0)));
|
||||
const int satd = vget_lane_s32(s1, 0);
|
||||
return satd;
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_int_pro_row_neon(int16_t hbuf[16], uint8_t const *ref,
|
||||
const int ref_stride, const int height) {
|
||||
int i;
|
||||
uint16x8_t vec_sum_lo = vdupq_n_u16(0);
|
||||
uint16x8_t vec_sum_hi = vdupq_n_u16(0);
|
||||
const int shift_factor = ((height >> 5) + 3) * -1;
|
||||
const int16x8_t vec_shift = vdupq_n_s16(shift_factor);
|
||||
|
||||
for (i = 0; i < height; i += 8) {
|
||||
const uint8x16_t vec_row1 = vld1q_u8(ref);
|
||||
const uint8x16_t vec_row2 = vld1q_u8(ref + ref_stride);
|
||||
const uint8x16_t vec_row3 = vld1q_u8(ref + ref_stride * 2);
|
||||
const uint8x16_t vec_row4 = vld1q_u8(ref + ref_stride * 3);
|
||||
const uint8x16_t vec_row5 = vld1q_u8(ref + ref_stride * 4);
|
||||
const uint8x16_t vec_row6 = vld1q_u8(ref + ref_stride * 5);
|
||||
const uint8x16_t vec_row7 = vld1q_u8(ref + ref_stride * 6);
|
||||
const uint8x16_t vec_row8 = vld1q_u8(ref + ref_stride * 7);
|
||||
|
||||
vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row1));
|
||||
vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row1));
|
||||
|
||||
vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row2));
|
||||
vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row2));
|
||||
|
||||
vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row3));
|
||||
vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row3));
|
||||
|
||||
vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row4));
|
||||
vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row4));
|
||||
|
||||
vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row5));
|
||||
vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row5));
|
||||
|
||||
vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row6));
|
||||
vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row6));
|
||||
|
||||
vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row7));
|
||||
vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row7));
|
||||
|
||||
vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row8));
|
||||
vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row8));
|
||||
|
||||
ref += ref_stride * 8;
|
||||
}
|
||||
|
||||
vec_sum_lo = vshlq_u16(vec_sum_lo, vec_shift);
|
||||
vec_sum_hi = vshlq_u16(vec_sum_hi, vec_shift);
|
||||
|
||||
vst1q_s16(hbuf, vreinterpretq_s16_u16(vec_sum_lo));
|
||||
hbuf += 8;
|
||||
vst1q_s16(hbuf, vreinterpretq_s16_u16(vec_sum_hi));
|
||||
}
|
||||
|
||||
int16_t vpx_int_pro_col_neon(uint8_t const *ref, const int width) {
|
||||
int i;
|
||||
uint16x8_t vec_sum = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < width; i += 16) {
|
||||
const uint8x16_t vec_row = vld1q_u8(ref);
|
||||
vec_sum = vaddw_u8(vec_sum, vget_low_u8(vec_row));
|
||||
vec_sum = vaddw_u8(vec_sum, vget_high_u8(vec_row));
|
||||
ref += 16;
|
||||
}
|
||||
|
||||
return vget_lane_s16(vreinterpret_s16_u32(horizontal_add_uint16x8(vec_sum)),
|
||||
0);
|
||||
}
|
||||
|
||||
// ref, src = [0, 510] - max diff = 16-bits
|
||||
// bwl = {2, 3, 4}, width = {16, 32, 64}
|
||||
int vpx_vector_var_neon(int16_t const *ref, int16_t const *src, const int bwl) {
|
||||
int width = 4 << bwl;
|
||||
int32x4_t sse = vdupq_n_s32(0);
|
||||
int16x8_t total = vdupq_n_s16(0);
|
||||
|
||||
assert(width >= 8);
|
||||
assert((width % 8) == 0);
|
||||
|
||||
do {
|
||||
const int16x8_t r = vld1q_s16(ref);
|
||||
const int16x8_t s = vld1q_s16(src);
|
||||
const int16x8_t diff = vsubq_s16(r, s); // [-510, 510], 10 bits.
|
||||
const int16x4_t diff_lo = vget_low_s16(diff);
|
||||
const int16x4_t diff_hi = vget_high_s16(diff);
|
||||
sse = vmlal_s16(sse, diff_lo, diff_lo); // dynamic range 26 bits.
|
||||
sse = vmlal_s16(sse, diff_hi, diff_hi);
|
||||
total = vaddq_s16(total, diff); // dynamic range 16 bits.
|
||||
|
||||
ref += 8;
|
||||
src += 8;
|
||||
width -= 8;
|
||||
} while (width != 0);
|
||||
|
||||
{
|
||||
// Note: 'total''s pairwise addition could be implemented similarly to
|
||||
// horizontal_add_uint16x8(), but one less vpaddl with 'total' when paired
|
||||
// with the summation of 'sse' performed better on a Cortex-A15.
|
||||
const int32x4_t t0 = vpaddlq_s16(total); // cascading summation of 'total'
|
||||
const int32x2_t t1 = vadd_s32(vget_low_s32(t0), vget_high_s32(t0));
|
||||
const int32x2_t t2 = vpadd_s32(t1, t1);
|
||||
const int t = vget_lane_s32(t2, 0);
|
||||
const int64x2_t s0 = vpaddlq_s32(sse); // cascading summation of 'sse'.
|
||||
const int32x2_t s1 = vadd_s32(vreinterpret_s32_s64(vget_low_s64(s0)),
|
||||
vreinterpret_s32_s64(vget_high_s64(s0)));
|
||||
const int s = vget_lane_s32(s1, 0);
|
||||
const int shift_factor = bwl + 2;
|
||||
return s - ((t * t) >> shift_factor);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_minmax_8x8_neon(const uint8_t *a, int a_stride, const uint8_t *b,
|
||||
int b_stride, int *min, int *max) {
|
||||
// Load and concatenate.
|
||||
const uint8x16_t a01 = vcombine_u8(vld1_u8(a), vld1_u8(a + a_stride));
|
||||
const uint8x16_t a23 =
|
||||
vcombine_u8(vld1_u8(a + 2 * a_stride), vld1_u8(a + 3 * a_stride));
|
||||
const uint8x16_t a45 =
|
||||
vcombine_u8(vld1_u8(a + 4 * a_stride), vld1_u8(a + 5 * a_stride));
|
||||
const uint8x16_t a67 =
|
||||
vcombine_u8(vld1_u8(a + 6 * a_stride), vld1_u8(a + 7 * a_stride));
|
||||
|
||||
const uint8x16_t b01 = vcombine_u8(vld1_u8(b), vld1_u8(b + b_stride));
|
||||
const uint8x16_t b23 =
|
||||
vcombine_u8(vld1_u8(b + 2 * b_stride), vld1_u8(b + 3 * b_stride));
|
||||
const uint8x16_t b45 =
|
||||
vcombine_u8(vld1_u8(b + 4 * b_stride), vld1_u8(b + 5 * b_stride));
|
||||
const uint8x16_t b67 =
|
||||
vcombine_u8(vld1_u8(b + 6 * b_stride), vld1_u8(b + 7 * b_stride));
|
||||
|
||||
// Absolute difference.
|
||||
const uint8x16_t ab01_diff = vabdq_u8(a01, b01);
|
||||
const uint8x16_t ab23_diff = vabdq_u8(a23, b23);
|
||||
const uint8x16_t ab45_diff = vabdq_u8(a45, b45);
|
||||
const uint8x16_t ab67_diff = vabdq_u8(a67, b67);
|
||||
|
||||
// Max values between the Q vectors.
|
||||
const uint8x16_t ab0123_max = vmaxq_u8(ab01_diff, ab23_diff);
|
||||
const uint8x16_t ab4567_max = vmaxq_u8(ab45_diff, ab67_diff);
|
||||
const uint8x16_t ab0123_min = vminq_u8(ab01_diff, ab23_diff);
|
||||
const uint8x16_t ab4567_min = vminq_u8(ab45_diff, ab67_diff);
|
||||
|
||||
const uint8x16_t ab07_max = vmaxq_u8(ab0123_max, ab4567_max);
|
||||
const uint8x16_t ab07_min = vminq_u8(ab0123_min, ab4567_min);
|
||||
|
||||
// Split to D and start doing pairwise.
|
||||
uint8x8_t ab_max = vmax_u8(vget_high_u8(ab07_max), vget_low_u8(ab07_max));
|
||||
uint8x8_t ab_min = vmin_u8(vget_high_u8(ab07_min), vget_low_u8(ab07_min));
|
||||
|
||||
// Enough runs of vpmax/min propogate the max/min values to every position.
|
||||
ab_max = vpmax_u8(ab_max, ab_max);
|
||||
ab_min = vpmin_u8(ab_min, ab_min);
|
||||
|
||||
ab_max = vpmax_u8(ab_max, ab_max);
|
||||
ab_min = vpmin_u8(ab_min, ab_min);
|
||||
|
||||
ab_max = vpmax_u8(ab_max, ab_max);
|
||||
ab_min = vpmin_u8(ab_min, ab_min);
|
||||
|
||||
*min = *max = 0; // Clear high bits
|
||||
// Store directly to avoid costly neon->gpr transfer.
|
||||
vst1_lane_u8((uint8_t *)max, ab_max, 0);
|
||||
vst1_lane_u8((uint8_t *)min, ab_min, 0);
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
|
||||
void vpx_comp_avg_pred_neon(uint8_t *comp, const uint8_t *pred, int width,
|
||||
int height, const uint8_t *ref, int ref_stride) {
|
||||
if (width > 8) {
|
||||
int x, y = height;
|
||||
do {
|
||||
for (x = 0; x < width; x += 16) {
|
||||
const uint8x16_t p = vld1q_u8(pred + x);
|
||||
const uint8x16_t r = vld1q_u8(ref + x);
|
||||
const uint8x16_t avg = vrhaddq_u8(p, r);
|
||||
vst1q_u8(comp + x, avg);
|
||||
}
|
||||
comp += width;
|
||||
pred += width;
|
||||
ref += ref_stride;
|
||||
} while (--y);
|
||||
} else if (width == 8) {
|
||||
int i = width * height;
|
||||
do {
|
||||
const uint8x16_t p = vld1q_u8(pred);
|
||||
uint8x16_t r;
|
||||
const uint8x8_t r_0 = vld1_u8(ref);
|
||||
const uint8x8_t r_1 = vld1_u8(ref + ref_stride);
|
||||
r = vcombine_u8(r_0, r_1);
|
||||
ref += 2 * ref_stride;
|
||||
r = vrhaddq_u8(r, p);
|
||||
vst1q_u8(comp, r);
|
||||
|
||||
pred += 16;
|
||||
comp += 16;
|
||||
i -= 16;
|
||||
} while (i);
|
||||
} else {
|
||||
int i = width * height;
|
||||
assert(width == 4);
|
||||
do {
|
||||
const uint8x16_t p = vld1q_u8(pred);
|
||||
uint8x16_t r;
|
||||
|
||||
r = load_unaligned_u8q(ref, ref_stride);
|
||||
ref += 4 * ref_stride;
|
||||
r = vrhaddq_u8(r, p);
|
||||
vst1q_u8(comp, r);
|
||||
|
||||
pred += 16;
|
||||
comp += 16;
|
||||
i -= 16;
|
||||
} while (i);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
|
||||
extern const int16_t vpx_rv[];
|
||||
|
||||
static uint8x8_t average_k_out(const uint8x8_t a2, const uint8x8_t a1,
|
||||
const uint8x8_t v0, const uint8x8_t b1,
|
||||
const uint8x8_t b2) {
|
||||
const uint8x8_t k1 = vrhadd_u8(a2, a1);
|
||||
const uint8x8_t k2 = vrhadd_u8(b2, b1);
|
||||
const uint8x8_t k3 = vrhadd_u8(k1, k2);
|
||||
return vrhadd_u8(k3, v0);
|
||||
}
|
||||
|
||||
static uint8x8_t generate_mask(const uint8x8_t a2, const uint8x8_t a1,
|
||||
const uint8x8_t v0, const uint8x8_t b1,
|
||||
const uint8x8_t b2, const uint8x8_t filter) {
|
||||
const uint8x8_t a2_v0 = vabd_u8(a2, v0);
|
||||
const uint8x8_t a1_v0 = vabd_u8(a1, v0);
|
||||
const uint8x8_t b1_v0 = vabd_u8(b1, v0);
|
||||
const uint8x8_t b2_v0 = vabd_u8(b2, v0);
|
||||
|
||||
uint8x8_t max = vmax_u8(a2_v0, a1_v0);
|
||||
max = vmax_u8(b1_v0, max);
|
||||
max = vmax_u8(b2_v0, max);
|
||||
return vclt_u8(max, filter);
|
||||
}
|
||||
|
||||
static uint8x8_t generate_output(const uint8x8_t a2, const uint8x8_t a1,
|
||||
const uint8x8_t v0, const uint8x8_t b1,
|
||||
const uint8x8_t b2, const uint8x8_t filter) {
|
||||
const uint8x8_t k_out = average_k_out(a2, a1, v0, b1, b2);
|
||||
const uint8x8_t mask = generate_mask(a2, a1, v0, b1, b2, filter);
|
||||
|
||||
return vbsl_u8(mask, k_out, v0);
|
||||
}
|
||||
|
||||
// Same functions but for uint8x16_t.
|
||||
static uint8x16_t average_k_outq(const uint8x16_t a2, const uint8x16_t a1,
|
||||
const uint8x16_t v0, const uint8x16_t b1,
|
||||
const uint8x16_t b2) {
|
||||
const uint8x16_t k1 = vrhaddq_u8(a2, a1);
|
||||
const uint8x16_t k2 = vrhaddq_u8(b2, b1);
|
||||
const uint8x16_t k3 = vrhaddq_u8(k1, k2);
|
||||
return vrhaddq_u8(k3, v0);
|
||||
}
|
||||
|
||||
static uint8x16_t generate_maskq(const uint8x16_t a2, const uint8x16_t a1,
|
||||
const uint8x16_t v0, const uint8x16_t b1,
|
||||
const uint8x16_t b2, const uint8x16_t filter) {
|
||||
const uint8x16_t a2_v0 = vabdq_u8(a2, v0);
|
||||
const uint8x16_t a1_v0 = vabdq_u8(a1, v0);
|
||||
const uint8x16_t b1_v0 = vabdq_u8(b1, v0);
|
||||
const uint8x16_t b2_v0 = vabdq_u8(b2, v0);
|
||||
|
||||
uint8x16_t max = vmaxq_u8(a2_v0, a1_v0);
|
||||
max = vmaxq_u8(b1_v0, max);
|
||||
max = vmaxq_u8(b2_v0, max);
|
||||
return vcltq_u8(max, filter);
|
||||
}
|
||||
|
||||
static uint8x16_t generate_outputq(const uint8x16_t a2, const uint8x16_t a1,
|
||||
const uint8x16_t v0, const uint8x16_t b1,
|
||||
const uint8x16_t b2,
|
||||
const uint8x16_t filter) {
|
||||
const uint8x16_t k_out = average_k_outq(a2, a1, v0, b1, b2);
|
||||
const uint8x16_t mask = generate_maskq(a2, a1, v0, b1, b2, filter);
|
||||
|
||||
return vbslq_u8(mask, k_out, v0);
|
||||
}
|
||||
|
||||
void vpx_post_proc_down_and_across_mb_row_neon(uint8_t *src_ptr,
|
||||
uint8_t *dst_ptr, int src_stride,
|
||||
int dst_stride, int cols,
|
||||
uint8_t *f, int size) {
|
||||
uint8_t *src, *dst;
|
||||
int row;
|
||||
int col;
|
||||
|
||||
// While columns of length 16 can be processed, load them.
|
||||
for (col = 0; col < cols - 8; col += 16) {
|
||||
uint8x16_t a0, a1, a2, a3, a4, a5, a6, a7;
|
||||
src = src_ptr - 2 * src_stride;
|
||||
dst = dst_ptr;
|
||||
|
||||
a0 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
a1 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
a2 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
a3 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
for (row = 0; row < size; row += 4) {
|
||||
uint8x16_t v_out_0, v_out_1, v_out_2, v_out_3;
|
||||
const uint8x16_t filterq = vld1q_u8(f + col);
|
||||
|
||||
a4 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
a5 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
a6 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
a7 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
v_out_0 = generate_outputq(a0, a1, a2, a3, a4, filterq);
|
||||
v_out_1 = generate_outputq(a1, a2, a3, a4, a5, filterq);
|
||||
v_out_2 = generate_outputq(a2, a3, a4, a5, a6, filterq);
|
||||
v_out_3 = generate_outputq(a3, a4, a5, a6, a7, filterq);
|
||||
|
||||
vst1q_u8(dst, v_out_0);
|
||||
dst += dst_stride;
|
||||
vst1q_u8(dst, v_out_1);
|
||||
dst += dst_stride;
|
||||
vst1q_u8(dst, v_out_2);
|
||||
dst += dst_stride;
|
||||
vst1q_u8(dst, v_out_3);
|
||||
dst += dst_stride;
|
||||
|
||||
// Rotate over to the next slot.
|
||||
a0 = a4;
|
||||
a1 = a5;
|
||||
a2 = a6;
|
||||
a3 = a7;
|
||||
}
|
||||
|
||||
src_ptr += 16;
|
||||
dst_ptr += 16;
|
||||
}
|
||||
|
||||
// Clean up any left over column of length 8.
|
||||
if (col != cols) {
|
||||
uint8x8_t a0, a1, a2, a3, a4, a5, a6, a7;
|
||||
src = src_ptr - 2 * src_stride;
|
||||
dst = dst_ptr;
|
||||
|
||||
a0 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
a1 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
a2 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
a3 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
for (row = 0; row < size; row += 4) {
|
||||
uint8x8_t v_out_0, v_out_1, v_out_2, v_out_3;
|
||||
const uint8x8_t filter = vld1_u8(f + col);
|
||||
|
||||
a4 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
a5 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
a6 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
a7 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
v_out_0 = generate_output(a0, a1, a2, a3, a4, filter);
|
||||
v_out_1 = generate_output(a1, a2, a3, a4, a5, filter);
|
||||
v_out_2 = generate_output(a2, a3, a4, a5, a6, filter);
|
||||
v_out_3 = generate_output(a3, a4, a5, a6, a7, filter);
|
||||
|
||||
vst1_u8(dst, v_out_0);
|
||||
dst += dst_stride;
|
||||
vst1_u8(dst, v_out_1);
|
||||
dst += dst_stride;
|
||||
vst1_u8(dst, v_out_2);
|
||||
dst += dst_stride;
|
||||
vst1_u8(dst, v_out_3);
|
||||
dst += dst_stride;
|
||||
|
||||
// Rotate over to the next slot.
|
||||
a0 = a4;
|
||||
a1 = a5;
|
||||
a2 = a6;
|
||||
a3 = a7;
|
||||
}
|
||||
|
||||
// Not strictly necessary but makes resetting dst_ptr easier.
|
||||
dst_ptr += 8;
|
||||
}
|
||||
|
||||
dst_ptr -= cols;
|
||||
|
||||
for (row = 0; row < size; row += 8) {
|
||||
uint8x8_t a0, a1, a2, a3;
|
||||
uint8x8_t b0, b1, b2, b3, b4, b5, b6, b7;
|
||||
|
||||
src = dst_ptr;
|
||||
dst = dst_ptr;
|
||||
|
||||
// Load 8 values, transpose 4 of them, and discard 2 because they will be
|
||||
// reloaded later.
|
||||
load_and_transpose_u8_4x8(src, dst_stride, &a0, &a1, &a2, &a3);
|
||||
a3 = a1;
|
||||
a2 = a1 = a0; // Extend left border.
|
||||
|
||||
src += 2;
|
||||
|
||||
for (col = 0; col < cols; col += 8) {
|
||||
uint8x8_t v_out_0, v_out_1, v_out_2, v_out_3, v_out_4, v_out_5, v_out_6,
|
||||
v_out_7;
|
||||
// Although the filter is meant to be applied vertically and is instead
|
||||
// being applied horizontally here it's OK because it's set in blocks of 8
|
||||
// (or 16).
|
||||
const uint8x8_t filter = vld1_u8(f + col);
|
||||
|
||||
load_and_transpose_u8_8x8(src, dst_stride, &b0, &b1, &b2, &b3, &b4, &b5,
|
||||
&b6, &b7);
|
||||
|
||||
if (col + 8 == cols) {
|
||||
// Last row. Extend border (b5).
|
||||
b6 = b7 = b5;
|
||||
}
|
||||
|
||||
v_out_0 = generate_output(a0, a1, a2, a3, b0, filter);
|
||||
v_out_1 = generate_output(a1, a2, a3, b0, b1, filter);
|
||||
v_out_2 = generate_output(a2, a3, b0, b1, b2, filter);
|
||||
v_out_3 = generate_output(a3, b0, b1, b2, b3, filter);
|
||||
v_out_4 = generate_output(b0, b1, b2, b3, b4, filter);
|
||||
v_out_5 = generate_output(b1, b2, b3, b4, b5, filter);
|
||||
v_out_6 = generate_output(b2, b3, b4, b5, b6, filter);
|
||||
v_out_7 = generate_output(b3, b4, b5, b6, b7, filter);
|
||||
|
||||
transpose_and_store_u8_8x8(dst, dst_stride, v_out_0, v_out_1, v_out_2,
|
||||
v_out_3, v_out_4, v_out_5, v_out_6, v_out_7);
|
||||
|
||||
a0 = b4;
|
||||
a1 = b5;
|
||||
a2 = b6;
|
||||
a3 = b7;
|
||||
|
||||
src += 8;
|
||||
dst += 8;
|
||||
}
|
||||
|
||||
dst_ptr += 8 * dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
// sum += x;
|
||||
// sumsq += x * y;
|
||||
static void accumulate_sum_sumsq(const int16x4_t x, const int32x4_t xy,
|
||||
int16x4_t *const sum, int32x4_t *const sumsq) {
|
||||
const int16x4_t zero = vdup_n_s16(0);
|
||||
const int32x4_t zeroq = vdupq_n_s32(0);
|
||||
|
||||
// Add in the first set because vext doesn't work with '0'.
|
||||
*sum = vadd_s16(*sum, x);
|
||||
*sumsq = vaddq_s32(*sumsq, xy);
|
||||
|
||||
// Shift x and xy to the right and sum. vext requires an immediate.
|
||||
*sum = vadd_s16(*sum, vext_s16(zero, x, 1));
|
||||
*sumsq = vaddq_s32(*sumsq, vextq_s32(zeroq, xy, 1));
|
||||
|
||||
*sum = vadd_s16(*sum, vext_s16(zero, x, 2));
|
||||
*sumsq = vaddq_s32(*sumsq, vextq_s32(zeroq, xy, 2));
|
||||
|
||||
*sum = vadd_s16(*sum, vext_s16(zero, x, 3));
|
||||
*sumsq = vaddq_s32(*sumsq, vextq_s32(zeroq, xy, 3));
|
||||
}
|
||||
|
||||
// Generate mask based on (sumsq * 15 - sum * sum < flimit)
|
||||
static uint16x4_t calculate_mask(const int16x4_t sum, const int32x4_t sumsq,
|
||||
const int32x4_t f, const int32x4_t fifteen) {
|
||||
const int32x4_t a = vmulq_s32(sumsq, fifteen);
|
||||
const int32x4_t b = vmlsl_s16(a, sum, sum);
|
||||
const uint32x4_t mask32 = vcltq_s32(b, f);
|
||||
return vmovn_u32(mask32);
|
||||
}
|
||||
|
||||
static uint8x8_t combine_mask(const int16x4_t sum_low, const int16x4_t sum_high,
|
||||
const int32x4_t sumsq_low,
|
||||
const int32x4_t sumsq_high, const int32x4_t f) {
|
||||
const int32x4_t fifteen = vdupq_n_s32(15);
|
||||
const uint16x4_t mask16_low = calculate_mask(sum_low, sumsq_low, f, fifteen);
|
||||
const uint16x4_t mask16_high =
|
||||
calculate_mask(sum_high, sumsq_high, f, fifteen);
|
||||
return vmovn_u16(vcombine_u16(mask16_low, mask16_high));
|
||||
}
|
||||
|
||||
// Apply filter of (8 + sum + s[c]) >> 4.
|
||||
static uint8x8_t filter_pixels(const int16x8_t sum, const uint8x8_t s) {
|
||||
const int16x8_t s16 = vreinterpretq_s16_u16(vmovl_u8(s));
|
||||
const int16x8_t sum_s = vaddq_s16(sum, s16);
|
||||
|
||||
return vqrshrun_n_s16(sum_s, 4);
|
||||
}
|
||||
|
||||
void vpx_mbpost_proc_across_ip_neon(uint8_t *src, int pitch, int rows, int cols,
|
||||
int flimit) {
|
||||
int row, col;
|
||||
const int32x4_t f = vdupq_n_s32(flimit);
|
||||
|
||||
assert(cols % 8 == 0);
|
||||
|
||||
for (row = 0; row < rows; ++row) {
|
||||
// Sum the first 8 elements, which are extended from s[0].
|
||||
// sumsq gets primed with +16.
|
||||
int sumsq = src[0] * src[0] * 9 + 16;
|
||||
int sum = src[0] * 9;
|
||||
|
||||
uint8x8_t left_context, s, right_context;
|
||||
int16x4_t sum_low, sum_high;
|
||||
int32x4_t sumsq_low, sumsq_high;
|
||||
|
||||
// Sum (+square) the next 6 elements.
|
||||
// Skip [0] because it's included above.
|
||||
for (col = 1; col <= 6; ++col) {
|
||||
sumsq += src[col] * src[col];
|
||||
sum += src[col];
|
||||
}
|
||||
|
||||
// Prime the sums. Later the loop uses the _high values to prime the new
|
||||
// vectors.
|
||||
sumsq_high = vdupq_n_s32(sumsq);
|
||||
sum_high = vdup_n_s16(sum);
|
||||
|
||||
// Manually extend the left border.
|
||||
left_context = vdup_n_u8(src[0]);
|
||||
|
||||
for (col = 0; col < cols; col += 8) {
|
||||
uint8x8_t mask, output;
|
||||
int16x8_t x, y;
|
||||
int32x4_t xy_low, xy_high;
|
||||
|
||||
s = vld1_u8(src + col);
|
||||
|
||||
if (col + 8 == cols) {
|
||||
// Last row. Extend border.
|
||||
right_context = vdup_n_u8(src[col + 7]);
|
||||
} else {
|
||||
right_context = vld1_u8(src + col + 7);
|
||||
}
|
||||
|
||||
x = vreinterpretq_s16_u16(vsubl_u8(right_context, left_context));
|
||||
y = vreinterpretq_s16_u16(vaddl_u8(right_context, left_context));
|
||||
xy_low = vmull_s16(vget_low_s16(x), vget_low_s16(y));
|
||||
xy_high = vmull_s16(vget_high_s16(x), vget_high_s16(y));
|
||||
|
||||
// Catch up to the last sum'd value.
|
||||
sum_low = vdup_lane_s16(sum_high, 3);
|
||||
sumsq_low = vdupq_lane_s32(vget_high_s32(sumsq_high), 1);
|
||||
|
||||
accumulate_sum_sumsq(vget_low_s16(x), xy_low, &sum_low, &sumsq_low);
|
||||
|
||||
// Need to do this sequentially because we need the max value from
|
||||
// sum_low.
|
||||
sum_high = vdup_lane_s16(sum_low, 3);
|
||||
sumsq_high = vdupq_lane_s32(vget_high_s32(sumsq_low), 1);
|
||||
|
||||
accumulate_sum_sumsq(vget_high_s16(x), xy_high, &sum_high, &sumsq_high);
|
||||
|
||||
mask = combine_mask(sum_low, sum_high, sumsq_low, sumsq_high, f);
|
||||
|
||||
output = filter_pixels(vcombine_s16(sum_low, sum_high), s);
|
||||
output = vbsl_u8(mask, output, s);
|
||||
|
||||
vst1_u8(src + col, output);
|
||||
|
||||
left_context = s;
|
||||
}
|
||||
|
||||
src += pitch;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filter of (vpx_rv + sum + s[c]) >> 4.
|
||||
static uint8x8_t filter_pixels_rv(const int16x8_t sum, const uint8x8_t s,
|
||||
const int16x8_t rv) {
|
||||
const int16x8_t s16 = vreinterpretq_s16_u16(vmovl_u8(s));
|
||||
const int16x8_t sum_s = vaddq_s16(sum, s16);
|
||||
const int16x8_t rounded = vaddq_s16(sum_s, rv);
|
||||
|
||||
return vqshrun_n_s16(rounded, 4);
|
||||
}
|
||||
|
||||
void vpx_mbpost_proc_down_neon(uint8_t *dst, int pitch, int rows, int cols,
|
||||
int flimit) {
|
||||
int row, col, i;
|
||||
const int32x4_t f = vdupq_n_s32(flimit);
|
||||
uint8x8_t below_context = vdup_n_u8(0);
|
||||
|
||||
// 8 columns are processed at a time.
|
||||
// If rows is less than 8 the bottom border extension fails.
|
||||
assert(cols % 8 == 0);
|
||||
assert(rows >= 8);
|
||||
|
||||
// Load and keep the first 8 values in memory. Process a vertical stripe that
|
||||
// is 8 wide.
|
||||
for (col = 0; col < cols; col += 8) {
|
||||
uint8x8_t s, above_context[8];
|
||||
int16x8_t sum, sum_tmp;
|
||||
int32x4_t sumsq_low, sumsq_high;
|
||||
|
||||
// Load and extend the top border.
|
||||
s = vld1_u8(dst);
|
||||
for (i = 0; i < 8; i++) {
|
||||
above_context[i] = s;
|
||||
}
|
||||
|
||||
sum_tmp = vreinterpretq_s16_u16(vmovl_u8(s));
|
||||
|
||||
// sum * 9
|
||||
sum = vmulq_n_s16(sum_tmp, 9);
|
||||
|
||||
// (sum * 9) * sum == sum * sum * 9
|
||||
sumsq_low = vmull_s16(vget_low_s16(sum), vget_low_s16(sum_tmp));
|
||||
sumsq_high = vmull_s16(vget_high_s16(sum), vget_high_s16(sum_tmp));
|
||||
|
||||
// Load and discard the next 6 values to prime sum and sumsq.
|
||||
for (i = 1; i <= 6; ++i) {
|
||||
const uint8x8_t a = vld1_u8(dst + i * pitch);
|
||||
const int16x8_t b = vreinterpretq_s16_u16(vmovl_u8(a));
|
||||
sum = vaddq_s16(sum, b);
|
||||
|
||||
sumsq_low = vmlal_s16(sumsq_low, vget_low_s16(b), vget_low_s16(b));
|
||||
sumsq_high = vmlal_s16(sumsq_high, vget_high_s16(b), vget_high_s16(b));
|
||||
}
|
||||
|
||||
for (row = 0; row < rows; ++row) {
|
||||
uint8x8_t mask, output;
|
||||
int16x8_t x, y;
|
||||
int32x4_t xy_low, xy_high;
|
||||
|
||||
s = vld1_u8(dst + row * pitch);
|
||||
|
||||
// Extend the bottom border.
|
||||
if (row + 7 < rows) {
|
||||
below_context = vld1_u8(dst + (row + 7) * pitch);
|
||||
}
|
||||
|
||||
x = vreinterpretq_s16_u16(vsubl_u8(below_context, above_context[0]));
|
||||
y = vreinterpretq_s16_u16(vaddl_u8(below_context, above_context[0]));
|
||||
xy_low = vmull_s16(vget_low_s16(x), vget_low_s16(y));
|
||||
xy_high = vmull_s16(vget_high_s16(x), vget_high_s16(y));
|
||||
|
||||
sum = vaddq_s16(sum, x);
|
||||
|
||||
sumsq_low = vaddq_s32(sumsq_low, xy_low);
|
||||
sumsq_high = vaddq_s32(sumsq_high, xy_high);
|
||||
|
||||
mask = combine_mask(vget_low_s16(sum), vget_high_s16(sum), sumsq_low,
|
||||
sumsq_high, f);
|
||||
|
||||
output = filter_pixels_rv(sum, s, vld1q_s16(vpx_rv + (row & 127)));
|
||||
output = vbsl_u8(mask, output, s);
|
||||
|
||||
vst1_u8(dst + row * pitch, output);
|
||||
|
||||
above_context[0] = above_context[1];
|
||||
above_context[1] = above_context[2];
|
||||
above_context[2] = above_context[3];
|
||||
above_context[3] = above_context[4];
|
||||
above_context[4] = above_context[5];
|
||||
above_context[5] = above_context[6];
|
||||
above_context[6] = above_context[7];
|
||||
above_context[7] = s;
|
||||
}
|
||||
|
||||
dst += 8;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
|
||||
// Some builds of gcc 4.9.2 and .3 have trouble with some of the inline
|
||||
// functions.
|
||||
#if !defined(__clang__) && !defined(__ANDROID__) && defined(__GNUC__) && \
|
||||
__GNUC__ == 4 && __GNUC_MINOR__ == 9 && __GNUC_PATCHLEVEL__ < 4
|
||||
|
||||
void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
vpx_fdct16x16_c(input, output, stride);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) {
|
||||
b[0] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[1] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[2] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[3] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[4] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[5] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[6] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[7] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[8] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[9] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[10] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[11] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[12] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[13] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[14] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[15] = vld1q_s16(a);
|
||||
}
|
||||
|
||||
// Store 8 16x8 values, assuming stride == 16.
|
||||
static INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) {
|
||||
store_s16q_to_tran_low(a, b[0]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[1]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[2]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[3]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[4]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[5]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[6]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[7]);
|
||||
}
|
||||
|
||||
// Load step of each pass. Add and subtract clear across the input, requiring
|
||||
// all 16 values to be loaded. For the first pass it also multiplies by 4.
|
||||
|
||||
// To maybe reduce register usage this could be combined with the load() step to
|
||||
// get the first 4 and last 4 values, cross those, then load the middle 8 values
|
||||
// and cross them.
|
||||
static INLINE void cross_input(const int16x8_t *a /*[16]*/,
|
||||
int16x8_t *b /*[16]*/, const int pass) {
|
||||
if (pass == 0) {
|
||||
b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2);
|
||||
b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2);
|
||||
b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2);
|
||||
b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2);
|
||||
b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2);
|
||||
b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2);
|
||||
b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2);
|
||||
b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2);
|
||||
|
||||
b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2);
|
||||
b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2);
|
||||
b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2);
|
||||
b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2);
|
||||
b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2);
|
||||
b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2);
|
||||
b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2);
|
||||
b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2);
|
||||
} else {
|
||||
b[0] = vaddq_s16(a[0], a[15]);
|
||||
b[1] = vaddq_s16(a[1], a[14]);
|
||||
b[2] = vaddq_s16(a[2], a[13]);
|
||||
b[3] = vaddq_s16(a[3], a[12]);
|
||||
b[4] = vaddq_s16(a[4], a[11]);
|
||||
b[5] = vaddq_s16(a[5], a[10]);
|
||||
b[6] = vaddq_s16(a[6], a[9]);
|
||||
b[7] = vaddq_s16(a[7], a[8]);
|
||||
|
||||
b[8] = vsubq_s16(a[7], a[8]);
|
||||
b[9] = vsubq_s16(a[6], a[9]);
|
||||
b[10] = vsubq_s16(a[5], a[10]);
|
||||
b[11] = vsubq_s16(a[4], a[11]);
|
||||
b[12] = vsubq_s16(a[3], a[12]);
|
||||
b[13] = vsubq_s16(a[2], a[13]);
|
||||
b[14] = vsubq_s16(a[1], a[14]);
|
||||
b[15] = vsubq_s16(a[0], a[15]);
|
||||
}
|
||||
}
|
||||
|
||||
// Quarter round at the beginning of the second pass. Can't use vrshr (rounding)
|
||||
// because this only adds 1, not 1 << 2.
|
||||
static INLINE void partial_round_shift(int16x8_t *a /*[16]*/) {
|
||||
const int16x8_t one = vdupq_n_s16(1);
|
||||
a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2);
|
||||
a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2);
|
||||
a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2);
|
||||
a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2);
|
||||
a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2);
|
||||
a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2);
|
||||
a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2);
|
||||
a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2);
|
||||
a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2);
|
||||
a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2);
|
||||
a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2);
|
||||
a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2);
|
||||
a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2);
|
||||
a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2);
|
||||
a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2);
|
||||
a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2);
|
||||
}
|
||||
|
||||
// fdct_round_shift((a +/- b) * c)
|
||||
static INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b,
|
||||
const tran_high_t c, int16x8_t *add,
|
||||
int16x8_t *sub) {
|
||||
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c);
|
||||
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c);
|
||||
const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c);
|
||||
const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c);
|
||||
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c);
|
||||
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c);
|
||||
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
|
||||
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
|
||||
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
|
||||
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
|
||||
*add = vcombine_s16(rounded0, rounded1);
|
||||
*sub = vcombine_s16(rounded2, rounded3);
|
||||
}
|
||||
|
||||
// fdct_round_shift(a * c0 +/- b * c1)
|
||||
static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b,
|
||||
const tran_coef_t c0,
|
||||
const tran_coef_t c1, int16x8_t *add,
|
||||
int16x8_t *sub) {
|
||||
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0);
|
||||
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0);
|
||||
const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1);
|
||||
const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1);
|
||||
const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0);
|
||||
const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0);
|
||||
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1);
|
||||
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1);
|
||||
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
|
||||
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
|
||||
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
|
||||
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
|
||||
*add = vcombine_s16(rounded0, rounded1);
|
||||
*sub = vcombine_s16(rounded2, rounded3);
|
||||
}
|
||||
|
||||
// Transpose 8x8 to a new location. Don't use transpose_neon.h because those
|
||||
// are all in-place.
|
||||
static INLINE void transpose_8x8(const int16x8_t *a /*[8]*/,
|
||||
int16x8_t *b /*[8]*/) {
|
||||
// Swap 16 bit elements.
|
||||
const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]);
|
||||
const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]);
|
||||
const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]);
|
||||
const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]);
|
||||
|
||||
// Swap 32 bit elements.
|
||||
const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]),
|
||||
vreinterpretq_s32_s16(c1.val[0]));
|
||||
const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]),
|
||||
vreinterpretq_s32_s16(c1.val[1]));
|
||||
const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]),
|
||||
vreinterpretq_s32_s16(c3.val[0]));
|
||||
const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]),
|
||||
vreinterpretq_s32_s16(c3.val[1]));
|
||||
|
||||
// Swap 64 bit elements
|
||||
const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]);
|
||||
const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]);
|
||||
const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]);
|
||||
const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]);
|
||||
|
||||
b[0] = e0.val[0];
|
||||
b[1] = e1.val[0];
|
||||
b[2] = e2.val[0];
|
||||
b[3] = e3.val[0];
|
||||
b[4] = e0.val[1];
|
||||
b[5] = e1.val[1];
|
||||
b[6] = e2.val[1];
|
||||
b[7] = e3.val[1];
|
||||
}
|
||||
|
||||
// Main body of fdct16x16.
|
||||
static void dct_body(const int16x8_t *in /*[16]*/, int16x8_t *out /*[16]*/) {
|
||||
int16x8_t s[8];
|
||||
int16x8_t x[4];
|
||||
int16x8_t step[8];
|
||||
|
||||
// stage 1
|
||||
// From fwd_txfm.c: Work on the first eight values; fdct8(input,
|
||||
// even_results);"
|
||||
s[0] = vaddq_s16(in[0], in[7]);
|
||||
s[1] = vaddq_s16(in[1], in[6]);
|
||||
s[2] = vaddq_s16(in[2], in[5]);
|
||||
s[3] = vaddq_s16(in[3], in[4]);
|
||||
s[4] = vsubq_s16(in[3], in[4]);
|
||||
s[5] = vsubq_s16(in[2], in[5]);
|
||||
s[6] = vsubq_s16(in[1], in[6]);
|
||||
s[7] = vsubq_s16(in[0], in[7]);
|
||||
|
||||
// fdct4(step, step);
|
||||
x[0] = vaddq_s16(s[0], s[3]);
|
||||
x[1] = vaddq_s16(s[1], s[2]);
|
||||
x[2] = vsubq_s16(s[1], s[2]);
|
||||
x[3] = vsubq_s16(s[0], s[3]);
|
||||
|
||||
// out[0] = fdct_round_shift((x0 + x1) * cospi_16_64)
|
||||
// out[8] = fdct_round_shift((x0 - x1) * cospi_16_64)
|
||||
butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]);
|
||||
// out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64);
|
||||
// out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64);
|
||||
butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]);
|
||||
|
||||
// Stage 2
|
||||
// Re-using source s5/s6
|
||||
// s5 = fdct_round_shift((s6 - s5) * cospi_16_64)
|
||||
// s6 = fdct_round_shift((s6 + s5) * cospi_16_64)
|
||||
butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]);
|
||||
|
||||
// Stage 3
|
||||
x[0] = vaddq_s16(s[4], s[5]);
|
||||
x[1] = vsubq_s16(s[4], s[5]);
|
||||
x[2] = vsubq_s16(s[7], s[6]);
|
||||
x[3] = vaddq_s16(s[7], s[6]);
|
||||
|
||||
// Stage 4
|
||||
// out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64)
|
||||
// out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64)
|
||||
butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]);
|
||||
// out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64)
|
||||
// out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64)
|
||||
butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]);
|
||||
|
||||
// step 2
|
||||
// From fwd_txfm.c: Work on the next eight values; step1 -> odd_results"
|
||||
// That file distinguished between "in_high" and "step1" but the only
|
||||
// difference is that "in_high" is the first 8 values and "step 1" is the
|
||||
// second. Here, since they are all in one array, "step1" values are += 8.
|
||||
|
||||
// step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64)
|
||||
// step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64)
|
||||
// step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64)
|
||||
// step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64)
|
||||
butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]);
|
||||
butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]);
|
||||
|
||||
// step 3
|
||||
s[0] = vaddq_s16(in[8], s[3]);
|
||||
s[1] = vaddq_s16(in[9], s[2]);
|
||||
x[0] = vsubq_s16(in[9], s[2]);
|
||||
x[1] = vsubq_s16(in[8], s[3]);
|
||||
x[2] = vsubq_s16(in[15], s[4]);
|
||||
x[3] = vsubq_s16(in[14], s[5]);
|
||||
s[6] = vaddq_s16(in[14], s[5]);
|
||||
s[7] = vaddq_s16(in[15], s[4]);
|
||||
|
||||
// step 4
|
||||
// step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64)
|
||||
// step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64)
|
||||
butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]);
|
||||
|
||||
// step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64)
|
||||
// step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64)
|
||||
butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]);
|
||||
|
||||
// step 5
|
||||
step[0] = vaddq_s16(s[0], s[1]);
|
||||
step[1] = vsubq_s16(s[0], s[1]);
|
||||
step[2] = vaddq_s16(x[1], s[2]);
|
||||
step[3] = vsubq_s16(x[1], s[2]);
|
||||
step[4] = vsubq_s16(x[2], s[5]);
|
||||
step[5] = vaddq_s16(x[2], s[5]);
|
||||
step[6] = vsubq_s16(s[7], s[6]);
|
||||
step[7] = vaddq_s16(s[7], s[6]);
|
||||
|
||||
// step 6
|
||||
// out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64)
|
||||
// out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64)
|
||||
// out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64)
|
||||
// out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64)
|
||||
// out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64)
|
||||
// out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] *
|
||||
// cospi_22_64)
|
||||
// out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64)
|
||||
// out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64)
|
||||
butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9],
|
||||
&out[7]);
|
||||
butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1],
|
||||
&out[15]);
|
||||
butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13],
|
||||
&out[3]);
|
||||
butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5],
|
||||
&out[11]);
|
||||
}
|
||||
|
||||
void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
int16x8_t temp0[16];
|
||||
int16x8_t temp1[16];
|
||||
int16x8_t temp2[16];
|
||||
int16x8_t temp3[16];
|
||||
|
||||
// Left half.
|
||||
load(input, stride, temp0);
|
||||
cross_input(temp0, temp1, 0);
|
||||
dct_body(temp1, temp0);
|
||||
|
||||
// Right half.
|
||||
load(input + 8, stride, temp1);
|
||||
cross_input(temp1, temp2, 0);
|
||||
dct_body(temp2, temp1);
|
||||
|
||||
// Transpose top left and top right quarters into one contiguous location to
|
||||
// process to the top half.
|
||||
transpose_8x8(&temp0[0], &temp2[0]);
|
||||
transpose_8x8(&temp1[0], &temp2[8]);
|
||||
partial_round_shift(temp2);
|
||||
cross_input(temp2, temp3, 1);
|
||||
dct_body(temp3, temp2);
|
||||
transpose_s16_8x8(&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4],
|
||||
&temp2[5], &temp2[6], &temp2[7]);
|
||||
transpose_s16_8x8(&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12],
|
||||
&temp2[13], &temp2[14], &temp2[15]);
|
||||
store(output, temp2);
|
||||
store(output + 8, temp2 + 8);
|
||||
output += 8 * 16;
|
||||
|
||||
// Transpose bottom left and bottom right quarters into one contiguous
|
||||
// location to process to the bottom half.
|
||||
transpose_8x8(&temp0[8], &temp1[0]);
|
||||
transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12],
|
||||
&temp1[13], &temp1[14], &temp1[15]);
|
||||
partial_round_shift(temp1);
|
||||
cross_input(temp1, temp0, 1);
|
||||
dct_body(temp0, temp1);
|
||||
transpose_s16_8x8(&temp1[0], &temp1[1], &temp1[2], &temp1[3], &temp1[4],
|
||||
&temp1[5], &temp1[6], &temp1[7]);
|
||||
transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12],
|
||||
&temp1[13], &temp1[14], &temp1[15]);
|
||||
store(output, temp1);
|
||||
store(output + 8, temp1 + 8);
|
||||
}
|
||||
#endif // !defined(__clang__) && !defined(__ANDROID__) && defined(__GNUC__) &&
|
||||
// __GNUC__ == 4 && __GNUC_MINOR__ == 9 && __GNUC_PATCHLEVEL__ < 4
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
|
||||
void vpx_fdct4x4_neon(const int16_t *input, tran_low_t *final_output,
|
||||
int stride) {
|
||||
int i;
|
||||
// input[M * stride] * 16
|
||||
int16x4_t input_0 = vshl_n_s16(vld1_s16(input + 0 * stride), 4);
|
||||
int16x4_t input_1 = vshl_n_s16(vld1_s16(input + 1 * stride), 4);
|
||||
int16x4_t input_2 = vshl_n_s16(vld1_s16(input + 2 * stride), 4);
|
||||
int16x4_t input_3 = vshl_n_s16(vld1_s16(input + 3 * stride), 4);
|
||||
|
||||
// If the very first value != 0, then add 1.
|
||||
if (input[0] != 0) {
|
||||
const int16x4_t one = vreinterpret_s16_s64(vdup_n_s64(1));
|
||||
input_0 = vadd_s16(input_0, one);
|
||||
}
|
||||
|
||||
for (i = 0; i < 2; ++i) {
|
||||
const int16x8_t input_01 = vcombine_s16(input_0, input_1);
|
||||
const int16x8_t input_32 = vcombine_s16(input_3, input_2);
|
||||
|
||||
// in_0 +/- in_3, in_1 +/- in_2
|
||||
const int16x8_t s_01 = vaddq_s16(input_01, input_32);
|
||||
const int16x8_t s_32 = vsubq_s16(input_01, input_32);
|
||||
|
||||
// step_0 +/- step_1, step_2 +/- step_3
|
||||
const int16x4_t s_0 = vget_low_s16(s_01);
|
||||
const int16x4_t s_1 = vget_high_s16(s_01);
|
||||
const int16x4_t s_2 = vget_high_s16(s_32);
|
||||
const int16x4_t s_3 = vget_low_s16(s_32);
|
||||
|
||||
// (s_0 +/- s_1) * cospi_16_64
|
||||
// Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c.
|
||||
const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1);
|
||||
const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1);
|
||||
const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64);
|
||||
const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64);
|
||||
|
||||
// fdct_round_shift
|
||||
int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS);
|
||||
int16x4_t out_2 = vrshrn_n_s32(temp2, DCT_CONST_BITS);
|
||||
|
||||
// s_3 * cospi_8_64 + s_2 * cospi_24_64
|
||||
// s_3 * cospi_24_64 - s_2 * cospi_8_64
|
||||
const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64);
|
||||
const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64);
|
||||
|
||||
const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64);
|
||||
const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64);
|
||||
|
||||
// fdct_round_shift
|
||||
int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS);
|
||||
int16x4_t out_3 = vrshrn_n_s32(temp4, DCT_CONST_BITS);
|
||||
|
||||
transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3);
|
||||
|
||||
input_0 = out_0;
|
||||
input_1 = out_1;
|
||||
input_2 = out_2;
|
||||
input_3 = out_3;
|
||||
}
|
||||
|
||||
{
|
||||
// Not quite a rounding shift. Only add 1 despite shifting by 2.
|
||||
const int16x8_t one = vdupq_n_s16(1);
|
||||
int16x8_t out_01 = vcombine_s16(input_0, input_1);
|
||||
int16x8_t out_23 = vcombine_s16(input_2, input_3);
|
||||
out_01 = vshrq_n_s16(vaddq_s16(out_01, one), 2);
|
||||
out_23 = vshrq_n_s16(vaddq_s16(out_23, one), 2);
|
||||
store_s16q_to_tran_low(final_output + 0 * 8, out_01);
|
||||
store_s16q_to_tran_low(final_output + 1 * 8, out_23);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "./vpx_config.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/sum_neon.h"
|
||||
|
||||
static INLINE tran_low_t get_lane(const int32x2_t a) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
return vget_lane_s32(a, 0);
|
||||
#else
|
||||
return vget_lane_s16(vreinterpret_s16_s32(a), 0);
|
||||
#endif // CONFIG_VP9_HIGHBITDETPH
|
||||
}
|
||||
|
||||
void vpx_fdct4x4_1_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
int16x4_t a0, a1, a2, a3;
|
||||
int16x8_t b0, b1;
|
||||
int16x8_t c;
|
||||
int32x2_t d;
|
||||
|
||||
a0 = vld1_s16(input);
|
||||
input += stride;
|
||||
a1 = vld1_s16(input);
|
||||
input += stride;
|
||||
a2 = vld1_s16(input);
|
||||
input += stride;
|
||||
a3 = vld1_s16(input);
|
||||
|
||||
b0 = vcombine_s16(a0, a1);
|
||||
b1 = vcombine_s16(a2, a3);
|
||||
|
||||
c = vaddq_s16(b0, b1);
|
||||
|
||||
d = horizontal_add_int16x8(c);
|
||||
|
||||
output[0] = get_lane(vshl_n_s32(d, 1));
|
||||
output[1] = 0;
|
||||
}
|
||||
|
||||
void vpx_fdct8x8_1_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
int r;
|
||||
int16x8_t sum = vld1q_s16(&input[0]);
|
||||
|
||||
for (r = 1; r < 8; ++r) {
|
||||
const int16x8_t input_00 = vld1q_s16(&input[r * stride]);
|
||||
sum = vaddq_s16(sum, input_00);
|
||||
}
|
||||
|
||||
output[0] = get_lane(horizontal_add_int16x8(sum));
|
||||
output[1] = 0;
|
||||
}
|
||||
|
||||
void vpx_fdct16x16_1_neon(const int16_t *input, tran_low_t *output,
|
||||
int stride) {
|
||||
int r;
|
||||
int16x8_t left = vld1q_s16(input);
|
||||
int16x8_t right = vld1q_s16(input + 8);
|
||||
int32x2_t sum;
|
||||
input += stride;
|
||||
|
||||
for (r = 1; r < 16; ++r) {
|
||||
const int16x8_t a = vld1q_s16(input);
|
||||
const int16x8_t b = vld1q_s16(input + 8);
|
||||
input += stride;
|
||||
left = vaddq_s16(left, a);
|
||||
right = vaddq_s16(right, b);
|
||||
}
|
||||
|
||||
sum = vadd_s32(horizontal_add_int16x8(left), horizontal_add_int16x8(right));
|
||||
|
||||
output[0] = get_lane(vshr_n_s32(sum, 1));
|
||||
output[1] = 0;
|
||||
}
|
||||
|
||||
void vpx_fdct32x32_1_neon(const int16_t *input, tran_low_t *output,
|
||||
int stride) {
|
||||
int r;
|
||||
int16x8_t a0 = vld1q_s16(input);
|
||||
int16x8_t a1 = vld1q_s16(input + 8);
|
||||
int16x8_t a2 = vld1q_s16(input + 16);
|
||||
int16x8_t a3 = vld1q_s16(input + 24);
|
||||
int32x2_t sum;
|
||||
input += stride;
|
||||
|
||||
for (r = 1; r < 32; ++r) {
|
||||
const int16x8_t b0 = vld1q_s16(input);
|
||||
const int16x8_t b1 = vld1q_s16(input + 8);
|
||||
const int16x8_t b2 = vld1q_s16(input + 16);
|
||||
const int16x8_t b3 = vld1q_s16(input + 24);
|
||||
input += stride;
|
||||
a0 = vaddq_s16(a0, b0);
|
||||
a1 = vaddq_s16(a1, b1);
|
||||
a2 = vaddq_s16(a2, b2);
|
||||
a3 = vaddq_s16(a3, b3);
|
||||
}
|
||||
|
||||
sum = vadd_s32(horizontal_add_int16x8(a0), horizontal_add_int16x8(a1));
|
||||
sum = vadd_s32(sum, horizontal_add_int16x8(a2));
|
||||
sum = vadd_s32(sum, horizontal_add_int16x8(a3));
|
||||
output[0] = get_lane(vshr_n_s32(sum, 3));
|
||||
output[1] = 0;
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
* Copyright (c) 2015 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
|
||||
void vpx_fdct8x8_neon(const int16_t *input, tran_low_t *final_output,
|
||||
int stride) {
|
||||
int i;
|
||||
// stage 1
|
||||
int16x8_t input_0 = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2);
|
||||
int16x8_t input_1 = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2);
|
||||
int16x8_t input_2 = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2);
|
||||
int16x8_t input_3 = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2);
|
||||
int16x8_t input_4 = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2);
|
||||
int16x8_t input_5 = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2);
|
||||
int16x8_t input_6 = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2);
|
||||
int16x8_t input_7 = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2);
|
||||
for (i = 0; i < 2; ++i) {
|
||||
int16x8_t out_0, out_1, out_2, out_3, out_4, out_5, out_6, out_7;
|
||||
const int16x8_t v_s0 = vaddq_s16(input_0, input_7);
|
||||
const int16x8_t v_s1 = vaddq_s16(input_1, input_6);
|
||||
const int16x8_t v_s2 = vaddq_s16(input_2, input_5);
|
||||
const int16x8_t v_s3 = vaddq_s16(input_3, input_4);
|
||||
const int16x8_t v_s4 = vsubq_s16(input_3, input_4);
|
||||
const int16x8_t v_s5 = vsubq_s16(input_2, input_5);
|
||||
const int16x8_t v_s6 = vsubq_s16(input_1, input_6);
|
||||
const int16x8_t v_s7 = vsubq_s16(input_0, input_7);
|
||||
// fdct4(step, step);
|
||||
int16x8_t v_x0 = vaddq_s16(v_s0, v_s3);
|
||||
int16x8_t v_x1 = vaddq_s16(v_s1, v_s2);
|
||||
int16x8_t v_x2 = vsubq_s16(v_s1, v_s2);
|
||||
int16x8_t v_x3 = vsubq_s16(v_s0, v_s3);
|
||||
// fdct4(step, step);
|
||||
int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
|
||||
int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
|
||||
int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
|
||||
int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
|
||||
int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_24_64);
|
||||
int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_24_64);
|
||||
int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_24_64);
|
||||
int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_24_64);
|
||||
v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), cospi_8_64);
|
||||
v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), cospi_8_64);
|
||||
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), cospi_8_64);
|
||||
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), cospi_8_64);
|
||||
v_t0_lo = vmulq_n_s32(v_t0_lo, cospi_16_64);
|
||||
v_t0_hi = vmulq_n_s32(v_t0_hi, cospi_16_64);
|
||||
v_t1_lo = vmulq_n_s32(v_t1_lo, cospi_16_64);
|
||||
v_t1_hi = vmulq_n_s32(v_t1_hi, cospi_16_64);
|
||||
{
|
||||
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
|
||||
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
|
||||
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
|
||||
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
|
||||
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
|
||||
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
|
||||
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
|
||||
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
|
||||
out_0 = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43
|
||||
out_2 = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63
|
||||
out_4 = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47
|
||||
out_6 = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67
|
||||
}
|
||||
// Stage 2
|
||||
v_x0 = vsubq_s16(v_s6, v_s5);
|
||||
v_x1 = vaddq_s16(v_s6, v_s5);
|
||||
v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), cospi_16_64);
|
||||
v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), cospi_16_64);
|
||||
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_16_64);
|
||||
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_16_64);
|
||||
{
|
||||
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
|
||||
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
|
||||
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
|
||||
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
|
||||
const int16x8_t ab = vcombine_s16(a, b);
|
||||
const int16x8_t cd = vcombine_s16(c, d);
|
||||
// Stage 3
|
||||
v_x0 = vaddq_s16(v_s4, ab);
|
||||
v_x1 = vsubq_s16(v_s4, ab);
|
||||
v_x2 = vsubq_s16(v_s7, cd);
|
||||
v_x3 = vaddq_s16(v_s7, cd);
|
||||
}
|
||||
// Stage 4
|
||||
v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_4_64);
|
||||
v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_4_64);
|
||||
v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), cospi_28_64);
|
||||
v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), cospi_28_64);
|
||||
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_12_64);
|
||||
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_12_64);
|
||||
v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), cospi_20_64);
|
||||
v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), cospi_20_64);
|
||||
v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_12_64);
|
||||
v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_12_64);
|
||||
v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), cospi_20_64);
|
||||
v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), cospi_20_64);
|
||||
v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_28_64);
|
||||
v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_28_64);
|
||||
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), cospi_4_64);
|
||||
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), cospi_4_64);
|
||||
{
|
||||
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
|
||||
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
|
||||
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
|
||||
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
|
||||
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
|
||||
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
|
||||
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
|
||||
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
|
||||
out_1 = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53
|
||||
out_3 = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73
|
||||
out_5 = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57
|
||||
out_7 = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77
|
||||
}
|
||||
// transpose 8x8
|
||||
// Can't use transpose_s16_8x8() because the values are arranged in two 4x8
|
||||
// columns.
|
||||
{
|
||||
// 00 01 02 03 40 41 42 43
|
||||
// 10 11 12 13 50 51 52 53
|
||||
// 20 21 22 23 60 61 62 63
|
||||
// 30 31 32 33 70 71 72 73
|
||||
// 04 05 06 07 44 45 46 47
|
||||
// 14 15 16 17 54 55 56 57
|
||||
// 24 25 26 27 64 65 66 67
|
||||
// 34 35 36 37 74 75 76 77
|
||||
const int32x4x2_t r02_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out_0), vreinterpretq_s32_s16(out_2));
|
||||
const int32x4x2_t r13_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out_1), vreinterpretq_s32_s16(out_3));
|
||||
const int32x4x2_t r46_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out_4), vreinterpretq_s32_s16(out_6));
|
||||
const int32x4x2_t r57_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out_5), vreinterpretq_s32_s16(out_7));
|
||||
const int16x8x2_t r01_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]),
|
||||
vreinterpretq_s16_s32(r13_s32.val[0]));
|
||||
const int16x8x2_t r23_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]),
|
||||
vreinterpretq_s16_s32(r13_s32.val[1]));
|
||||
const int16x8x2_t r45_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]),
|
||||
vreinterpretq_s16_s32(r57_s32.val[0]));
|
||||
const int16x8x2_t r67_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]),
|
||||
vreinterpretq_s16_s32(r57_s32.val[1]));
|
||||
input_0 = r01_s16.val[0];
|
||||
input_1 = r01_s16.val[1];
|
||||
input_2 = r23_s16.val[0];
|
||||
input_3 = r23_s16.val[1];
|
||||
input_4 = r45_s16.val[0];
|
||||
input_5 = r45_s16.val[1];
|
||||
input_6 = r67_s16.val[0];
|
||||
input_7 = r67_s16.val[1];
|
||||
// 00 10 20 30 40 50 60 70
|
||||
// 01 11 21 31 41 51 61 71
|
||||
// 02 12 22 32 42 52 62 72
|
||||
// 03 13 23 33 43 53 63 73
|
||||
// 04 14 24 34 44 54 64 74
|
||||
// 05 15 25 35 45 55 65 75
|
||||
// 06 16 26 36 46 56 66 76
|
||||
// 07 17 27 37 47 57 67 77
|
||||
}
|
||||
} // for
|
||||
{
|
||||
// from vpx_dct_sse2.c
|
||||
// Post-condition (division by two)
|
||||
// division of two 16 bits signed numbers using shifts
|
||||
// n / 2 = (n - (n >> 15)) >> 1
|
||||
const int16x8_t sign_in0 = vshrq_n_s16(input_0, 15);
|
||||
const int16x8_t sign_in1 = vshrq_n_s16(input_1, 15);
|
||||
const int16x8_t sign_in2 = vshrq_n_s16(input_2, 15);
|
||||
const int16x8_t sign_in3 = vshrq_n_s16(input_3, 15);
|
||||
const int16x8_t sign_in4 = vshrq_n_s16(input_4, 15);
|
||||
const int16x8_t sign_in5 = vshrq_n_s16(input_5, 15);
|
||||
const int16x8_t sign_in6 = vshrq_n_s16(input_6, 15);
|
||||
const int16x8_t sign_in7 = vshrq_n_s16(input_7, 15);
|
||||
input_0 = vhsubq_s16(input_0, sign_in0);
|
||||
input_1 = vhsubq_s16(input_1, sign_in1);
|
||||
input_2 = vhsubq_s16(input_2, sign_in2);
|
||||
input_3 = vhsubq_s16(input_3, sign_in3);
|
||||
input_4 = vhsubq_s16(input_4, sign_in4);
|
||||
input_5 = vhsubq_s16(input_5, sign_in5);
|
||||
input_6 = vhsubq_s16(input_6, sign_in6);
|
||||
input_7 = vhsubq_s16(input_7, sign_in7);
|
||||
// store results
|
||||
store_s16q_to_tran_low(final_output + 0 * 8, input_0);
|
||||
store_s16q_to_tran_low(final_output + 1 * 8, input_1);
|
||||
store_s16q_to_tran_low(final_output + 2 * 8, input_2);
|
||||
store_s16q_to_tran_low(final_output + 3 * 8, input_3);
|
||||
store_s16q_to_tran_low(final_output + 4 * 8, input_4);
|
||||
store_s16q_to_tran_low(final_output + 5 * 8, input_5);
|
||||
store_s16q_to_tran_low(final_output + 6 * 8, input_6);
|
||||
store_s16q_to_tran_low(final_output + 7 * 8, input_7);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
|
||||
static void hadamard8x8_one_pass(int16x8_t *a0, int16x8_t *a1, int16x8_t *a2,
|
||||
int16x8_t *a3, int16x8_t *a4, int16x8_t *a5,
|
||||
int16x8_t *a6, int16x8_t *a7) {
|
||||
const int16x8_t b0 = vaddq_s16(*a0, *a1);
|
||||
const int16x8_t b1 = vsubq_s16(*a0, *a1);
|
||||
const int16x8_t b2 = vaddq_s16(*a2, *a3);
|
||||
const int16x8_t b3 = vsubq_s16(*a2, *a3);
|
||||
const int16x8_t b4 = vaddq_s16(*a4, *a5);
|
||||
const int16x8_t b5 = vsubq_s16(*a4, *a5);
|
||||
const int16x8_t b6 = vaddq_s16(*a6, *a7);
|
||||
const int16x8_t b7 = vsubq_s16(*a6, *a7);
|
||||
|
||||
const int16x8_t c0 = vaddq_s16(b0, b2);
|
||||
const int16x8_t c1 = vaddq_s16(b1, b3);
|
||||
const int16x8_t c2 = vsubq_s16(b0, b2);
|
||||
const int16x8_t c3 = vsubq_s16(b1, b3);
|
||||
const int16x8_t c4 = vaddq_s16(b4, b6);
|
||||
const int16x8_t c5 = vaddq_s16(b5, b7);
|
||||
const int16x8_t c6 = vsubq_s16(b4, b6);
|
||||
const int16x8_t c7 = vsubq_s16(b5, b7);
|
||||
|
||||
*a0 = vaddq_s16(c0, c4);
|
||||
*a1 = vsubq_s16(c2, c6);
|
||||
*a2 = vsubq_s16(c0, c4);
|
||||
*a3 = vaddq_s16(c2, c6);
|
||||
*a4 = vaddq_s16(c3, c7);
|
||||
*a5 = vsubq_s16(c3, c7);
|
||||
*a6 = vsubq_s16(c1, c5);
|
||||
*a7 = vaddq_s16(c1, c5);
|
||||
}
|
||||
|
||||
void vpx_hadamard_8x8_neon(const int16_t *src_diff, ptrdiff_t src_stride,
|
||||
tran_low_t *coeff) {
|
||||
int16x8_t a0 = vld1q_s16(src_diff);
|
||||
int16x8_t a1 = vld1q_s16(src_diff + src_stride);
|
||||
int16x8_t a2 = vld1q_s16(src_diff + 2 * src_stride);
|
||||
int16x8_t a3 = vld1q_s16(src_diff + 3 * src_stride);
|
||||
int16x8_t a4 = vld1q_s16(src_diff + 4 * src_stride);
|
||||
int16x8_t a5 = vld1q_s16(src_diff + 5 * src_stride);
|
||||
int16x8_t a6 = vld1q_s16(src_diff + 6 * src_stride);
|
||||
int16x8_t a7 = vld1q_s16(src_diff + 7 * src_stride);
|
||||
|
||||
hadamard8x8_one_pass(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
|
||||
|
||||
transpose_s16_8x8(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
|
||||
|
||||
hadamard8x8_one_pass(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
|
||||
|
||||
// Skip the second transpose because it is not required.
|
||||
|
||||
store_s16q_to_tran_low(coeff + 0, a0);
|
||||
store_s16q_to_tran_low(coeff + 8, a1);
|
||||
store_s16q_to_tran_low(coeff + 16, a2);
|
||||
store_s16q_to_tran_low(coeff + 24, a3);
|
||||
store_s16q_to_tran_low(coeff + 32, a4);
|
||||
store_s16q_to_tran_low(coeff + 40, a5);
|
||||
store_s16q_to_tran_low(coeff + 48, a6);
|
||||
store_s16q_to_tran_low(coeff + 56, a7);
|
||||
}
|
||||
|
||||
void vpx_hadamard_16x16_neon(const int16_t *src_diff, ptrdiff_t src_stride,
|
||||
tran_low_t *coeff) {
|
||||
int i;
|
||||
|
||||
/* Rearrange 16x16 to 8x32 and remove stride.
|
||||
* Top left first. */
|
||||
vpx_hadamard_8x8_neon(src_diff + 0 + 0 * src_stride, src_stride, coeff + 0);
|
||||
/* Top right. */
|
||||
vpx_hadamard_8x8_neon(src_diff + 8 + 0 * src_stride, src_stride, coeff + 64);
|
||||
/* Bottom left. */
|
||||
vpx_hadamard_8x8_neon(src_diff + 0 + 8 * src_stride, src_stride, coeff + 128);
|
||||
/* Bottom right. */
|
||||
vpx_hadamard_8x8_neon(src_diff + 8 + 8 * src_stride, src_stride, coeff + 192);
|
||||
|
||||
for (i = 0; i < 64; i += 8) {
|
||||
const int16x8_t a0 = load_tran_low_to_s16q(coeff + 0);
|
||||
const int16x8_t a1 = load_tran_low_to_s16q(coeff + 64);
|
||||
const int16x8_t a2 = load_tran_low_to_s16q(coeff + 128);
|
||||
const int16x8_t a3 = load_tran_low_to_s16q(coeff + 192);
|
||||
|
||||
const int16x8_t b0 = vhaddq_s16(a0, a1);
|
||||
const int16x8_t b1 = vhsubq_s16(a0, a1);
|
||||
const int16x8_t b2 = vhaddq_s16(a2, a3);
|
||||
const int16x8_t b3 = vhsubq_s16(a2, a3);
|
||||
|
||||
const int16x8_t c0 = vaddq_s16(b0, b2);
|
||||
const int16x8_t c1 = vaddq_s16(b1, b3);
|
||||
const int16x8_t c2 = vsubq_s16(b0, b2);
|
||||
const int16x8_t c3 = vsubq_s16(b1, b3);
|
||||
|
||||
store_s16q_to_tran_low(coeff + 0, c0);
|
||||
store_s16q_to_tran_low(coeff + 64, c1);
|
||||
store_s16q_to_tran_low(coeff + 128, c2);
|
||||
store_s16q_to_tran_low(coeff + 192, c3);
|
||||
|
||||
coeff += 8;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,640 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
|
||||
static INLINE void load_from_transformed(const int32_t *const trans_buf,
|
||||
const int first, const int second,
|
||||
int32x4x2_t *const q0,
|
||||
int32x4x2_t *const q1) {
|
||||
q0->val[0] = vld1q_s32(trans_buf + first * 8);
|
||||
q0->val[1] = vld1q_s32(trans_buf + first * 8 + 4);
|
||||
q1->val[0] = vld1q_s32(trans_buf + second * 8);
|
||||
q1->val[1] = vld1q_s32(trans_buf + second * 8 + 4);
|
||||
}
|
||||
|
||||
static INLINE void load_from_output(const int32_t *const out, const int first,
|
||||
const int second, int32x4x2_t *const q0,
|
||||
int32x4x2_t *const q1) {
|
||||
q0->val[0] = vld1q_s32(out + first * 32);
|
||||
q0->val[1] = vld1q_s32(out + first * 32 + 4);
|
||||
q1->val[0] = vld1q_s32(out + second * 32);
|
||||
q1->val[1] = vld1q_s32(out + second * 32 + 4);
|
||||
}
|
||||
|
||||
static INLINE void store_in_output(int32_t *const out, const int first,
|
||||
const int second, const int32x4x2_t q0,
|
||||
const int32x4x2_t q1) {
|
||||
vst1q_s32(out + first * 32, q0.val[0]);
|
||||
vst1q_s32(out + first * 32 + 4, q0.val[1]);
|
||||
vst1q_s32(out + second * 32, q1.val[0]);
|
||||
vst1q_s32(out + second * 32 + 4, q1.val[1]);
|
||||
}
|
||||
|
||||
static INLINE void highbd_store_combine_results(
|
||||
uint16_t *p1, uint16_t *p2, const int stride, const int32x4x2_t q0,
|
||||
const int32x4x2_t q1, const int32x4x2_t q2, const int32x4x2_t q3,
|
||||
const int16x8_t max) {
|
||||
int16x8_t o[4];
|
||||
uint16x8_t d[4];
|
||||
|
||||
d[0] = vld1q_u16(p1);
|
||||
p1 += stride;
|
||||
d[1] = vld1q_u16(p1);
|
||||
d[3] = vld1q_u16(p2);
|
||||
p2 -= stride;
|
||||
d[2] = vld1q_u16(p2);
|
||||
|
||||
o[0] = vcombine_s16(vrshrn_n_s32(q0.val[0], 6), vrshrn_n_s32(q0.val[1], 6));
|
||||
o[1] = vcombine_s16(vrshrn_n_s32(q1.val[0], 6), vrshrn_n_s32(q1.val[1], 6));
|
||||
o[2] = vcombine_s16(vrshrn_n_s32(q2.val[0], 6), vrshrn_n_s32(q2.val[1], 6));
|
||||
o[3] = vcombine_s16(vrshrn_n_s32(q3.val[0], 6), vrshrn_n_s32(q3.val[1], 6));
|
||||
|
||||
o[0] = vqaddq_s16(o[0], vreinterpretq_s16_u16(d[0]));
|
||||
o[1] = vqaddq_s16(o[1], vreinterpretq_s16_u16(d[1]));
|
||||
o[2] = vqaddq_s16(o[2], vreinterpretq_s16_u16(d[2]));
|
||||
o[3] = vqaddq_s16(o[3], vreinterpretq_s16_u16(d[3]));
|
||||
o[0] = vminq_s16(o[0], max);
|
||||
o[1] = vminq_s16(o[1], max);
|
||||
o[2] = vminq_s16(o[2], max);
|
||||
o[3] = vminq_s16(o[3], max);
|
||||
d[0] = vqshluq_n_s16(o[0], 0);
|
||||
d[1] = vqshluq_n_s16(o[1], 0);
|
||||
d[2] = vqshluq_n_s16(o[2], 0);
|
||||
d[3] = vqshluq_n_s16(o[3], 0);
|
||||
|
||||
vst1q_u16(p1, d[1]);
|
||||
p1 -= stride;
|
||||
vst1q_u16(p1, d[0]);
|
||||
vst1q_u16(p2, d[2]);
|
||||
p2 += stride;
|
||||
vst1q_u16(p2, d[3]);
|
||||
}
|
||||
|
||||
static INLINE void do_butterfly(const int32x4x2_t qIn0, const int32x4x2_t qIn1,
|
||||
const int32_t first_const,
|
||||
const int32_t second_const,
|
||||
int32x4x2_t *const qOut0,
|
||||
int32x4x2_t *const qOut1) {
|
||||
int64x2x2_t q[4];
|
||||
int32x2_t d[6];
|
||||
|
||||
// Note: using v{mul, mla, mls}l_n_s32 here slows down 35% with gcc 4.9.
|
||||
d[4] = vdup_n_s32(first_const);
|
||||
d[5] = vdup_n_s32(second_const);
|
||||
|
||||
q[0].val[0] = vmull_s32(vget_low_s32(qIn0.val[0]), d[4]);
|
||||
q[0].val[1] = vmull_s32(vget_high_s32(qIn0.val[0]), d[4]);
|
||||
q[1].val[0] = vmull_s32(vget_low_s32(qIn0.val[1]), d[4]);
|
||||
q[1].val[1] = vmull_s32(vget_high_s32(qIn0.val[1]), d[4]);
|
||||
q[0].val[0] = vmlsl_s32(q[0].val[0], vget_low_s32(qIn1.val[0]), d[5]);
|
||||
q[0].val[1] = vmlsl_s32(q[0].val[1], vget_high_s32(qIn1.val[0]), d[5]);
|
||||
q[1].val[0] = vmlsl_s32(q[1].val[0], vget_low_s32(qIn1.val[1]), d[5]);
|
||||
q[1].val[1] = vmlsl_s32(q[1].val[1], vget_high_s32(qIn1.val[1]), d[5]);
|
||||
|
||||
q[2].val[0] = vmull_s32(vget_low_s32(qIn0.val[0]), d[5]);
|
||||
q[2].val[1] = vmull_s32(vget_high_s32(qIn0.val[0]), d[5]);
|
||||
q[3].val[0] = vmull_s32(vget_low_s32(qIn0.val[1]), d[5]);
|
||||
q[3].val[1] = vmull_s32(vget_high_s32(qIn0.val[1]), d[5]);
|
||||
q[2].val[0] = vmlal_s32(q[2].val[0], vget_low_s32(qIn1.val[0]), d[4]);
|
||||
q[2].val[1] = vmlal_s32(q[2].val[1], vget_high_s32(qIn1.val[0]), d[4]);
|
||||
q[3].val[0] = vmlal_s32(q[3].val[0], vget_low_s32(qIn1.val[1]), d[4]);
|
||||
q[3].val[1] = vmlal_s32(q[3].val[1], vget_high_s32(qIn1.val[1]), d[4]);
|
||||
|
||||
qOut0->val[0] = vcombine_s32(vrshrn_n_s64(q[0].val[0], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(q[0].val[1], DCT_CONST_BITS));
|
||||
qOut0->val[1] = vcombine_s32(vrshrn_n_s64(q[1].val[0], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(q[1].val[1], DCT_CONST_BITS));
|
||||
qOut1->val[0] = vcombine_s32(vrshrn_n_s64(q[2].val[0], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(q[2].val[1], DCT_CONST_BITS));
|
||||
qOut1->val[1] = vcombine_s32(vrshrn_n_s64(q[3].val[0], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(q[3].val[1], DCT_CONST_BITS));
|
||||
}
|
||||
|
||||
static INLINE void load_s32x4q_dual(const int32_t *in, int32x4x2_t *const s) {
|
||||
s[0].val[0] = vld1q_s32(in);
|
||||
s[0].val[1] = vld1q_s32(in + 4);
|
||||
in += 32;
|
||||
s[1].val[0] = vld1q_s32(in);
|
||||
s[1].val[1] = vld1q_s32(in + 4);
|
||||
in += 32;
|
||||
s[2].val[0] = vld1q_s32(in);
|
||||
s[2].val[1] = vld1q_s32(in + 4);
|
||||
in += 32;
|
||||
s[3].val[0] = vld1q_s32(in);
|
||||
s[3].val[1] = vld1q_s32(in + 4);
|
||||
in += 32;
|
||||
s[4].val[0] = vld1q_s32(in);
|
||||
s[4].val[1] = vld1q_s32(in + 4);
|
||||
in += 32;
|
||||
s[5].val[0] = vld1q_s32(in);
|
||||
s[5].val[1] = vld1q_s32(in + 4);
|
||||
in += 32;
|
||||
s[6].val[0] = vld1q_s32(in);
|
||||
s[6].val[1] = vld1q_s32(in + 4);
|
||||
in += 32;
|
||||
s[7].val[0] = vld1q_s32(in);
|
||||
s[7].val[1] = vld1q_s32(in + 4);
|
||||
}
|
||||
|
||||
static INLINE void transpose_and_store_s32_8x8(int32x4x2_t *const a,
|
||||
int32_t **out) {
|
||||
transpose_s32_8x8(&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], &a[7]);
|
||||
|
||||
vst1q_s32(*out, a[0].val[0]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[0].val[1]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[1].val[0]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[1].val[1]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[2].val[0]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[2].val[1]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[3].val[0]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[3].val[1]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[4].val[0]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[4].val[1]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[5].val[0]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[5].val[1]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[6].val[0]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[6].val[1]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[7].val[0]);
|
||||
*out += 4;
|
||||
vst1q_s32(*out, a[7].val[1]);
|
||||
*out += 4;
|
||||
}
|
||||
|
||||
static INLINE void idct32_transpose_pair(const int32_t *input, int32_t *t_buf) {
|
||||
int i;
|
||||
int32x4x2_t s[8];
|
||||
|
||||
for (i = 0; i < 4; i++, input += 8) {
|
||||
load_s32x4q_dual(input, s);
|
||||
transpose_and_store_s32_8x8(s, &t_buf);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void idct32_bands_end_1st_pass(int32_t *const out,
|
||||
int32x4x2_t *const q) {
|
||||
store_in_output(out, 16, 17, q[6], q[7]);
|
||||
store_in_output(out, 14, 15, q[8], q[9]);
|
||||
|
||||
load_from_output(out, 30, 31, &q[0], &q[1]);
|
||||
q[4] = highbd_idct_add_dual(q[2], q[1]);
|
||||
q[5] = highbd_idct_add_dual(q[3], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[3], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[2], q[1]);
|
||||
store_in_output(out, 30, 31, q[6], q[7]);
|
||||
store_in_output(out, 0, 1, q[4], q[5]);
|
||||
|
||||
load_from_output(out, 12, 13, &q[0], &q[1]);
|
||||
q[2] = highbd_idct_add_dual(q[10], q[1]);
|
||||
q[3] = highbd_idct_add_dual(q[11], q[0]);
|
||||
q[4] = highbd_idct_sub_dual(q[11], q[0]);
|
||||
q[5] = highbd_idct_sub_dual(q[10], q[1]);
|
||||
|
||||
load_from_output(out, 18, 19, &q[0], &q[1]);
|
||||
q[8] = highbd_idct_add_dual(q[4], q[1]);
|
||||
q[9] = highbd_idct_add_dual(q[5], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[5], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[4], q[1]);
|
||||
store_in_output(out, 18, 19, q[6], q[7]);
|
||||
store_in_output(out, 12, 13, q[8], q[9]);
|
||||
|
||||
load_from_output(out, 28, 29, &q[0], &q[1]);
|
||||
q[4] = highbd_idct_add_dual(q[2], q[1]);
|
||||
q[5] = highbd_idct_add_dual(q[3], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[3], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[2], q[1]);
|
||||
store_in_output(out, 28, 29, q[6], q[7]);
|
||||
store_in_output(out, 2, 3, q[4], q[5]);
|
||||
|
||||
load_from_output(out, 10, 11, &q[0], &q[1]);
|
||||
q[2] = highbd_idct_add_dual(q[12], q[1]);
|
||||
q[3] = highbd_idct_add_dual(q[13], q[0]);
|
||||
q[4] = highbd_idct_sub_dual(q[13], q[0]);
|
||||
q[5] = highbd_idct_sub_dual(q[12], q[1]);
|
||||
|
||||
load_from_output(out, 20, 21, &q[0], &q[1]);
|
||||
q[8] = highbd_idct_add_dual(q[4], q[1]);
|
||||
q[9] = highbd_idct_add_dual(q[5], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[5], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[4], q[1]);
|
||||
store_in_output(out, 20, 21, q[6], q[7]);
|
||||
store_in_output(out, 10, 11, q[8], q[9]);
|
||||
|
||||
load_from_output(out, 26, 27, &q[0], &q[1]);
|
||||
q[4] = highbd_idct_add_dual(q[2], q[1]);
|
||||
q[5] = highbd_idct_add_dual(q[3], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[3], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[2], q[1]);
|
||||
store_in_output(out, 26, 27, q[6], q[7]);
|
||||
store_in_output(out, 4, 5, q[4], q[5]);
|
||||
|
||||
load_from_output(out, 8, 9, &q[0], &q[1]);
|
||||
q[2] = highbd_idct_add_dual(q[14], q[1]);
|
||||
q[3] = highbd_idct_add_dual(q[15], q[0]);
|
||||
q[4] = highbd_idct_sub_dual(q[15], q[0]);
|
||||
q[5] = highbd_idct_sub_dual(q[14], q[1]);
|
||||
|
||||
load_from_output(out, 22, 23, &q[0], &q[1]);
|
||||
q[8] = highbd_idct_add_dual(q[4], q[1]);
|
||||
q[9] = highbd_idct_add_dual(q[5], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[5], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[4], q[1]);
|
||||
store_in_output(out, 22, 23, q[6], q[7]);
|
||||
store_in_output(out, 8, 9, q[8], q[9]);
|
||||
|
||||
load_from_output(out, 24, 25, &q[0], &q[1]);
|
||||
q[4] = highbd_idct_add_dual(q[2], q[1]);
|
||||
q[5] = highbd_idct_add_dual(q[3], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[3], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[2], q[1]);
|
||||
store_in_output(out, 24, 25, q[6], q[7]);
|
||||
store_in_output(out, 6, 7, q[4], q[5]);
|
||||
}
|
||||
|
||||
static INLINE void idct32_bands_end_2nd_pass(const int32_t *const out,
|
||||
uint16_t *const dest,
|
||||
const int stride,
|
||||
const int16x8_t max,
|
||||
int32x4x2_t *const q) {
|
||||
uint16_t *dest0 = dest + 0 * stride;
|
||||
uint16_t *dest1 = dest + 31 * stride;
|
||||
uint16_t *dest2 = dest + 16 * stride;
|
||||
uint16_t *dest3 = dest + 15 * stride;
|
||||
const int str2 = stride << 1;
|
||||
|
||||
highbd_store_combine_results(dest2, dest3, stride, q[6], q[7], q[8], q[9],
|
||||
max);
|
||||
dest2 += str2;
|
||||
dest3 -= str2;
|
||||
|
||||
load_from_output(out, 30, 31, &q[0], &q[1]);
|
||||
q[4] = highbd_idct_add_dual(q[2], q[1]);
|
||||
q[5] = highbd_idct_add_dual(q[3], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[3], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[2], q[1]);
|
||||
highbd_store_combine_results(dest0, dest1, stride, q[4], q[5], q[6], q[7],
|
||||
max);
|
||||
dest0 += str2;
|
||||
dest1 -= str2;
|
||||
|
||||
load_from_output(out, 12, 13, &q[0], &q[1]);
|
||||
q[2] = highbd_idct_add_dual(q[10], q[1]);
|
||||
q[3] = highbd_idct_add_dual(q[11], q[0]);
|
||||
q[4] = highbd_idct_sub_dual(q[11], q[0]);
|
||||
q[5] = highbd_idct_sub_dual(q[10], q[1]);
|
||||
|
||||
load_from_output(out, 18, 19, &q[0], &q[1]);
|
||||
q[8] = highbd_idct_add_dual(q[4], q[1]);
|
||||
q[9] = highbd_idct_add_dual(q[5], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[5], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[4], q[1]);
|
||||
highbd_store_combine_results(dest2, dest3, stride, q[6], q[7], q[8], q[9],
|
||||
max);
|
||||
dest2 += str2;
|
||||
dest3 -= str2;
|
||||
|
||||
load_from_output(out, 28, 29, &q[0], &q[1]);
|
||||
q[4] = highbd_idct_add_dual(q[2], q[1]);
|
||||
q[5] = highbd_idct_add_dual(q[3], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[3], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[2], q[1]);
|
||||
highbd_store_combine_results(dest0, dest1, stride, q[4], q[5], q[6], q[7],
|
||||
max);
|
||||
dest0 += str2;
|
||||
dest1 -= str2;
|
||||
|
||||
load_from_output(out, 10, 11, &q[0], &q[1]);
|
||||
q[2] = highbd_idct_add_dual(q[12], q[1]);
|
||||
q[3] = highbd_idct_add_dual(q[13], q[0]);
|
||||
q[4] = highbd_idct_sub_dual(q[13], q[0]);
|
||||
q[5] = highbd_idct_sub_dual(q[12], q[1]);
|
||||
|
||||
load_from_output(out, 20, 21, &q[0], &q[1]);
|
||||
q[8] = highbd_idct_add_dual(q[4], q[1]);
|
||||
q[9] = highbd_idct_add_dual(q[5], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[5], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[4], q[1]);
|
||||
highbd_store_combine_results(dest2, dest3, stride, q[6], q[7], q[8], q[9],
|
||||
max);
|
||||
dest2 += str2;
|
||||
dest3 -= str2;
|
||||
|
||||
load_from_output(out, 26, 27, &q[0], &q[1]);
|
||||
q[4] = highbd_idct_add_dual(q[2], q[1]);
|
||||
q[5] = highbd_idct_add_dual(q[3], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[3], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[2], q[1]);
|
||||
highbd_store_combine_results(dest0, dest1, stride, q[4], q[5], q[6], q[7],
|
||||
max);
|
||||
dest0 += str2;
|
||||
dest1 -= str2;
|
||||
|
||||
load_from_output(out, 8, 9, &q[0], &q[1]);
|
||||
q[2] = highbd_idct_add_dual(q[14], q[1]);
|
||||
q[3] = highbd_idct_add_dual(q[15], q[0]);
|
||||
q[4] = highbd_idct_sub_dual(q[15], q[0]);
|
||||
q[5] = highbd_idct_sub_dual(q[14], q[1]);
|
||||
|
||||
load_from_output(out, 22, 23, &q[0], &q[1]);
|
||||
q[8] = highbd_idct_add_dual(q[4], q[1]);
|
||||
q[9] = highbd_idct_add_dual(q[5], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[5], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[4], q[1]);
|
||||
highbd_store_combine_results(dest2, dest3, stride, q[6], q[7], q[8], q[9],
|
||||
max);
|
||||
|
||||
load_from_output(out, 24, 25, &q[0], &q[1]);
|
||||
q[4] = highbd_idct_add_dual(q[2], q[1]);
|
||||
q[5] = highbd_idct_add_dual(q[3], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[3], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[2], q[1]);
|
||||
highbd_store_combine_results(dest0, dest1, stride, q[4], q[5], q[6], q[7],
|
||||
max);
|
||||
}
|
||||
|
||||
static INLINE void vpx_highbd_idct32_32_neon(const tran_low_t *input,
|
||||
uint16_t *dst, const int stride,
|
||||
const int bd) {
|
||||
int i, idct32_pass_loop;
|
||||
int32_t trans_buf[32 * 8];
|
||||
int32_t pass1[32 * 32];
|
||||
int32_t pass2[32 * 32];
|
||||
int32_t *out;
|
||||
int32x4x2_t q[16];
|
||||
|
||||
for (idct32_pass_loop = 0, out = pass1; idct32_pass_loop < 2;
|
||||
idct32_pass_loop++, input = pass1, out = pass2) {
|
||||
for (i = 0; i < 4; i++, out += 8) { // idct32_bands_loop
|
||||
idct32_transpose_pair(input, trans_buf);
|
||||
input += 32 * 8;
|
||||
|
||||
// -----------------------------------------
|
||||
// BLOCK A: 16-19,28-31
|
||||
// -----------------------------------------
|
||||
// generate 16,17,30,31
|
||||
// part of stage 1
|
||||
load_from_transformed(trans_buf, 1, 31, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_31_64, cospi_1_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 17, 15, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_15_64, cospi_17_64, &q[1], &q[3]);
|
||||
// part of stage 2
|
||||
q[4] = highbd_idct_add_dual(q[0], q[1]);
|
||||
q[13] = highbd_idct_sub_dual(q[0], q[1]);
|
||||
q[6] = highbd_idct_add_dual(q[2], q[3]);
|
||||
q[14] = highbd_idct_sub_dual(q[2], q[3]);
|
||||
// part of stage 3
|
||||
do_butterfly(q[14], q[13], cospi_28_64, cospi_4_64, &q[5], &q[7]);
|
||||
|
||||
// generate 18,19,28,29
|
||||
// part of stage 1
|
||||
load_from_transformed(trans_buf, 9, 23, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_23_64, cospi_9_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 25, 7, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_7_64, cospi_25_64, &q[1], &q[3]);
|
||||
// part of stage 2
|
||||
q[13] = highbd_idct_sub_dual(q[3], q[2]);
|
||||
q[3] = highbd_idct_add_dual(q[3], q[2]);
|
||||
q[14] = highbd_idct_sub_dual(q[1], q[0]);
|
||||
q[2] = highbd_idct_add_dual(q[1], q[0]);
|
||||
// part of stage 3
|
||||
do_butterfly(q[14], q[13], -cospi_4_64, -cospi_28_64, &q[1], &q[0]);
|
||||
// part of stage 4
|
||||
q[8] = highbd_idct_add_dual(q[4], q[2]);
|
||||
q[9] = highbd_idct_add_dual(q[5], q[0]);
|
||||
q[10] = highbd_idct_add_dual(q[7], q[1]);
|
||||
q[15] = highbd_idct_add_dual(q[6], q[3]);
|
||||
q[13] = highbd_idct_sub_dual(q[5], q[0]);
|
||||
q[14] = highbd_idct_sub_dual(q[7], q[1]);
|
||||
store_in_output(out, 16, 31, q[8], q[15]);
|
||||
store_in_output(out, 17, 30, q[9], q[10]);
|
||||
// part of stage 5
|
||||
do_butterfly(q[14], q[13], cospi_24_64, cospi_8_64, &q[0], &q[1]);
|
||||
store_in_output(out, 29, 18, q[1], q[0]);
|
||||
// part of stage 4
|
||||
q[13] = highbd_idct_sub_dual(q[4], q[2]);
|
||||
q[14] = highbd_idct_sub_dual(q[6], q[3]);
|
||||
// part of stage 5
|
||||
do_butterfly(q[14], q[13], cospi_24_64, cospi_8_64, &q[4], &q[6]);
|
||||
store_in_output(out, 19, 28, q[4], q[6]);
|
||||
|
||||
// -----------------------------------------
|
||||
// BLOCK B: 20-23,24-27
|
||||
// -----------------------------------------
|
||||
// generate 20,21,26,27
|
||||
// part of stage 1
|
||||
load_from_transformed(trans_buf, 5, 27, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_27_64, cospi_5_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 21, 11, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_11_64, cospi_21_64, &q[1], &q[3]);
|
||||
// part of stage 2
|
||||
q[13] = highbd_idct_sub_dual(q[0], q[1]);
|
||||
q[0] = highbd_idct_add_dual(q[0], q[1]);
|
||||
q[14] = highbd_idct_sub_dual(q[2], q[3]);
|
||||
q[2] = highbd_idct_add_dual(q[2], q[3]);
|
||||
// part of stage 3
|
||||
do_butterfly(q[14], q[13], cospi_12_64, cospi_20_64, &q[1], &q[3]);
|
||||
|
||||
// generate 22,23,24,25
|
||||
// part of stage 1
|
||||
load_from_transformed(trans_buf, 13, 19, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_19_64, cospi_13_64, &q[5], &q[7]);
|
||||
load_from_transformed(trans_buf, 29, 3, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_3_64, cospi_29_64, &q[4], &q[6]);
|
||||
// part of stage 2
|
||||
q[14] = highbd_idct_sub_dual(q[4], q[5]);
|
||||
q[5] = highbd_idct_add_dual(q[4], q[5]);
|
||||
q[13] = highbd_idct_sub_dual(q[6], q[7]);
|
||||
q[6] = highbd_idct_add_dual(q[6], q[7]);
|
||||
// part of stage 3
|
||||
do_butterfly(q[14], q[13], -cospi_20_64, -cospi_12_64, &q[4], &q[7]);
|
||||
// part of stage 4
|
||||
q[10] = highbd_idct_add_dual(q[7], q[1]);
|
||||
q[11] = highbd_idct_add_dual(q[5], q[0]);
|
||||
q[12] = highbd_idct_add_dual(q[6], q[2]);
|
||||
q[15] = highbd_idct_add_dual(q[4], q[3]);
|
||||
// part of stage 6
|
||||
load_from_output(out, 16, 17, &q[14], &q[13]);
|
||||
q[8] = highbd_idct_add_dual(q[14], q[11]);
|
||||
q[9] = highbd_idct_add_dual(q[13], q[10]);
|
||||
q[13] = highbd_idct_sub_dual(q[13], q[10]);
|
||||
q[11] = highbd_idct_sub_dual(q[14], q[11]);
|
||||
store_in_output(out, 17, 16, q[9], q[8]);
|
||||
load_from_output(out, 30, 31, &q[14], &q[9]);
|
||||
q[8] = highbd_idct_sub_dual(q[9], q[12]);
|
||||
q[10] = highbd_idct_add_dual(q[14], q[15]);
|
||||
q[14] = highbd_idct_sub_dual(q[14], q[15]);
|
||||
q[12] = highbd_idct_add_dual(q[9], q[12]);
|
||||
store_in_output(out, 30, 31, q[10], q[12]);
|
||||
// part of stage 7
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[13], &q[14]);
|
||||
store_in_output(out, 25, 22, q[14], q[13]);
|
||||
do_butterfly(q[8], q[11], cospi_16_64, cospi_16_64, &q[13], &q[14]);
|
||||
store_in_output(out, 24, 23, q[14], q[13]);
|
||||
// part of stage 4
|
||||
q[14] = highbd_idct_sub_dual(q[5], q[0]);
|
||||
q[13] = highbd_idct_sub_dual(q[6], q[2]);
|
||||
do_butterfly(q[14], q[13], -cospi_8_64, -cospi_24_64, &q[5], &q[6]);
|
||||
q[14] = highbd_idct_sub_dual(q[7], q[1]);
|
||||
q[13] = highbd_idct_sub_dual(q[4], q[3]);
|
||||
do_butterfly(q[14], q[13], -cospi_8_64, -cospi_24_64, &q[0], &q[1]);
|
||||
// part of stage 6
|
||||
load_from_output(out, 18, 19, &q[14], &q[13]);
|
||||
q[8] = highbd_idct_add_dual(q[14], q[1]);
|
||||
q[9] = highbd_idct_add_dual(q[13], q[6]);
|
||||
q[13] = highbd_idct_sub_dual(q[13], q[6]);
|
||||
q[1] = highbd_idct_sub_dual(q[14], q[1]);
|
||||
store_in_output(out, 18, 19, q[8], q[9]);
|
||||
load_from_output(out, 28, 29, &q[8], &q[9]);
|
||||
q[14] = highbd_idct_sub_dual(q[8], q[5]);
|
||||
q[10] = highbd_idct_add_dual(q[8], q[5]);
|
||||
q[11] = highbd_idct_add_dual(q[9], q[0]);
|
||||
q[0] = highbd_idct_sub_dual(q[9], q[0]);
|
||||
store_in_output(out, 28, 29, q[10], q[11]);
|
||||
// part of stage 7
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[13], &q[14]);
|
||||
store_in_output(out, 20, 27, q[13], q[14]);
|
||||
do_butterfly(q[0], q[1], cospi_16_64, cospi_16_64, &q[1], &q[0]);
|
||||
store_in_output(out, 21, 26, q[1], q[0]);
|
||||
|
||||
// -----------------------------------------
|
||||
// BLOCK C: 8-10,11-15
|
||||
// -----------------------------------------
|
||||
// generate 8,9,14,15
|
||||
// part of stage 2
|
||||
load_from_transformed(trans_buf, 2, 30, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_30_64, cospi_2_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 18, 14, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_14_64, cospi_18_64, &q[1], &q[3]);
|
||||
// part of stage 3
|
||||
q[13] = highbd_idct_sub_dual(q[0], q[1]);
|
||||
q[0] = highbd_idct_add_dual(q[0], q[1]);
|
||||
q[14] = highbd_idct_sub_dual(q[2], q[3]);
|
||||
q[2] = highbd_idct_add_dual(q[2], q[3]);
|
||||
// part of stage 4
|
||||
do_butterfly(q[14], q[13], cospi_24_64, cospi_8_64, &q[1], &q[3]);
|
||||
|
||||
// generate 10,11,12,13
|
||||
// part of stage 2
|
||||
load_from_transformed(trans_buf, 10, 22, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_22_64, cospi_10_64, &q[5], &q[7]);
|
||||
load_from_transformed(trans_buf, 26, 6, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_6_64, cospi_26_64, &q[4], &q[6]);
|
||||
// part of stage 3
|
||||
q[14] = highbd_idct_sub_dual(q[4], q[5]);
|
||||
q[5] = highbd_idct_add_dual(q[4], q[5]);
|
||||
q[13] = highbd_idct_sub_dual(q[6], q[7]);
|
||||
q[6] = highbd_idct_add_dual(q[6], q[7]);
|
||||
// part of stage 4
|
||||
do_butterfly(q[14], q[13], -cospi_8_64, -cospi_24_64, &q[4], &q[7]);
|
||||
// part of stage 5
|
||||
q[8] = highbd_idct_add_dual(q[0], q[5]);
|
||||
q[9] = highbd_idct_add_dual(q[1], q[7]);
|
||||
q[13] = highbd_idct_sub_dual(q[1], q[7]);
|
||||
q[14] = highbd_idct_sub_dual(q[3], q[4]);
|
||||
q[10] = highbd_idct_add_dual(q[3], q[4]);
|
||||
q[15] = highbd_idct_add_dual(q[2], q[6]);
|
||||
store_in_output(out, 8, 15, q[8], q[15]);
|
||||
store_in_output(out, 9, 14, q[9], q[10]);
|
||||
// part of stage 6
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[1], &q[3]);
|
||||
store_in_output(out, 13, 10, q[3], q[1]);
|
||||
q[13] = highbd_idct_sub_dual(q[0], q[5]);
|
||||
q[14] = highbd_idct_sub_dual(q[2], q[6]);
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[1], &q[3]);
|
||||
store_in_output(out, 11, 12, q[1], q[3]);
|
||||
|
||||
// -----------------------------------------
|
||||
// BLOCK D: 0-3,4-7
|
||||
// -----------------------------------------
|
||||
// generate 4,5,6,7
|
||||
// part of stage 3
|
||||
load_from_transformed(trans_buf, 4, 28, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_28_64, cospi_4_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 20, 12, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_12_64, cospi_20_64, &q[1], &q[3]);
|
||||
// part of stage 4
|
||||
q[13] = highbd_idct_sub_dual(q[0], q[1]);
|
||||
q[0] = highbd_idct_add_dual(q[0], q[1]);
|
||||
q[14] = highbd_idct_sub_dual(q[2], q[3]);
|
||||
q[2] = highbd_idct_add_dual(q[2], q[3]);
|
||||
// part of stage 5
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[1], &q[3]);
|
||||
|
||||
// generate 0,1,2,3
|
||||
// part of stage 4
|
||||
load_from_transformed(trans_buf, 0, 16, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[5], &q[7]);
|
||||
load_from_transformed(trans_buf, 8, 24, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_24_64, cospi_8_64, &q[14], &q[6]);
|
||||
// part of stage 5
|
||||
q[4] = highbd_idct_add_dual(q[7], q[6]);
|
||||
q[7] = highbd_idct_sub_dual(q[7], q[6]);
|
||||
q[6] = highbd_idct_sub_dual(q[5], q[14]);
|
||||
q[5] = highbd_idct_add_dual(q[5], q[14]);
|
||||
// part of stage 6
|
||||
q[8] = highbd_idct_add_dual(q[4], q[2]);
|
||||
q[9] = highbd_idct_add_dual(q[5], q[3]);
|
||||
q[10] = highbd_idct_add_dual(q[6], q[1]);
|
||||
q[11] = highbd_idct_add_dual(q[7], q[0]);
|
||||
q[12] = highbd_idct_sub_dual(q[7], q[0]);
|
||||
q[13] = highbd_idct_sub_dual(q[6], q[1]);
|
||||
q[14] = highbd_idct_sub_dual(q[5], q[3]);
|
||||
q[15] = highbd_idct_sub_dual(q[4], q[2]);
|
||||
// part of stage 7
|
||||
load_from_output(out, 14, 15, &q[0], &q[1]);
|
||||
q[2] = highbd_idct_add_dual(q[8], q[1]);
|
||||
q[3] = highbd_idct_add_dual(q[9], q[0]);
|
||||
q[4] = highbd_idct_sub_dual(q[9], q[0]);
|
||||
q[5] = highbd_idct_sub_dual(q[8], q[1]);
|
||||
load_from_output(out, 16, 17, &q[0], &q[1]);
|
||||
q[8] = highbd_idct_add_dual(q[4], q[1]);
|
||||
q[9] = highbd_idct_add_dual(q[5], q[0]);
|
||||
q[6] = highbd_idct_sub_dual(q[5], q[0]);
|
||||
q[7] = highbd_idct_sub_dual(q[4], q[1]);
|
||||
|
||||
if (idct32_pass_loop == 0) {
|
||||
idct32_bands_end_1st_pass(out, q);
|
||||
} else {
|
||||
const int16x8_t max = vdupq_n_s16((1 << bd) - 1);
|
||||
idct32_bands_end_2nd_pass(out, dst, stride, max, q);
|
||||
dst += 8;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_highbd_idct32x32_1024_add_neon(const tran_low_t *input, uint16_t *dest,
|
||||
int stride, int bd) {
|
||||
if (bd == 8) {
|
||||
vpx_idct32_32_neon(input, CAST_TO_BYTEPTR(dest), stride, 1);
|
||||
} else {
|
||||
vpx_highbd_idct32_32_neon(input, dest, stride, bd);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,757 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/highbd_idct_neon.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
|
||||
static INLINE void load_8x8_s32_dual(
|
||||
const tran_low_t *input, int32x4x2_t *const in0, int32x4x2_t *const in1,
|
||||
int32x4x2_t *const in2, int32x4x2_t *const in3, int32x4x2_t *const in4,
|
||||
int32x4x2_t *const in5, int32x4x2_t *const in6, int32x4x2_t *const in7) {
|
||||
in0->val[0] = vld1q_s32(input);
|
||||
in0->val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in1->val[0] = vld1q_s32(input);
|
||||
in1->val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in2->val[0] = vld1q_s32(input);
|
||||
in2->val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in3->val[0] = vld1q_s32(input);
|
||||
in3->val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in4->val[0] = vld1q_s32(input);
|
||||
in4->val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in5->val[0] = vld1q_s32(input);
|
||||
in5->val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in6->val[0] = vld1q_s32(input);
|
||||
in6->val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in7->val[0] = vld1q_s32(input);
|
||||
in7->val[1] = vld1q_s32(input + 4);
|
||||
}
|
||||
|
||||
static INLINE void load_4x8_s32_dual(const tran_low_t *input,
|
||||
int32x4_t *const in0, int32x4_t *const in1,
|
||||
int32x4_t *const in2, int32x4_t *const in3,
|
||||
int32x4_t *const in4, int32x4_t *const in5,
|
||||
int32x4_t *const in6,
|
||||
int32x4_t *const in7) {
|
||||
*in0 = vld1q_s32(input);
|
||||
input += 32;
|
||||
*in1 = vld1q_s32(input);
|
||||
input += 32;
|
||||
*in2 = vld1q_s32(input);
|
||||
input += 32;
|
||||
*in3 = vld1q_s32(input);
|
||||
input += 32;
|
||||
*in4 = vld1q_s32(input);
|
||||
input += 32;
|
||||
*in5 = vld1q_s32(input);
|
||||
input += 32;
|
||||
*in6 = vld1q_s32(input);
|
||||
input += 32;
|
||||
*in7 = vld1q_s32(input);
|
||||
}
|
||||
|
||||
// Only for the first pass of the _135_ variant. Since it only uses values from
|
||||
// the top left 16x16 it can safely assume all the remaining values are 0 and
|
||||
// skip an awful lot of calculations. In fact, only the first 12 columns make
|
||||
// the cut. None of the elements in the 13th, 14th, 15th or 16th columns are
|
||||
// used so it skips any calls to input[12|13|14|15] too.
|
||||
// In C this does a single row of 32 for each call. Here it transposes the top
|
||||
// left 12x8 to allow using SIMD.
|
||||
|
||||
// vp9/common/vp9_scan.c:vp9_default_iscan_32x32 arranges the first 135 non-zero
|
||||
// coefficients as follows:
|
||||
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
|
||||
// 0 0 2 5 10 17 25 38 47 62 83 101 121
|
||||
// 1 1 4 8 15 22 30 45 58 74 92 112 133
|
||||
// 2 3 7 12 18 28 36 52 64 82 102 118
|
||||
// 3 6 11 16 23 31 43 60 73 90 109 126
|
||||
// 4 9 14 19 29 37 50 65 78 98 116 134
|
||||
// 5 13 20 26 35 44 54 72 85 105 123
|
||||
// 6 21 27 33 42 53 63 80 94 113 132
|
||||
// 7 24 32 39 48 57 71 88 104 120
|
||||
// 8 34 40 46 56 68 81 96 111 130
|
||||
// 9 41 49 55 67 77 91 107 124
|
||||
// 10 51 59 66 76 89 99 119 131
|
||||
// 11 61 69 75 87 100 114 129
|
||||
// 12 70 79 86 97 108 122
|
||||
// 13 84 93 103 110 125
|
||||
// 14 98 106 115 127
|
||||
// 15 117 128
|
||||
static void vpx_highbd_idct32_12_neon(const tran_low_t *const input,
|
||||
int32_t *output) {
|
||||
int32x4x2_t in[12], s1[32], s2[32], s3[32], s4[32], s5[32], s6[32], s7[32],
|
||||
s8[32];
|
||||
|
||||
load_8x8_s32_dual(input, &in[0], &in[1], &in[2], &in[3], &in[4], &in[5],
|
||||
&in[6], &in[7]);
|
||||
transpose_s32_8x8(&in[0], &in[1], &in[2], &in[3], &in[4], &in[5], &in[6],
|
||||
&in[7]);
|
||||
|
||||
load_4x8_s32_dual(input + 8, &in[8].val[0], &in[8].val[1], &in[9].val[0],
|
||||
&in[9].val[1], &in[10].val[0], &in[10].val[1],
|
||||
&in[11].val[0], &in[11].val[1]);
|
||||
transpose_s32_4x8(&in[8].val[0], &in[8].val[1], &in[9].val[0], &in[9].val[1],
|
||||
&in[10].val[0], &in[10].val[1], &in[11].val[0],
|
||||
&in[11].val[1]);
|
||||
|
||||
// stage 1
|
||||
s1[16] = multiply_shift_and_narrow_s32_dual(in[1], cospi_31_64);
|
||||
s1[31] = multiply_shift_and_narrow_s32_dual(in[1], cospi_1_64);
|
||||
|
||||
s1[18] = multiply_shift_and_narrow_s32_dual(in[9], cospi_23_64);
|
||||
s1[29] = multiply_shift_and_narrow_s32_dual(in[9], cospi_9_64);
|
||||
|
||||
s1[19] = multiply_shift_and_narrow_s32_dual(in[7], -cospi_25_64);
|
||||
s1[28] = multiply_shift_and_narrow_s32_dual(in[7], cospi_7_64);
|
||||
|
||||
s1[20] = multiply_shift_and_narrow_s32_dual(in[5], cospi_27_64);
|
||||
s1[27] = multiply_shift_and_narrow_s32_dual(in[5], cospi_5_64);
|
||||
|
||||
s1[21] = multiply_shift_and_narrow_s32_dual(in[11], -cospi_21_64);
|
||||
s1[26] = multiply_shift_and_narrow_s32_dual(in[11], cospi_11_64);
|
||||
|
||||
s1[23] = multiply_shift_and_narrow_s32_dual(in[3], -cospi_29_64);
|
||||
s1[24] = multiply_shift_and_narrow_s32_dual(in[3], cospi_3_64);
|
||||
|
||||
// stage 2
|
||||
s2[8] = multiply_shift_and_narrow_s32_dual(in[2], cospi_30_64);
|
||||
s2[15] = multiply_shift_and_narrow_s32_dual(in[2], cospi_2_64);
|
||||
|
||||
s2[10] = multiply_shift_and_narrow_s32_dual(in[10], cospi_22_64);
|
||||
s2[13] = multiply_shift_and_narrow_s32_dual(in[10], cospi_10_64);
|
||||
|
||||
s2[11] = multiply_shift_and_narrow_s32_dual(in[6], -cospi_26_64);
|
||||
s2[12] = multiply_shift_and_narrow_s32_dual(in[6], cospi_6_64);
|
||||
|
||||
s2[18] = highbd_idct_sub_dual(s1[19], s1[18]);
|
||||
s2[19] = highbd_idct_add_dual(s1[18], s1[19]);
|
||||
s2[20] = highbd_idct_add_dual(s1[20], s1[21]);
|
||||
s2[21] = highbd_idct_sub_dual(s1[20], s1[21]);
|
||||
s2[26] = highbd_idct_sub_dual(s1[27], s1[26]);
|
||||
s2[27] = highbd_idct_add_dual(s1[26], s1[27]);
|
||||
s2[28] = highbd_idct_add_dual(s1[28], s1[29]);
|
||||
s2[29] = highbd_idct_sub_dual(s1[28], s1[29]);
|
||||
|
||||
// stage 3
|
||||
s3[4] = multiply_shift_and_narrow_s32_dual(in[4], cospi_28_64);
|
||||
s3[7] = multiply_shift_and_narrow_s32_dual(in[4], cospi_4_64);
|
||||
|
||||
s3[10] = highbd_idct_sub_dual(s2[11], s2[10]);
|
||||
s3[11] = highbd_idct_add_dual(s2[10], s2[11]);
|
||||
s3[12] = highbd_idct_add_dual(s2[12], s2[13]);
|
||||
s3[13] = highbd_idct_sub_dual(s2[12], s2[13]);
|
||||
|
||||
s3[17] = multiply_accumulate_shift_and_narrow_s32_dual(s1[16], -cospi_4_64,
|
||||
s1[31], cospi_28_64);
|
||||
s3[30] = multiply_accumulate_shift_and_narrow_s32_dual(s1[16], cospi_28_64,
|
||||
s1[31], cospi_4_64);
|
||||
|
||||
s3[18] = multiply_accumulate_shift_and_narrow_s32_dual(s2[18], -cospi_28_64,
|
||||
s2[29], -cospi_4_64);
|
||||
s3[29] = multiply_accumulate_shift_and_narrow_s32_dual(s2[18], -cospi_4_64,
|
||||
s2[29], cospi_28_64);
|
||||
|
||||
s3[21] = multiply_accumulate_shift_and_narrow_s32_dual(s2[21], -cospi_20_64,
|
||||
s2[26], cospi_12_64);
|
||||
s3[26] = multiply_accumulate_shift_and_narrow_s32_dual(s2[21], cospi_12_64,
|
||||
s2[26], cospi_20_64);
|
||||
|
||||
s3[22] = multiply_accumulate_shift_and_narrow_s32_dual(s1[23], -cospi_12_64,
|
||||
s1[24], -cospi_20_64);
|
||||
s3[25] = multiply_accumulate_shift_and_narrow_s32_dual(s1[23], -cospi_20_64,
|
||||
s1[24], cospi_12_64);
|
||||
|
||||
// stage 4
|
||||
s4[0] = multiply_shift_and_narrow_s32_dual(in[0], cospi_16_64);
|
||||
s4[2] = multiply_shift_and_narrow_s32_dual(in[8], cospi_24_64);
|
||||
s4[3] = multiply_shift_and_narrow_s32_dual(in[8], cospi_8_64);
|
||||
|
||||
s4[9] = multiply_accumulate_shift_and_narrow_s32_dual(s2[8], -cospi_8_64,
|
||||
s2[15], cospi_24_64);
|
||||
s4[14] = multiply_accumulate_shift_and_narrow_s32_dual(s2[8], cospi_24_64,
|
||||
s2[15], cospi_8_64);
|
||||
|
||||
s4[10] = multiply_accumulate_shift_and_narrow_s32_dual(s3[10], -cospi_24_64,
|
||||
s3[13], -cospi_8_64);
|
||||
s4[13] = multiply_accumulate_shift_and_narrow_s32_dual(s3[10], -cospi_8_64,
|
||||
s3[13], cospi_24_64);
|
||||
|
||||
s4[16] = highbd_idct_add_dual(s1[16], s2[19]);
|
||||
s4[17] = highbd_idct_add_dual(s3[17], s3[18]);
|
||||
s4[18] = highbd_idct_sub_dual(s3[17], s3[18]);
|
||||
s4[19] = highbd_idct_sub_dual(s1[16], s2[19]);
|
||||
s4[20] = highbd_idct_sub_dual(s1[23], s2[20]);
|
||||
s4[21] = highbd_idct_sub_dual(s3[22], s3[21]);
|
||||
s4[22] = highbd_idct_add_dual(s3[21], s3[22]);
|
||||
s4[23] = highbd_idct_add_dual(s2[20], s1[23]);
|
||||
s4[24] = highbd_idct_add_dual(s1[24], s2[27]);
|
||||
s4[25] = highbd_idct_add_dual(s3[25], s3[26]);
|
||||
s4[26] = highbd_idct_sub_dual(s3[25], s3[26]);
|
||||
s4[27] = highbd_idct_sub_dual(s1[24], s2[27]);
|
||||
s4[28] = highbd_idct_sub_dual(s1[31], s2[28]);
|
||||
s4[29] = highbd_idct_sub_dual(s3[30], s3[29]);
|
||||
s4[30] = highbd_idct_add_dual(s3[29], s3[30]);
|
||||
s4[31] = highbd_idct_add_dual(s2[28], s1[31]);
|
||||
|
||||
// stage 5
|
||||
s5[0] = highbd_idct_add_dual(s4[0], s4[3]);
|
||||
s5[1] = highbd_idct_add_dual(s4[0], s4[2]);
|
||||
s5[2] = highbd_idct_sub_dual(s4[0], s4[2]);
|
||||
s5[3] = highbd_idct_sub_dual(s4[0], s4[3]);
|
||||
|
||||
s5[5] = sub_multiply_shift_and_narrow_s32_dual(s3[7], s3[4], cospi_16_64);
|
||||
s5[6] = add_multiply_shift_and_narrow_s32_dual(s3[4], s3[7], cospi_16_64);
|
||||
|
||||
s5[8] = highbd_idct_add_dual(s2[8], s3[11]);
|
||||
s5[9] = highbd_idct_add_dual(s4[9], s4[10]);
|
||||
s5[10] = highbd_idct_sub_dual(s4[9], s4[10]);
|
||||
s5[11] = highbd_idct_sub_dual(s2[8], s3[11]);
|
||||
s5[12] = highbd_idct_sub_dual(s2[15], s3[12]);
|
||||
s5[13] = highbd_idct_sub_dual(s4[14], s4[13]);
|
||||
s5[14] = highbd_idct_add_dual(s4[13], s4[14]);
|
||||
s5[15] = highbd_idct_add_dual(s2[15], s3[12]);
|
||||
|
||||
s5[18] = multiply_accumulate_shift_and_narrow_s32_dual(s4[18], -cospi_8_64,
|
||||
s4[29], cospi_24_64);
|
||||
s5[29] = multiply_accumulate_shift_and_narrow_s32_dual(s4[18], cospi_24_64,
|
||||
s4[29], cospi_8_64);
|
||||
|
||||
s5[19] = multiply_accumulate_shift_and_narrow_s32_dual(s4[19], -cospi_8_64,
|
||||
s4[28], cospi_24_64);
|
||||
s5[28] = multiply_accumulate_shift_and_narrow_s32_dual(s4[19], cospi_24_64,
|
||||
s4[28], cospi_8_64);
|
||||
|
||||
s5[20] = multiply_accumulate_shift_and_narrow_s32_dual(s4[20], -cospi_24_64,
|
||||
s4[27], -cospi_8_64);
|
||||
s5[27] = multiply_accumulate_shift_and_narrow_s32_dual(s4[20], -cospi_8_64,
|
||||
s4[27], cospi_24_64);
|
||||
|
||||
s5[21] = multiply_accumulate_shift_and_narrow_s32_dual(s4[21], -cospi_24_64,
|
||||
s4[26], -cospi_8_64);
|
||||
s5[26] = multiply_accumulate_shift_and_narrow_s32_dual(s4[21], -cospi_8_64,
|
||||
s4[26], cospi_24_64);
|
||||
|
||||
// stage 6
|
||||
s6[0] = highbd_idct_add_dual(s5[0], s3[7]);
|
||||
s6[1] = highbd_idct_add_dual(s5[1], s5[6]);
|
||||
s6[2] = highbd_idct_add_dual(s5[2], s5[5]);
|
||||
s6[3] = highbd_idct_add_dual(s5[3], s3[4]);
|
||||
s6[4] = highbd_idct_sub_dual(s5[3], s3[4]);
|
||||
s6[5] = highbd_idct_sub_dual(s5[2], s5[5]);
|
||||
s6[6] = highbd_idct_sub_dual(s5[1], s5[6]);
|
||||
s6[7] = highbd_idct_sub_dual(s5[0], s3[7]);
|
||||
|
||||
s6[10] = sub_multiply_shift_and_narrow_s32_dual(s5[13], s5[10], cospi_16_64);
|
||||
s6[13] = add_multiply_shift_and_narrow_s32_dual(s5[10], s5[13], cospi_16_64);
|
||||
|
||||
s6[11] = sub_multiply_shift_and_narrow_s32_dual(s5[12], s5[11], cospi_16_64);
|
||||
s6[12] = add_multiply_shift_and_narrow_s32_dual(s5[11], s5[12], cospi_16_64);
|
||||
|
||||
s6[16] = highbd_idct_add_dual(s4[16], s4[23]);
|
||||
s6[17] = highbd_idct_add_dual(s4[17], s4[22]);
|
||||
s6[18] = highbd_idct_add_dual(s5[18], s5[21]);
|
||||
s6[19] = highbd_idct_add_dual(s5[19], s5[20]);
|
||||
s6[20] = highbd_idct_sub_dual(s5[19], s5[20]);
|
||||
s6[21] = highbd_idct_sub_dual(s5[18], s5[21]);
|
||||
s6[22] = highbd_idct_sub_dual(s4[17], s4[22]);
|
||||
s6[23] = highbd_idct_sub_dual(s4[16], s4[23]);
|
||||
|
||||
s6[24] = highbd_idct_sub_dual(s4[31], s4[24]);
|
||||
s6[25] = highbd_idct_sub_dual(s4[30], s4[25]);
|
||||
s6[26] = highbd_idct_sub_dual(s5[29], s5[26]);
|
||||
s6[27] = highbd_idct_sub_dual(s5[28], s5[27]);
|
||||
s6[28] = highbd_idct_add_dual(s5[27], s5[28]);
|
||||
s6[29] = highbd_idct_add_dual(s5[26], s5[29]);
|
||||
s6[30] = highbd_idct_add_dual(s4[25], s4[30]);
|
||||
s6[31] = highbd_idct_add_dual(s4[24], s4[31]);
|
||||
|
||||
// stage 7
|
||||
s7[0] = highbd_idct_add_dual(s6[0], s5[15]);
|
||||
s7[1] = highbd_idct_add_dual(s6[1], s5[14]);
|
||||
s7[2] = highbd_idct_add_dual(s6[2], s6[13]);
|
||||
s7[3] = highbd_idct_add_dual(s6[3], s6[12]);
|
||||
s7[4] = highbd_idct_add_dual(s6[4], s6[11]);
|
||||
s7[5] = highbd_idct_add_dual(s6[5], s6[10]);
|
||||
s7[6] = highbd_idct_add_dual(s6[6], s5[9]);
|
||||
s7[7] = highbd_idct_add_dual(s6[7], s5[8]);
|
||||
s7[8] = highbd_idct_sub_dual(s6[7], s5[8]);
|
||||
s7[9] = highbd_idct_sub_dual(s6[6], s5[9]);
|
||||
s7[10] = highbd_idct_sub_dual(s6[5], s6[10]);
|
||||
s7[11] = highbd_idct_sub_dual(s6[4], s6[11]);
|
||||
s7[12] = highbd_idct_sub_dual(s6[3], s6[12]);
|
||||
s7[13] = highbd_idct_sub_dual(s6[2], s6[13]);
|
||||
s7[14] = highbd_idct_sub_dual(s6[1], s5[14]);
|
||||
s7[15] = highbd_idct_sub_dual(s6[0], s5[15]);
|
||||
|
||||
s7[20] = sub_multiply_shift_and_narrow_s32_dual(s6[27], s6[20], cospi_16_64);
|
||||
s7[27] = add_multiply_shift_and_narrow_s32_dual(s6[20], s6[27], cospi_16_64);
|
||||
|
||||
s7[21] = sub_multiply_shift_and_narrow_s32_dual(s6[26], s6[21], cospi_16_64);
|
||||
s7[26] = add_multiply_shift_and_narrow_s32_dual(s6[21], s6[26], cospi_16_64);
|
||||
|
||||
s7[22] = sub_multiply_shift_and_narrow_s32_dual(s6[25], s6[22], cospi_16_64);
|
||||
s7[25] = add_multiply_shift_and_narrow_s32_dual(s6[22], s6[25], cospi_16_64);
|
||||
|
||||
s7[23] = sub_multiply_shift_and_narrow_s32_dual(s6[24], s6[23], cospi_16_64);
|
||||
s7[24] = add_multiply_shift_and_narrow_s32_dual(s6[23], s6[24], cospi_16_64);
|
||||
|
||||
// final stage
|
||||
s8[0] = highbd_idct_add_dual(s7[0], s6[31]);
|
||||
s8[1] = highbd_idct_add_dual(s7[1], s6[30]);
|
||||
s8[2] = highbd_idct_add_dual(s7[2], s6[29]);
|
||||
s8[3] = highbd_idct_add_dual(s7[3], s6[28]);
|
||||
s8[4] = highbd_idct_add_dual(s7[4], s7[27]);
|
||||
s8[5] = highbd_idct_add_dual(s7[5], s7[26]);
|
||||
s8[6] = highbd_idct_add_dual(s7[6], s7[25]);
|
||||
s8[7] = highbd_idct_add_dual(s7[7], s7[24]);
|
||||
s8[8] = highbd_idct_add_dual(s7[8], s7[23]);
|
||||
s8[9] = highbd_idct_add_dual(s7[9], s7[22]);
|
||||
s8[10] = highbd_idct_add_dual(s7[10], s7[21]);
|
||||
s8[11] = highbd_idct_add_dual(s7[11], s7[20]);
|
||||
s8[12] = highbd_idct_add_dual(s7[12], s6[19]);
|
||||
s8[13] = highbd_idct_add_dual(s7[13], s6[18]);
|
||||
s8[14] = highbd_idct_add_dual(s7[14], s6[17]);
|
||||
s8[15] = highbd_idct_add_dual(s7[15], s6[16]);
|
||||
s8[16] = highbd_idct_sub_dual(s7[15], s6[16]);
|
||||
s8[17] = highbd_idct_sub_dual(s7[14], s6[17]);
|
||||
s8[18] = highbd_idct_sub_dual(s7[13], s6[18]);
|
||||
s8[19] = highbd_idct_sub_dual(s7[12], s6[19]);
|
||||
s8[20] = highbd_idct_sub_dual(s7[11], s7[20]);
|
||||
s8[21] = highbd_idct_sub_dual(s7[10], s7[21]);
|
||||
s8[22] = highbd_idct_sub_dual(s7[9], s7[22]);
|
||||
s8[23] = highbd_idct_sub_dual(s7[8], s7[23]);
|
||||
s8[24] = highbd_idct_sub_dual(s7[7], s7[24]);
|
||||
s8[25] = highbd_idct_sub_dual(s7[6], s7[25]);
|
||||
s8[26] = highbd_idct_sub_dual(s7[5], s7[26]);
|
||||
s8[27] = highbd_idct_sub_dual(s7[4], s7[27]);
|
||||
s8[28] = highbd_idct_sub_dual(s7[3], s6[28]);
|
||||
s8[29] = highbd_idct_sub_dual(s7[2], s6[29]);
|
||||
s8[30] = highbd_idct_sub_dual(s7[1], s6[30]);
|
||||
s8[31] = highbd_idct_sub_dual(s7[0], s6[31]);
|
||||
|
||||
vst1q_s32(output + 0, s8[0].val[0]);
|
||||
vst1q_s32(output + 4, s8[0].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[1].val[0]);
|
||||
vst1q_s32(output + 4, s8[1].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[2].val[0]);
|
||||
vst1q_s32(output + 4, s8[2].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[3].val[0]);
|
||||
vst1q_s32(output + 4, s8[3].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[4].val[0]);
|
||||
vst1q_s32(output + 4, s8[4].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[5].val[0]);
|
||||
vst1q_s32(output + 4, s8[5].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[6].val[0]);
|
||||
vst1q_s32(output + 4, s8[6].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[7].val[0]);
|
||||
vst1q_s32(output + 4, s8[7].val[1]);
|
||||
output += 16;
|
||||
|
||||
vst1q_s32(output + 0, s8[8].val[0]);
|
||||
vst1q_s32(output + 4, s8[8].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[9].val[0]);
|
||||
vst1q_s32(output + 4, s8[9].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[10].val[0]);
|
||||
vst1q_s32(output + 4, s8[10].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[11].val[0]);
|
||||
vst1q_s32(output + 4, s8[11].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[12].val[0]);
|
||||
vst1q_s32(output + 4, s8[12].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[13].val[0]);
|
||||
vst1q_s32(output + 4, s8[13].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[14].val[0]);
|
||||
vst1q_s32(output + 4, s8[14].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[15].val[0]);
|
||||
vst1q_s32(output + 4, s8[15].val[1]);
|
||||
output += 16;
|
||||
|
||||
vst1q_s32(output + 0, s8[16].val[0]);
|
||||
vst1q_s32(output + 4, s8[16].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[17].val[0]);
|
||||
vst1q_s32(output + 4, s8[17].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[18].val[0]);
|
||||
vst1q_s32(output + 4, s8[18].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[19].val[0]);
|
||||
vst1q_s32(output + 4, s8[19].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[20].val[0]);
|
||||
vst1q_s32(output + 4, s8[20].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[21].val[0]);
|
||||
vst1q_s32(output + 4, s8[21].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[22].val[0]);
|
||||
vst1q_s32(output + 4, s8[22].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[23].val[0]);
|
||||
vst1q_s32(output + 4, s8[23].val[1]);
|
||||
output += 16;
|
||||
|
||||
vst1q_s32(output + 0, s8[24].val[0]);
|
||||
vst1q_s32(output + 4, s8[24].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[25].val[0]);
|
||||
vst1q_s32(output + 4, s8[25].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[26].val[0]);
|
||||
vst1q_s32(output + 4, s8[26].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[27].val[0]);
|
||||
vst1q_s32(output + 4, s8[27].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[28].val[0]);
|
||||
vst1q_s32(output + 4, s8[28].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[29].val[0]);
|
||||
vst1q_s32(output + 4, s8[29].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[30].val[0]);
|
||||
vst1q_s32(output + 4, s8[30].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, s8[31].val[0]);
|
||||
vst1q_s32(output + 4, s8[31].val[1]);
|
||||
}
|
||||
|
||||
static void vpx_highbd_idct32_16_neon(const int32_t *const input,
|
||||
uint16_t *const output, const int stride,
|
||||
const int bd) {
|
||||
int32x4x2_t in[16], s1[32], s2[32], s3[32], s4[32], s5[32], s6[32], s7[32],
|
||||
out[32];
|
||||
|
||||
load_and_transpose_s32_8x8(input, 16, &in[0], &in[1], &in[2], &in[3], &in[4],
|
||||
&in[5], &in[6], &in[7]);
|
||||
|
||||
load_and_transpose_s32_8x8(input + 8, 16, &in[8], &in[9], &in[10], &in[11],
|
||||
&in[12], &in[13], &in[14], &in[15]);
|
||||
|
||||
// stage 1
|
||||
s1[16] = multiply_shift_and_narrow_s32_dual(in[1], cospi_31_64);
|
||||
s1[31] = multiply_shift_and_narrow_s32_dual(in[1], cospi_1_64);
|
||||
|
||||
s1[17] = multiply_shift_and_narrow_s32_dual(in[15], -cospi_17_64);
|
||||
s1[30] = multiply_shift_and_narrow_s32_dual(in[15], cospi_15_64);
|
||||
|
||||
s1[18] = multiply_shift_and_narrow_s32_dual(in[9], cospi_23_64);
|
||||
s1[29] = multiply_shift_and_narrow_s32_dual(in[9], cospi_9_64);
|
||||
|
||||
s1[19] = multiply_shift_and_narrow_s32_dual(in[7], -cospi_25_64);
|
||||
s1[28] = multiply_shift_and_narrow_s32_dual(in[7], cospi_7_64);
|
||||
|
||||
s1[20] = multiply_shift_and_narrow_s32_dual(in[5], cospi_27_64);
|
||||
s1[27] = multiply_shift_and_narrow_s32_dual(in[5], cospi_5_64);
|
||||
|
||||
s1[21] = multiply_shift_and_narrow_s32_dual(in[11], -cospi_21_64);
|
||||
s1[26] = multiply_shift_and_narrow_s32_dual(in[11], cospi_11_64);
|
||||
|
||||
s1[22] = multiply_shift_and_narrow_s32_dual(in[13], cospi_19_64);
|
||||
s1[25] = multiply_shift_and_narrow_s32_dual(in[13], cospi_13_64);
|
||||
|
||||
s1[23] = multiply_shift_and_narrow_s32_dual(in[3], -cospi_29_64);
|
||||
s1[24] = multiply_shift_and_narrow_s32_dual(in[3], cospi_3_64);
|
||||
|
||||
// stage 2
|
||||
s2[8] = multiply_shift_and_narrow_s32_dual(in[2], cospi_30_64);
|
||||
s2[15] = multiply_shift_and_narrow_s32_dual(in[2], cospi_2_64);
|
||||
|
||||
s2[9] = multiply_shift_and_narrow_s32_dual(in[14], -cospi_18_64);
|
||||
s2[14] = multiply_shift_and_narrow_s32_dual(in[14], cospi_14_64);
|
||||
|
||||
s2[10] = multiply_shift_and_narrow_s32_dual(in[10], cospi_22_64);
|
||||
s2[13] = multiply_shift_and_narrow_s32_dual(in[10], cospi_10_64);
|
||||
|
||||
s2[11] = multiply_shift_and_narrow_s32_dual(in[6], -cospi_26_64);
|
||||
s2[12] = multiply_shift_and_narrow_s32_dual(in[6], cospi_6_64);
|
||||
|
||||
s2[16] = highbd_idct_add_dual(s1[16], s1[17]);
|
||||
s2[17] = highbd_idct_sub_dual(s1[16], s1[17]);
|
||||
s2[18] = highbd_idct_sub_dual(s1[19], s1[18]);
|
||||
s2[19] = highbd_idct_add_dual(s1[18], s1[19]);
|
||||
s2[20] = highbd_idct_add_dual(s1[20], s1[21]);
|
||||
s2[21] = highbd_idct_sub_dual(s1[20], s1[21]);
|
||||
s2[22] = highbd_idct_sub_dual(s1[23], s1[22]);
|
||||
s2[23] = highbd_idct_add_dual(s1[22], s1[23]);
|
||||
s2[24] = highbd_idct_add_dual(s1[24], s1[25]);
|
||||
s2[25] = highbd_idct_sub_dual(s1[24], s1[25]);
|
||||
s2[26] = highbd_idct_sub_dual(s1[27], s1[26]);
|
||||
s2[27] = highbd_idct_add_dual(s1[26], s1[27]);
|
||||
s2[28] = highbd_idct_add_dual(s1[28], s1[29]);
|
||||
s2[29] = highbd_idct_sub_dual(s1[28], s1[29]);
|
||||
s2[30] = highbd_idct_sub_dual(s1[31], s1[30]);
|
||||
s2[31] = highbd_idct_add_dual(s1[30], s1[31]);
|
||||
|
||||
// stage 3
|
||||
s3[4] = multiply_shift_and_narrow_s32_dual(in[4], cospi_28_64);
|
||||
s3[7] = multiply_shift_and_narrow_s32_dual(in[4], cospi_4_64);
|
||||
|
||||
s3[5] = multiply_shift_and_narrow_s32_dual(in[12], -cospi_20_64);
|
||||
s3[6] = multiply_shift_and_narrow_s32_dual(in[12], cospi_12_64);
|
||||
|
||||
s3[8] = highbd_idct_add_dual(s2[8], s2[9]);
|
||||
s3[9] = highbd_idct_sub_dual(s2[8], s2[9]);
|
||||
s3[10] = highbd_idct_sub_dual(s2[11], s2[10]);
|
||||
s3[11] = highbd_idct_add_dual(s2[10], s2[11]);
|
||||
s3[12] = highbd_idct_add_dual(s2[12], s2[13]);
|
||||
s3[13] = highbd_idct_sub_dual(s2[12], s2[13]);
|
||||
s3[14] = highbd_idct_sub_dual(s2[15], s2[14]);
|
||||
s3[15] = highbd_idct_add_dual(s2[14], s2[15]);
|
||||
|
||||
s3[17] = multiply_accumulate_shift_and_narrow_s32_dual(s2[17], -cospi_4_64,
|
||||
s2[30], cospi_28_64);
|
||||
s3[30] = multiply_accumulate_shift_and_narrow_s32_dual(s2[17], cospi_28_64,
|
||||
s2[30], cospi_4_64);
|
||||
|
||||
s3[18] = multiply_accumulate_shift_and_narrow_s32_dual(s2[18], -cospi_28_64,
|
||||
s2[29], -cospi_4_64);
|
||||
s3[29] = multiply_accumulate_shift_and_narrow_s32_dual(s2[18], -cospi_4_64,
|
||||
s2[29], cospi_28_64);
|
||||
|
||||
s3[21] = multiply_accumulate_shift_and_narrow_s32_dual(s2[21], -cospi_20_64,
|
||||
s2[26], cospi_12_64);
|
||||
s3[26] = multiply_accumulate_shift_and_narrow_s32_dual(s2[21], cospi_12_64,
|
||||
s2[26], cospi_20_64);
|
||||
|
||||
s3[22] = multiply_accumulate_shift_and_narrow_s32_dual(s2[22], -cospi_12_64,
|
||||
s2[25], -cospi_20_64);
|
||||
s3[25] = multiply_accumulate_shift_and_narrow_s32_dual(s2[22], -cospi_20_64,
|
||||
s2[25], cospi_12_64);
|
||||
|
||||
// stage 4
|
||||
s4[0] = multiply_shift_and_narrow_s32_dual(in[0], cospi_16_64);
|
||||
s4[2] = multiply_shift_and_narrow_s32_dual(in[8], cospi_24_64);
|
||||
s4[3] = multiply_shift_and_narrow_s32_dual(in[8], cospi_8_64);
|
||||
|
||||
s4[4] = highbd_idct_add_dual(s3[4], s3[5]);
|
||||
s4[5] = highbd_idct_sub_dual(s3[4], s3[5]);
|
||||
s4[6] = highbd_idct_sub_dual(s3[7], s3[6]);
|
||||
s4[7] = highbd_idct_add_dual(s3[6], s3[7]);
|
||||
|
||||
s4[9] = multiply_accumulate_shift_and_narrow_s32_dual(s3[9], -cospi_8_64,
|
||||
s3[14], cospi_24_64);
|
||||
s4[14] = multiply_accumulate_shift_and_narrow_s32_dual(s3[9], cospi_24_64,
|
||||
s3[14], cospi_8_64);
|
||||
|
||||
s4[10] = multiply_accumulate_shift_and_narrow_s32_dual(s3[10], -cospi_24_64,
|
||||
s3[13], -cospi_8_64);
|
||||
s4[13] = multiply_accumulate_shift_and_narrow_s32_dual(s3[10], -cospi_8_64,
|
||||
s3[13], cospi_24_64);
|
||||
|
||||
s4[16] = highbd_idct_add_dual(s2[16], s2[19]);
|
||||
s4[17] = highbd_idct_add_dual(s3[17], s3[18]);
|
||||
s4[18] = highbd_idct_sub_dual(s3[17], s3[18]);
|
||||
s4[19] = highbd_idct_sub_dual(s2[16], s2[19]);
|
||||
s4[20] = highbd_idct_sub_dual(s2[23], s2[20]);
|
||||
s4[21] = highbd_idct_sub_dual(s3[22], s3[21]);
|
||||
s4[22] = highbd_idct_add_dual(s3[21], s3[22]);
|
||||
s4[23] = highbd_idct_add_dual(s2[20], s2[23]);
|
||||
s4[24] = highbd_idct_add_dual(s2[24], s2[27]);
|
||||
s4[25] = highbd_idct_add_dual(s3[25], s3[26]);
|
||||
s4[26] = highbd_idct_sub_dual(s3[25], s3[26]);
|
||||
s4[27] = highbd_idct_sub_dual(s2[24], s2[27]);
|
||||
s4[28] = highbd_idct_sub_dual(s2[31], s2[28]);
|
||||
s4[29] = highbd_idct_sub_dual(s3[30], s3[29]);
|
||||
s4[30] = highbd_idct_add_dual(s3[29], s3[30]);
|
||||
s4[31] = highbd_idct_add_dual(s2[28], s2[31]);
|
||||
|
||||
// stage 5
|
||||
s5[0] = highbd_idct_add_dual(s4[0], s4[3]);
|
||||
s5[1] = highbd_idct_add_dual(s4[0], s4[2]);
|
||||
s5[2] = highbd_idct_sub_dual(s4[0], s4[2]);
|
||||
s5[3] = highbd_idct_sub_dual(s4[0], s4[3]);
|
||||
|
||||
s5[5] = sub_multiply_shift_and_narrow_s32_dual(s4[6], s4[5], cospi_16_64);
|
||||
s5[6] = add_multiply_shift_and_narrow_s32_dual(s4[5], s4[6], cospi_16_64);
|
||||
|
||||
s5[8] = highbd_idct_add_dual(s3[8], s3[11]);
|
||||
s5[9] = highbd_idct_add_dual(s4[9], s4[10]);
|
||||
s5[10] = highbd_idct_sub_dual(s4[9], s4[10]);
|
||||
s5[11] = highbd_idct_sub_dual(s3[8], s3[11]);
|
||||
s5[12] = highbd_idct_sub_dual(s3[15], s3[12]);
|
||||
s5[13] = highbd_idct_sub_dual(s4[14], s4[13]);
|
||||
s5[14] = highbd_idct_add_dual(s4[13], s4[14]);
|
||||
s5[15] = highbd_idct_add_dual(s3[15], s3[12]);
|
||||
|
||||
s5[18] = multiply_accumulate_shift_and_narrow_s32_dual(s4[18], -cospi_8_64,
|
||||
s4[29], cospi_24_64);
|
||||
s5[29] = multiply_accumulate_shift_and_narrow_s32_dual(s4[18], cospi_24_64,
|
||||
s4[29], cospi_8_64);
|
||||
|
||||
s5[19] = multiply_accumulate_shift_and_narrow_s32_dual(s4[19], -cospi_8_64,
|
||||
s4[28], cospi_24_64);
|
||||
s5[28] = multiply_accumulate_shift_and_narrow_s32_dual(s4[19], cospi_24_64,
|
||||
s4[28], cospi_8_64);
|
||||
|
||||
s5[20] = multiply_accumulate_shift_and_narrow_s32_dual(s4[20], -cospi_24_64,
|
||||
s4[27], -cospi_8_64);
|
||||
s5[27] = multiply_accumulate_shift_and_narrow_s32_dual(s4[20], -cospi_8_64,
|
||||
s4[27], cospi_24_64);
|
||||
|
||||
s5[21] = multiply_accumulate_shift_and_narrow_s32_dual(s4[21], -cospi_24_64,
|
||||
s4[26], -cospi_8_64);
|
||||
s5[26] = multiply_accumulate_shift_and_narrow_s32_dual(s4[21], -cospi_8_64,
|
||||
s4[26], cospi_24_64);
|
||||
|
||||
// stage 6
|
||||
s6[0] = highbd_idct_add_dual(s5[0], s4[7]);
|
||||
s6[1] = highbd_idct_add_dual(s5[1], s5[6]);
|
||||
s6[2] = highbd_idct_add_dual(s5[2], s5[5]);
|
||||
s6[3] = highbd_idct_add_dual(s5[3], s4[4]);
|
||||
s6[4] = highbd_idct_sub_dual(s5[3], s4[4]);
|
||||
s6[5] = highbd_idct_sub_dual(s5[2], s5[5]);
|
||||
s6[6] = highbd_idct_sub_dual(s5[1], s5[6]);
|
||||
s6[7] = highbd_idct_sub_dual(s5[0], s4[7]);
|
||||
|
||||
s6[10] = sub_multiply_shift_and_narrow_s32_dual(s5[13], s5[10], cospi_16_64);
|
||||
s6[13] = add_multiply_shift_and_narrow_s32_dual(s5[10], s5[13], cospi_16_64);
|
||||
|
||||
s6[11] = sub_multiply_shift_and_narrow_s32_dual(s5[12], s5[11], cospi_16_64);
|
||||
s6[12] = add_multiply_shift_and_narrow_s32_dual(s5[11], s5[12], cospi_16_64);
|
||||
|
||||
s6[16] = highbd_idct_add_dual(s4[16], s4[23]);
|
||||
s6[17] = highbd_idct_add_dual(s4[17], s4[22]);
|
||||
s6[18] = highbd_idct_add_dual(s5[18], s5[21]);
|
||||
s6[19] = highbd_idct_add_dual(s5[19], s5[20]);
|
||||
s6[20] = highbd_idct_sub_dual(s5[19], s5[20]);
|
||||
s6[21] = highbd_idct_sub_dual(s5[18], s5[21]);
|
||||
s6[22] = highbd_idct_sub_dual(s4[17], s4[22]);
|
||||
s6[23] = highbd_idct_sub_dual(s4[16], s4[23]);
|
||||
s6[24] = highbd_idct_sub_dual(s4[31], s4[24]);
|
||||
s6[25] = highbd_idct_sub_dual(s4[30], s4[25]);
|
||||
s6[26] = highbd_idct_sub_dual(s5[29], s5[26]);
|
||||
s6[27] = highbd_idct_sub_dual(s5[28], s5[27]);
|
||||
s6[28] = highbd_idct_add_dual(s5[27], s5[28]);
|
||||
s6[29] = highbd_idct_add_dual(s5[26], s5[29]);
|
||||
s6[30] = highbd_idct_add_dual(s4[25], s4[30]);
|
||||
s6[31] = highbd_idct_add_dual(s4[24], s4[31]);
|
||||
|
||||
// stage 7
|
||||
s7[0] = highbd_idct_add_dual(s6[0], s5[15]);
|
||||
s7[1] = highbd_idct_add_dual(s6[1], s5[14]);
|
||||
s7[2] = highbd_idct_add_dual(s6[2], s6[13]);
|
||||
s7[3] = highbd_idct_add_dual(s6[3], s6[12]);
|
||||
s7[4] = highbd_idct_add_dual(s6[4], s6[11]);
|
||||
s7[5] = highbd_idct_add_dual(s6[5], s6[10]);
|
||||
s7[6] = highbd_idct_add_dual(s6[6], s5[9]);
|
||||
s7[7] = highbd_idct_add_dual(s6[7], s5[8]);
|
||||
s7[8] = highbd_idct_sub_dual(s6[7], s5[8]);
|
||||
s7[9] = highbd_idct_sub_dual(s6[6], s5[9]);
|
||||
s7[10] = highbd_idct_sub_dual(s6[5], s6[10]);
|
||||
s7[11] = highbd_idct_sub_dual(s6[4], s6[11]);
|
||||
s7[12] = highbd_idct_sub_dual(s6[3], s6[12]);
|
||||
s7[13] = highbd_idct_sub_dual(s6[2], s6[13]);
|
||||
s7[14] = highbd_idct_sub_dual(s6[1], s5[14]);
|
||||
s7[15] = highbd_idct_sub_dual(s6[0], s5[15]);
|
||||
|
||||
s7[20] = sub_multiply_shift_and_narrow_s32_dual(s6[27], s6[20], cospi_16_64);
|
||||
s7[27] = add_multiply_shift_and_narrow_s32_dual(s6[20], s6[27], cospi_16_64);
|
||||
|
||||
s7[21] = sub_multiply_shift_and_narrow_s32_dual(s6[26], s6[21], cospi_16_64);
|
||||
s7[26] = add_multiply_shift_and_narrow_s32_dual(s6[21], s6[26], cospi_16_64);
|
||||
|
||||
s7[22] = sub_multiply_shift_and_narrow_s32_dual(s6[25], s6[22], cospi_16_64);
|
||||
s7[25] = add_multiply_shift_and_narrow_s32_dual(s6[22], s6[25], cospi_16_64);
|
||||
|
||||
s7[23] = sub_multiply_shift_and_narrow_s32_dual(s6[24], s6[23], cospi_16_64);
|
||||
s7[24] = add_multiply_shift_and_narrow_s32_dual(s6[23], s6[24], cospi_16_64);
|
||||
|
||||
// final stage
|
||||
out[0] = highbd_idct_add_dual(s7[0], s6[31]);
|
||||
out[1] = highbd_idct_add_dual(s7[1], s6[30]);
|
||||
out[2] = highbd_idct_add_dual(s7[2], s6[29]);
|
||||
out[3] = highbd_idct_add_dual(s7[3], s6[28]);
|
||||
out[4] = highbd_idct_add_dual(s7[4], s7[27]);
|
||||
out[5] = highbd_idct_add_dual(s7[5], s7[26]);
|
||||
out[6] = highbd_idct_add_dual(s7[6], s7[25]);
|
||||
out[7] = highbd_idct_add_dual(s7[7], s7[24]);
|
||||
out[8] = highbd_idct_add_dual(s7[8], s7[23]);
|
||||
out[9] = highbd_idct_add_dual(s7[9], s7[22]);
|
||||
out[10] = highbd_idct_add_dual(s7[10], s7[21]);
|
||||
out[11] = highbd_idct_add_dual(s7[11], s7[20]);
|
||||
out[12] = highbd_idct_add_dual(s7[12], s6[19]);
|
||||
out[13] = highbd_idct_add_dual(s7[13], s6[18]);
|
||||
out[14] = highbd_idct_add_dual(s7[14], s6[17]);
|
||||
out[15] = highbd_idct_add_dual(s7[15], s6[16]);
|
||||
out[16] = highbd_idct_sub_dual(s7[15], s6[16]);
|
||||
out[17] = highbd_idct_sub_dual(s7[14], s6[17]);
|
||||
out[18] = highbd_idct_sub_dual(s7[13], s6[18]);
|
||||
out[19] = highbd_idct_sub_dual(s7[12], s6[19]);
|
||||
out[20] = highbd_idct_sub_dual(s7[11], s7[20]);
|
||||
out[21] = highbd_idct_sub_dual(s7[10], s7[21]);
|
||||
out[22] = highbd_idct_sub_dual(s7[9], s7[22]);
|
||||
out[23] = highbd_idct_sub_dual(s7[8], s7[23]);
|
||||
out[24] = highbd_idct_sub_dual(s7[7], s7[24]);
|
||||
out[25] = highbd_idct_sub_dual(s7[6], s7[25]);
|
||||
out[26] = highbd_idct_sub_dual(s7[5], s7[26]);
|
||||
out[27] = highbd_idct_sub_dual(s7[4], s7[27]);
|
||||
out[28] = highbd_idct_sub_dual(s7[3], s6[28]);
|
||||
out[29] = highbd_idct_sub_dual(s7[2], s6[29]);
|
||||
out[30] = highbd_idct_sub_dual(s7[1], s6[30]);
|
||||
out[31] = highbd_idct_sub_dual(s7[0], s6[31]);
|
||||
|
||||
highbd_idct16x16_add_store(out, output, stride, bd);
|
||||
highbd_idct16x16_add_store(out + 16, output + 16 * stride, stride, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_idct32x32_135_add_neon(const tran_low_t *input, uint16_t *dest,
|
||||
int stride, int bd) {
|
||||
int i;
|
||||
|
||||
if (bd == 8) {
|
||||
int16_t temp[32 * 16];
|
||||
int16_t *t = temp;
|
||||
vpx_idct32_12_neon(input, temp);
|
||||
vpx_idct32_12_neon(input + 32 * 8, temp + 8);
|
||||
|
||||
for (i = 0; i < 32; i += 8) {
|
||||
vpx_idct32_16_neon(t, dest, stride, 1);
|
||||
t += (16 * 8);
|
||||
dest += 8;
|
||||
}
|
||||
} else {
|
||||
int32_t temp[32 * 16];
|
||||
int32_t *t = temp;
|
||||
vpx_highbd_idct32_12_neon(input, temp);
|
||||
vpx_highbd_idct32_12_neon(input + 32 * 8, temp + 8);
|
||||
|
||||
for (i = 0; i < 32; i += 8) {
|
||||
vpx_highbd_idct32_16_neon(t, dest, stride, bd);
|
||||
t += (16 * 8);
|
||||
dest += 8;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,625 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/highbd_idct_neon.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
|
||||
// Only for the first pass of the _34_ variant. Since it only uses values from
|
||||
// the top left 8x8 it can safely assume all the remaining values are 0 and skip
|
||||
// an awful lot of calculations. In fact, only the first 6 columns make the cut.
|
||||
// None of the elements in the 7th or 8th column are used so it skips any calls
|
||||
// to input[67] too.
|
||||
// In C this does a single row of 32 for each call. Here it transposes the top
|
||||
// left 8x8 to allow using SIMD.
|
||||
|
||||
// vp9/common/vp9_scan.c:vp9_default_iscan_32x32 arranges the first 34 non-zero
|
||||
// coefficients as follows:
|
||||
// 0 1 2 3 4 5 6 7
|
||||
// 0 0 2 5 10 17 25
|
||||
// 1 1 4 8 15 22 30
|
||||
// 2 3 7 12 18 28
|
||||
// 3 6 11 16 23 31
|
||||
// 4 9 14 19 29
|
||||
// 5 13 20 26
|
||||
// 6 21 27 33
|
||||
// 7 24 32
|
||||
static void vpx_highbd_idct32_6_neon(const tran_low_t *input, int32_t *output) {
|
||||
int32x4x2_t in[8], s1[32], s2[32], s3[32];
|
||||
|
||||
in[0].val[0] = vld1q_s32(input);
|
||||
in[0].val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in[1].val[0] = vld1q_s32(input);
|
||||
in[1].val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in[2].val[0] = vld1q_s32(input);
|
||||
in[2].val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in[3].val[0] = vld1q_s32(input);
|
||||
in[3].val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in[4].val[0] = vld1q_s32(input);
|
||||
in[4].val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in[5].val[0] = vld1q_s32(input);
|
||||
in[5].val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in[6].val[0] = vld1q_s32(input);
|
||||
in[6].val[1] = vld1q_s32(input + 4);
|
||||
input += 32;
|
||||
in[7].val[0] = vld1q_s32(input);
|
||||
in[7].val[1] = vld1q_s32(input + 4);
|
||||
transpose_s32_8x8(&in[0], &in[1], &in[2], &in[3], &in[4], &in[5], &in[6],
|
||||
&in[7]);
|
||||
|
||||
// stage 1
|
||||
// input[1] * cospi_31_64 - input[31] * cospi_1_64 (but input[31] == 0)
|
||||
s1[16] = multiply_shift_and_narrow_s32_dual(in[1], cospi_31_64);
|
||||
// input[1] * cospi_1_64 + input[31] * cospi_31_64 (but input[31] == 0)
|
||||
s1[31] = multiply_shift_and_narrow_s32_dual(in[1], cospi_1_64);
|
||||
|
||||
s1[20] = multiply_shift_and_narrow_s32_dual(in[5], cospi_27_64);
|
||||
s1[27] = multiply_shift_and_narrow_s32_dual(in[5], cospi_5_64);
|
||||
|
||||
s1[23] = multiply_shift_and_narrow_s32_dual(in[3], -cospi_29_64);
|
||||
s1[24] = multiply_shift_and_narrow_s32_dual(in[3], cospi_3_64);
|
||||
|
||||
// stage 2
|
||||
s2[8] = multiply_shift_and_narrow_s32_dual(in[2], cospi_30_64);
|
||||
s2[15] = multiply_shift_and_narrow_s32_dual(in[2], cospi_2_64);
|
||||
|
||||
// stage 3
|
||||
s1[4] = multiply_shift_and_narrow_s32_dual(in[4], cospi_28_64);
|
||||
s1[7] = multiply_shift_and_narrow_s32_dual(in[4], cospi_4_64);
|
||||
|
||||
s1[17] = multiply_accumulate_shift_and_narrow_s32_dual(s1[16], -cospi_4_64,
|
||||
s1[31], cospi_28_64);
|
||||
s1[30] = multiply_accumulate_shift_and_narrow_s32_dual(s1[16], cospi_28_64,
|
||||
s1[31], cospi_4_64);
|
||||
|
||||
s1[21] = multiply_accumulate_shift_and_narrow_s32_dual(s1[20], -cospi_20_64,
|
||||
s1[27], cospi_12_64);
|
||||
s1[26] = multiply_accumulate_shift_and_narrow_s32_dual(s1[20], cospi_12_64,
|
||||
s1[27], cospi_20_64);
|
||||
|
||||
s1[22] = multiply_accumulate_shift_and_narrow_s32_dual(s1[23], -cospi_12_64,
|
||||
s1[24], -cospi_20_64);
|
||||
s1[25] = multiply_accumulate_shift_and_narrow_s32_dual(s1[23], -cospi_20_64,
|
||||
s1[24], cospi_12_64);
|
||||
|
||||
// stage 4
|
||||
s1[0] = multiply_shift_and_narrow_s32_dual(in[0], cospi_16_64);
|
||||
|
||||
s2[9] = multiply_accumulate_shift_and_narrow_s32_dual(s2[8], -cospi_8_64,
|
||||
s2[15], cospi_24_64);
|
||||
s2[14] = multiply_accumulate_shift_and_narrow_s32_dual(s2[8], cospi_24_64,
|
||||
s2[15], cospi_8_64);
|
||||
|
||||
s2[20] = highbd_idct_sub_dual(s1[23], s1[20]);
|
||||
s2[21] = highbd_idct_sub_dual(s1[22], s1[21]);
|
||||
s2[22] = highbd_idct_add_dual(s1[21], s1[22]);
|
||||
s2[23] = highbd_idct_add_dual(s1[20], s1[23]);
|
||||
s2[24] = highbd_idct_add_dual(s1[24], s1[27]);
|
||||
s2[25] = highbd_idct_add_dual(s1[25], s1[26]);
|
||||
s2[26] = highbd_idct_sub_dual(s1[25], s1[26]);
|
||||
s2[27] = highbd_idct_sub_dual(s1[24], s1[27]);
|
||||
|
||||
// stage 5
|
||||
s1[5] = sub_multiply_shift_and_narrow_s32_dual(s1[7], s1[4], cospi_16_64);
|
||||
s1[6] = add_multiply_shift_and_narrow_s32_dual(s1[4], s1[7], cospi_16_64);
|
||||
|
||||
s1[18] = multiply_accumulate_shift_and_narrow_s32_dual(s1[17], -cospi_8_64,
|
||||
s1[30], cospi_24_64);
|
||||
s1[29] = multiply_accumulate_shift_and_narrow_s32_dual(s1[17], cospi_24_64,
|
||||
s1[30], cospi_8_64);
|
||||
|
||||
s1[19] = multiply_accumulate_shift_and_narrow_s32_dual(s1[16], -cospi_8_64,
|
||||
s1[31], cospi_24_64);
|
||||
s1[28] = multiply_accumulate_shift_and_narrow_s32_dual(s1[16], cospi_24_64,
|
||||
s1[31], cospi_8_64);
|
||||
|
||||
s1[20] = multiply_accumulate_shift_and_narrow_s32_dual(s2[20], -cospi_24_64,
|
||||
s2[27], -cospi_8_64);
|
||||
s1[27] = multiply_accumulate_shift_and_narrow_s32_dual(s2[20], -cospi_8_64,
|
||||
s2[27], cospi_24_64);
|
||||
|
||||
s1[21] = multiply_accumulate_shift_and_narrow_s32_dual(s2[21], -cospi_24_64,
|
||||
s2[26], -cospi_8_64);
|
||||
s1[26] = multiply_accumulate_shift_and_narrow_s32_dual(s2[21], -cospi_8_64,
|
||||
s2[26], cospi_24_64);
|
||||
|
||||
// stage 6
|
||||
s2[0] = highbd_idct_add_dual(s1[0], s1[7]);
|
||||
s2[1] = highbd_idct_add_dual(s1[0], s1[6]);
|
||||
s2[2] = highbd_idct_add_dual(s1[0], s1[5]);
|
||||
s2[3] = highbd_idct_add_dual(s1[0], s1[4]);
|
||||
s2[4] = highbd_idct_sub_dual(s1[0], s1[4]);
|
||||
s2[5] = highbd_idct_sub_dual(s1[0], s1[5]);
|
||||
s2[6] = highbd_idct_sub_dual(s1[0], s1[6]);
|
||||
s2[7] = highbd_idct_sub_dual(s1[0], s1[7]);
|
||||
|
||||
s2[10] = sub_multiply_shift_and_narrow_s32_dual(s2[14], s2[9], cospi_16_64);
|
||||
s2[13] = add_multiply_shift_and_narrow_s32_dual(s2[9], s2[14], cospi_16_64);
|
||||
|
||||
s2[11] = sub_multiply_shift_and_narrow_s32_dual(s2[15], s2[8], cospi_16_64);
|
||||
s2[12] = add_multiply_shift_and_narrow_s32_dual(s2[8], s2[15], cospi_16_64);
|
||||
|
||||
s2[16] = highbd_idct_add_dual(s1[16], s2[23]);
|
||||
s2[17] = highbd_idct_add_dual(s1[17], s2[22]);
|
||||
s2[18] = highbd_idct_add_dual(s1[18], s1[21]);
|
||||
s2[19] = highbd_idct_add_dual(s1[19], s1[20]);
|
||||
s2[20] = highbd_idct_sub_dual(s1[19], s1[20]);
|
||||
s2[21] = highbd_idct_sub_dual(s1[18], s1[21]);
|
||||
s2[22] = highbd_idct_sub_dual(s1[17], s2[22]);
|
||||
s2[23] = highbd_idct_sub_dual(s1[16], s2[23]);
|
||||
|
||||
s3[24] = highbd_idct_sub_dual(s1[31], s2[24]);
|
||||
s3[25] = highbd_idct_sub_dual(s1[30], s2[25]);
|
||||
s3[26] = highbd_idct_sub_dual(s1[29], s1[26]);
|
||||
s3[27] = highbd_idct_sub_dual(s1[28], s1[27]);
|
||||
s2[28] = highbd_idct_add_dual(s1[27], s1[28]);
|
||||
s2[29] = highbd_idct_add_dual(s1[26], s1[29]);
|
||||
s2[30] = highbd_idct_add_dual(s2[25], s1[30]);
|
||||
s2[31] = highbd_idct_add_dual(s2[24], s1[31]);
|
||||
|
||||
// stage 7
|
||||
s1[0] = highbd_idct_add_dual(s2[0], s2[15]);
|
||||
s1[1] = highbd_idct_add_dual(s2[1], s2[14]);
|
||||
s1[2] = highbd_idct_add_dual(s2[2], s2[13]);
|
||||
s1[3] = highbd_idct_add_dual(s2[3], s2[12]);
|
||||
s1[4] = highbd_idct_add_dual(s2[4], s2[11]);
|
||||
s1[5] = highbd_idct_add_dual(s2[5], s2[10]);
|
||||
s1[6] = highbd_idct_add_dual(s2[6], s2[9]);
|
||||
s1[7] = highbd_idct_add_dual(s2[7], s2[8]);
|
||||
s1[8] = highbd_idct_sub_dual(s2[7], s2[8]);
|
||||
s1[9] = highbd_idct_sub_dual(s2[6], s2[9]);
|
||||
s1[10] = highbd_idct_sub_dual(s2[5], s2[10]);
|
||||
s1[11] = highbd_idct_sub_dual(s2[4], s2[11]);
|
||||
s1[12] = highbd_idct_sub_dual(s2[3], s2[12]);
|
||||
s1[13] = highbd_idct_sub_dual(s2[2], s2[13]);
|
||||
s1[14] = highbd_idct_sub_dual(s2[1], s2[14]);
|
||||
s1[15] = highbd_idct_sub_dual(s2[0], s2[15]);
|
||||
|
||||
s1[20] = sub_multiply_shift_and_narrow_s32_dual(s3[27], s2[20], cospi_16_64);
|
||||
s1[27] = add_multiply_shift_and_narrow_s32_dual(s2[20], s3[27], cospi_16_64);
|
||||
|
||||
s1[21] = sub_multiply_shift_and_narrow_s32_dual(s3[26], s2[21], cospi_16_64);
|
||||
s1[26] = add_multiply_shift_and_narrow_s32_dual(s2[21], s3[26], cospi_16_64);
|
||||
|
||||
s1[22] = sub_multiply_shift_and_narrow_s32_dual(s3[25], s2[22], cospi_16_64);
|
||||
s1[25] = add_multiply_shift_and_narrow_s32_dual(s2[22], s3[25], cospi_16_64);
|
||||
|
||||
s1[23] = sub_multiply_shift_and_narrow_s32_dual(s3[24], s2[23], cospi_16_64);
|
||||
s1[24] = add_multiply_shift_and_narrow_s32_dual(s2[23], s3[24], cospi_16_64);
|
||||
|
||||
// final stage
|
||||
s3[0] = highbd_idct_add_dual(s1[0], s2[31]);
|
||||
s3[1] = highbd_idct_add_dual(s1[1], s2[30]);
|
||||
s3[2] = highbd_idct_add_dual(s1[2], s2[29]);
|
||||
s3[3] = highbd_idct_add_dual(s1[3], s2[28]);
|
||||
s3[4] = highbd_idct_add_dual(s1[4], s1[27]);
|
||||
s3[5] = highbd_idct_add_dual(s1[5], s1[26]);
|
||||
s3[6] = highbd_idct_add_dual(s1[6], s1[25]);
|
||||
s3[7] = highbd_idct_add_dual(s1[7], s1[24]);
|
||||
s3[8] = highbd_idct_add_dual(s1[8], s1[23]);
|
||||
s3[9] = highbd_idct_add_dual(s1[9], s1[22]);
|
||||
s3[10] = highbd_idct_add_dual(s1[10], s1[21]);
|
||||
s3[11] = highbd_idct_add_dual(s1[11], s1[20]);
|
||||
s3[12] = highbd_idct_add_dual(s1[12], s2[19]);
|
||||
s3[13] = highbd_idct_add_dual(s1[13], s2[18]);
|
||||
s3[14] = highbd_idct_add_dual(s1[14], s2[17]);
|
||||
s3[15] = highbd_idct_add_dual(s1[15], s2[16]);
|
||||
s3[16] = highbd_idct_sub_dual(s1[15], s2[16]);
|
||||
s3[17] = highbd_idct_sub_dual(s1[14], s2[17]);
|
||||
s3[18] = highbd_idct_sub_dual(s1[13], s2[18]);
|
||||
s3[19] = highbd_idct_sub_dual(s1[12], s2[19]);
|
||||
s3[20] = highbd_idct_sub_dual(s1[11], s1[20]);
|
||||
s3[21] = highbd_idct_sub_dual(s1[10], s1[21]);
|
||||
s3[22] = highbd_idct_sub_dual(s1[9], s1[22]);
|
||||
s3[23] = highbd_idct_sub_dual(s1[8], s1[23]);
|
||||
s3[24] = highbd_idct_sub_dual(s1[7], s1[24]);
|
||||
s3[25] = highbd_idct_sub_dual(s1[6], s1[25]);
|
||||
s3[26] = highbd_idct_sub_dual(s1[5], s1[26]);
|
||||
s3[27] = highbd_idct_sub_dual(s1[4], s1[27]);
|
||||
s3[28] = highbd_idct_sub_dual(s1[3], s2[28]);
|
||||
s3[29] = highbd_idct_sub_dual(s1[2], s2[29]);
|
||||
s3[30] = highbd_idct_sub_dual(s1[1], s2[30]);
|
||||
s3[31] = highbd_idct_sub_dual(s1[0], s2[31]);
|
||||
|
||||
vst1q_s32(output, s3[0].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[0].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[1].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[1].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[2].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[2].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[3].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[3].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[4].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[4].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[5].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[5].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[6].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[6].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[7].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[7].val[1]);
|
||||
output += 4;
|
||||
|
||||
vst1q_s32(output, s3[8].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[8].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[9].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[9].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[10].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[10].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[11].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[11].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[12].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[12].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[13].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[13].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[14].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[14].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[15].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[15].val[1]);
|
||||
output += 4;
|
||||
|
||||
vst1q_s32(output, s3[16].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[16].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[17].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[17].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[18].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[18].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[19].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[19].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[20].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[20].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[21].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[21].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[22].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[22].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[23].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[23].val[1]);
|
||||
output += 4;
|
||||
|
||||
vst1q_s32(output, s3[24].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[24].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[25].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[25].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[26].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[26].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[27].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[27].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[28].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[28].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[29].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[29].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[30].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[30].val[1]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[31].val[0]);
|
||||
output += 4;
|
||||
vst1q_s32(output, s3[31].val[1]);
|
||||
}
|
||||
|
||||
static void vpx_highbd_idct32_8_neon(const int32_t *input, uint16_t *output,
|
||||
int stride, const int bd) {
|
||||
int32x4x2_t in[8], s1[32], s2[32], s3[32], out[32];
|
||||
|
||||
load_and_transpose_s32_8x8(input, 8, &in[0], &in[1], &in[2], &in[3], &in[4],
|
||||
&in[5], &in[6], &in[7]);
|
||||
|
||||
// stage 1
|
||||
s1[16] = multiply_shift_and_narrow_s32_dual(in[1], cospi_31_64);
|
||||
s1[31] = multiply_shift_and_narrow_s32_dual(in[1], cospi_1_64);
|
||||
|
||||
// Different for _8_
|
||||
s1[19] = multiply_shift_and_narrow_s32_dual(in[7], -cospi_25_64);
|
||||
s1[28] = multiply_shift_and_narrow_s32_dual(in[7], cospi_7_64);
|
||||
|
||||
s1[20] = multiply_shift_and_narrow_s32_dual(in[5], cospi_27_64);
|
||||
s1[27] = multiply_shift_and_narrow_s32_dual(in[5], cospi_5_64);
|
||||
|
||||
s1[23] = multiply_shift_and_narrow_s32_dual(in[3], -cospi_29_64);
|
||||
s1[24] = multiply_shift_and_narrow_s32_dual(in[3], cospi_3_64);
|
||||
|
||||
// stage 2
|
||||
s2[8] = multiply_shift_and_narrow_s32_dual(in[2], cospi_30_64);
|
||||
s2[15] = multiply_shift_and_narrow_s32_dual(in[2], cospi_2_64);
|
||||
|
||||
s2[11] = multiply_shift_and_narrow_s32_dual(in[6], -cospi_26_64);
|
||||
s2[12] = multiply_shift_and_narrow_s32_dual(in[6], cospi_6_64);
|
||||
|
||||
// stage 3
|
||||
s1[4] = multiply_shift_and_narrow_s32_dual(in[4], cospi_28_64);
|
||||
s1[7] = multiply_shift_and_narrow_s32_dual(in[4], cospi_4_64);
|
||||
|
||||
s1[17] = multiply_accumulate_shift_and_narrow_s32_dual(s1[16], -cospi_4_64,
|
||||
s1[31], cospi_28_64);
|
||||
s1[30] = multiply_accumulate_shift_and_narrow_s32_dual(s1[16], cospi_28_64,
|
||||
s1[31], cospi_4_64);
|
||||
|
||||
// Different for _8_
|
||||
s1[18] = multiply_accumulate_shift_and_narrow_s32_dual(s1[19], -cospi_28_64,
|
||||
s1[28], -cospi_4_64);
|
||||
s1[29] = multiply_accumulate_shift_and_narrow_s32_dual(s1[19], -cospi_4_64,
|
||||
s1[28], cospi_28_64);
|
||||
|
||||
s1[21] = multiply_accumulate_shift_and_narrow_s32_dual(s1[20], -cospi_20_64,
|
||||
s1[27], cospi_12_64);
|
||||
s1[26] = multiply_accumulate_shift_and_narrow_s32_dual(s1[20], cospi_12_64,
|
||||
s1[27], cospi_20_64);
|
||||
|
||||
s1[22] = multiply_accumulate_shift_and_narrow_s32_dual(s1[23], -cospi_12_64,
|
||||
s1[24], -cospi_20_64);
|
||||
s1[25] = multiply_accumulate_shift_and_narrow_s32_dual(s1[23], -cospi_20_64,
|
||||
s1[24], cospi_12_64);
|
||||
|
||||
// stage 4
|
||||
s1[0] = multiply_shift_and_narrow_s32_dual(in[0], cospi_16_64);
|
||||
|
||||
s2[9] = multiply_accumulate_shift_and_narrow_s32_dual(s2[8], -cospi_8_64,
|
||||
s2[15], cospi_24_64);
|
||||
s2[14] = multiply_accumulate_shift_and_narrow_s32_dual(s2[8], cospi_24_64,
|
||||
s2[15], cospi_8_64);
|
||||
|
||||
s2[10] = multiply_accumulate_shift_and_narrow_s32_dual(s2[11], -cospi_24_64,
|
||||
s2[12], -cospi_8_64);
|
||||
s2[13] = multiply_accumulate_shift_and_narrow_s32_dual(s2[11], -cospi_8_64,
|
||||
s2[12], cospi_24_64);
|
||||
|
||||
s2[16] = highbd_idct_add_dual(s1[16], s1[19]);
|
||||
|
||||
s2[17] = highbd_idct_add_dual(s1[17], s1[18]);
|
||||
s2[18] = highbd_idct_sub_dual(s1[17], s1[18]);
|
||||
|
||||
s2[19] = highbd_idct_sub_dual(s1[16], s1[19]);
|
||||
|
||||
s2[20] = highbd_idct_sub_dual(s1[23], s1[20]);
|
||||
s2[21] = highbd_idct_sub_dual(s1[22], s1[21]);
|
||||
|
||||
s2[22] = highbd_idct_add_dual(s1[21], s1[22]);
|
||||
s2[23] = highbd_idct_add_dual(s1[20], s1[23]);
|
||||
|
||||
s2[24] = highbd_idct_add_dual(s1[24], s1[27]);
|
||||
s2[25] = highbd_idct_add_dual(s1[25], s1[26]);
|
||||
s2[26] = highbd_idct_sub_dual(s1[25], s1[26]);
|
||||
s2[27] = highbd_idct_sub_dual(s1[24], s1[27]);
|
||||
|
||||
s2[28] = highbd_idct_sub_dual(s1[31], s1[28]);
|
||||
s2[29] = highbd_idct_sub_dual(s1[30], s1[29]);
|
||||
s2[30] = highbd_idct_add_dual(s1[29], s1[30]);
|
||||
s2[31] = highbd_idct_add_dual(s1[28], s1[31]);
|
||||
|
||||
// stage 5
|
||||
s1[5] = sub_multiply_shift_and_narrow_s32_dual(s1[7], s1[4], cospi_16_64);
|
||||
s1[6] = add_multiply_shift_and_narrow_s32_dual(s1[4], s1[7], cospi_16_64);
|
||||
|
||||
s1[8] = highbd_idct_add_dual(s2[8], s2[11]);
|
||||
s1[9] = highbd_idct_add_dual(s2[9], s2[10]);
|
||||
s1[10] = highbd_idct_sub_dual(s2[9], s2[10]);
|
||||
s1[11] = highbd_idct_sub_dual(s2[8], s2[11]);
|
||||
s1[12] = highbd_idct_sub_dual(s2[15], s2[12]);
|
||||
s1[13] = highbd_idct_sub_dual(s2[14], s2[13]);
|
||||
s1[14] = highbd_idct_add_dual(s2[13], s2[14]);
|
||||
s1[15] = highbd_idct_add_dual(s2[12], s2[15]);
|
||||
|
||||
s1[18] = multiply_accumulate_shift_and_narrow_s32_dual(s2[18], -cospi_8_64,
|
||||
s2[29], cospi_24_64);
|
||||
s1[29] = multiply_accumulate_shift_and_narrow_s32_dual(s2[18], cospi_24_64,
|
||||
s2[29], cospi_8_64);
|
||||
|
||||
s1[19] = multiply_accumulate_shift_and_narrow_s32_dual(s2[19], -cospi_8_64,
|
||||
s2[28], cospi_24_64);
|
||||
s1[28] = multiply_accumulate_shift_and_narrow_s32_dual(s2[19], cospi_24_64,
|
||||
s2[28], cospi_8_64);
|
||||
|
||||
s1[20] = multiply_accumulate_shift_and_narrow_s32_dual(s2[20], -cospi_24_64,
|
||||
s2[27], -cospi_8_64);
|
||||
s1[27] = multiply_accumulate_shift_and_narrow_s32_dual(s2[20], -cospi_8_64,
|
||||
s2[27], cospi_24_64);
|
||||
|
||||
s1[21] = multiply_accumulate_shift_and_narrow_s32_dual(s2[21], -cospi_24_64,
|
||||
s2[26], -cospi_8_64);
|
||||
s1[26] = multiply_accumulate_shift_and_narrow_s32_dual(s2[21], -cospi_8_64,
|
||||
s2[26], cospi_24_64);
|
||||
|
||||
// stage 6
|
||||
s2[0] = highbd_idct_add_dual(s1[0], s1[7]);
|
||||
s2[1] = highbd_idct_add_dual(s1[0], s1[6]);
|
||||
s2[2] = highbd_idct_add_dual(s1[0], s1[5]);
|
||||
s2[3] = highbd_idct_add_dual(s1[0], s1[4]);
|
||||
s2[4] = highbd_idct_sub_dual(s1[0], s1[4]);
|
||||
s2[5] = highbd_idct_sub_dual(s1[0], s1[5]);
|
||||
s2[6] = highbd_idct_sub_dual(s1[0], s1[6]);
|
||||
s2[7] = highbd_idct_sub_dual(s1[0], s1[7]);
|
||||
|
||||
s2[10] = sub_multiply_shift_and_narrow_s32_dual(s1[13], s1[10], cospi_16_64);
|
||||
s2[13] = add_multiply_shift_and_narrow_s32_dual(s1[10], s1[13], cospi_16_64);
|
||||
|
||||
s2[11] = sub_multiply_shift_and_narrow_s32_dual(s1[12], s1[11], cospi_16_64);
|
||||
s2[12] = add_multiply_shift_and_narrow_s32_dual(s1[11], s1[12], cospi_16_64);
|
||||
|
||||
s1[16] = highbd_idct_add_dual(s2[16], s2[23]);
|
||||
s1[17] = highbd_idct_add_dual(s2[17], s2[22]);
|
||||
s2[18] = highbd_idct_add_dual(s1[18], s1[21]);
|
||||
s2[19] = highbd_idct_add_dual(s1[19], s1[20]);
|
||||
s2[20] = highbd_idct_sub_dual(s1[19], s1[20]);
|
||||
s2[21] = highbd_idct_sub_dual(s1[18], s1[21]);
|
||||
s1[22] = highbd_idct_sub_dual(s2[17], s2[22]);
|
||||
s1[23] = highbd_idct_sub_dual(s2[16], s2[23]);
|
||||
|
||||
s3[24] = highbd_idct_sub_dual(s2[31], s2[24]);
|
||||
s3[25] = highbd_idct_sub_dual(s2[30], s2[25]);
|
||||
s3[26] = highbd_idct_sub_dual(s1[29], s1[26]);
|
||||
s3[27] = highbd_idct_sub_dual(s1[28], s1[27]);
|
||||
s2[28] = highbd_idct_add_dual(s1[27], s1[28]);
|
||||
s2[29] = highbd_idct_add_dual(s1[26], s1[29]);
|
||||
s2[30] = highbd_idct_add_dual(s2[25], s2[30]);
|
||||
s2[31] = highbd_idct_add_dual(s2[24], s2[31]);
|
||||
|
||||
// stage 7
|
||||
s1[0] = highbd_idct_add_dual(s2[0], s1[15]);
|
||||
s1[1] = highbd_idct_add_dual(s2[1], s1[14]);
|
||||
s1[2] = highbd_idct_add_dual(s2[2], s2[13]);
|
||||
s1[3] = highbd_idct_add_dual(s2[3], s2[12]);
|
||||
s1[4] = highbd_idct_add_dual(s2[4], s2[11]);
|
||||
s1[5] = highbd_idct_add_dual(s2[5], s2[10]);
|
||||
s1[6] = highbd_idct_add_dual(s2[6], s1[9]);
|
||||
s1[7] = highbd_idct_add_dual(s2[7], s1[8]);
|
||||
s1[8] = highbd_idct_sub_dual(s2[7], s1[8]);
|
||||
s1[9] = highbd_idct_sub_dual(s2[6], s1[9]);
|
||||
s1[10] = highbd_idct_sub_dual(s2[5], s2[10]);
|
||||
s1[11] = highbd_idct_sub_dual(s2[4], s2[11]);
|
||||
s1[12] = highbd_idct_sub_dual(s2[3], s2[12]);
|
||||
s1[13] = highbd_idct_sub_dual(s2[2], s2[13]);
|
||||
s1[14] = highbd_idct_sub_dual(s2[1], s1[14]);
|
||||
s1[15] = highbd_idct_sub_dual(s2[0], s1[15]);
|
||||
|
||||
s1[20] = sub_multiply_shift_and_narrow_s32_dual(s3[27], s2[20], cospi_16_64);
|
||||
s1[27] = add_multiply_shift_and_narrow_s32_dual(s2[20], s3[27], cospi_16_64);
|
||||
|
||||
s1[21] = sub_multiply_shift_and_narrow_s32_dual(s3[26], s2[21], cospi_16_64);
|
||||
s1[26] = add_multiply_shift_and_narrow_s32_dual(s2[21], s3[26], cospi_16_64);
|
||||
|
||||
s2[22] = sub_multiply_shift_and_narrow_s32_dual(s3[25], s1[22], cospi_16_64);
|
||||
s1[25] = add_multiply_shift_and_narrow_s32_dual(s1[22], s3[25], cospi_16_64);
|
||||
|
||||
s2[23] = sub_multiply_shift_and_narrow_s32_dual(s3[24], s1[23], cospi_16_64);
|
||||
s1[24] = add_multiply_shift_and_narrow_s32_dual(s1[23], s3[24], cospi_16_64);
|
||||
|
||||
// final stage
|
||||
out[0] = highbd_idct_add_dual(s1[0], s2[31]);
|
||||
out[1] = highbd_idct_add_dual(s1[1], s2[30]);
|
||||
out[2] = highbd_idct_add_dual(s1[2], s2[29]);
|
||||
out[3] = highbd_idct_add_dual(s1[3], s2[28]);
|
||||
out[4] = highbd_idct_add_dual(s1[4], s1[27]);
|
||||
out[5] = highbd_idct_add_dual(s1[5], s1[26]);
|
||||
out[6] = highbd_idct_add_dual(s1[6], s1[25]);
|
||||
out[7] = highbd_idct_add_dual(s1[7], s1[24]);
|
||||
out[8] = highbd_idct_add_dual(s1[8], s2[23]);
|
||||
out[9] = highbd_idct_add_dual(s1[9], s2[22]);
|
||||
out[10] = highbd_idct_add_dual(s1[10], s1[21]);
|
||||
out[11] = highbd_idct_add_dual(s1[11], s1[20]);
|
||||
out[12] = highbd_idct_add_dual(s1[12], s2[19]);
|
||||
out[13] = highbd_idct_add_dual(s1[13], s2[18]);
|
||||
out[14] = highbd_idct_add_dual(s1[14], s1[17]);
|
||||
out[15] = highbd_idct_add_dual(s1[15], s1[16]);
|
||||
out[16] = highbd_idct_sub_dual(s1[15], s1[16]);
|
||||
out[17] = highbd_idct_sub_dual(s1[14], s1[17]);
|
||||
out[18] = highbd_idct_sub_dual(s1[13], s2[18]);
|
||||
out[19] = highbd_idct_sub_dual(s1[12], s2[19]);
|
||||
out[20] = highbd_idct_sub_dual(s1[11], s1[20]);
|
||||
out[21] = highbd_idct_sub_dual(s1[10], s1[21]);
|
||||
out[22] = highbd_idct_sub_dual(s1[9], s2[22]);
|
||||
out[23] = highbd_idct_sub_dual(s1[8], s2[23]);
|
||||
out[24] = highbd_idct_sub_dual(s1[7], s1[24]);
|
||||
out[25] = highbd_idct_sub_dual(s1[6], s1[25]);
|
||||
out[26] = highbd_idct_sub_dual(s1[5], s1[26]);
|
||||
out[27] = highbd_idct_sub_dual(s1[4], s1[27]);
|
||||
out[28] = highbd_idct_sub_dual(s1[3], s2[28]);
|
||||
out[29] = highbd_idct_sub_dual(s1[2], s2[29]);
|
||||
out[30] = highbd_idct_sub_dual(s1[1], s2[30]);
|
||||
out[31] = highbd_idct_sub_dual(s1[0], s2[31]);
|
||||
|
||||
highbd_idct16x16_add_store(out, output, stride, bd);
|
||||
highbd_idct16x16_add_store(out + 16, output + 16 * stride, stride, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_idct32x32_34_add_neon(const tran_low_t *input, uint16_t *dest,
|
||||
int stride, int bd) {
|
||||
int i;
|
||||
|
||||
if (bd == 8) {
|
||||
int16_t temp[32 * 8];
|
||||
int16_t *t = temp;
|
||||
|
||||
vpx_idct32_6_neon(input, t);
|
||||
|
||||
for (i = 0; i < 32; i += 8) {
|
||||
vpx_idct32_8_neon(t, dest, stride, 1);
|
||||
t += (8 * 8);
|
||||
dest += 8;
|
||||
}
|
||||
} else {
|
||||
int32_t temp[32 * 8];
|
||||
int32_t *t = temp;
|
||||
|
||||
vpx_highbd_idct32_6_neon(input, t);
|
||||
|
||||
for (i = 0; i < 32; i += 8) {
|
||||
vpx_highbd_idct32_8_neon(t, dest, stride, bd);
|
||||
t += (8 * 8);
|
||||
dest += 8;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/inv_txfm.h"
|
||||
|
||||
static INLINE void highbd_idct32x32_1_add_pos_kernel(uint16_t **dest,
|
||||
const int stride,
|
||||
const int16x8_t res,
|
||||
const int16x8_t max) {
|
||||
const uint16x8_t a0 = vld1q_u16(*dest);
|
||||
const uint16x8_t a1 = vld1q_u16(*dest + 8);
|
||||
const uint16x8_t a2 = vld1q_u16(*dest + 16);
|
||||
const uint16x8_t a3 = vld1q_u16(*dest + 24);
|
||||
const int16x8_t b0 = vaddq_s16(res, vreinterpretq_s16_u16(a0));
|
||||
const int16x8_t b1 = vaddq_s16(res, vreinterpretq_s16_u16(a1));
|
||||
const int16x8_t b2 = vaddq_s16(res, vreinterpretq_s16_u16(a2));
|
||||
const int16x8_t b3 = vaddq_s16(res, vreinterpretq_s16_u16(a3));
|
||||
const int16x8_t c0 = vminq_s16(b0, max);
|
||||
const int16x8_t c1 = vminq_s16(b1, max);
|
||||
const int16x8_t c2 = vminq_s16(b2, max);
|
||||
const int16x8_t c3 = vminq_s16(b3, max);
|
||||
vst1q_u16(*dest, vreinterpretq_u16_s16(c0));
|
||||
vst1q_u16(*dest + 8, vreinterpretq_u16_s16(c1));
|
||||
vst1q_u16(*dest + 16, vreinterpretq_u16_s16(c2));
|
||||
vst1q_u16(*dest + 24, vreinterpretq_u16_s16(c3));
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void highbd_idct32x32_1_add_neg_kernel(uint16_t **dest,
|
||||
const int stride,
|
||||
const int16x8_t res) {
|
||||
const uint16x8_t a0 = vld1q_u16(*dest);
|
||||
const uint16x8_t a1 = vld1q_u16(*dest + 8);
|
||||
const uint16x8_t a2 = vld1q_u16(*dest + 16);
|
||||
const uint16x8_t a3 = vld1q_u16(*dest + 24);
|
||||
const int16x8_t b0 = vaddq_s16(res, vreinterpretq_s16_u16(a0));
|
||||
const int16x8_t b1 = vaddq_s16(res, vreinterpretq_s16_u16(a1));
|
||||
const int16x8_t b2 = vaddq_s16(res, vreinterpretq_s16_u16(a2));
|
||||
const int16x8_t b3 = vaddq_s16(res, vreinterpretq_s16_u16(a3));
|
||||
const uint16x8_t c0 = vqshluq_n_s16(b0, 0);
|
||||
const uint16x8_t c1 = vqshluq_n_s16(b1, 0);
|
||||
const uint16x8_t c2 = vqshluq_n_s16(b2, 0);
|
||||
const uint16x8_t c3 = vqshluq_n_s16(b3, 0);
|
||||
vst1q_u16(*dest, c0);
|
||||
vst1q_u16(*dest + 8, c1);
|
||||
vst1q_u16(*dest + 16, c2);
|
||||
vst1q_u16(*dest + 24, c3);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
void vpx_highbd_idct32x32_1_add_neon(const tran_low_t *input, uint16_t *dest,
|
||||
int stride, int bd) {
|
||||
const tran_low_t out0 = HIGHBD_WRAPLOW(
|
||||
dct_const_round_shift(input[0] * (tran_high_t)cospi_16_64), bd);
|
||||
const tran_low_t out1 = HIGHBD_WRAPLOW(
|
||||
dct_const_round_shift(out0 * (tran_high_t)cospi_16_64), bd);
|
||||
const int16_t a1 = ROUND_POWER_OF_TWO(out1, 6);
|
||||
const int16x8_t dc = vdupq_n_s16(a1);
|
||||
int i;
|
||||
|
||||
if (a1 >= 0) {
|
||||
const int16x8_t max = vdupq_n_s16((1 << bd) - 1);
|
||||
for (i = 0; i < 8; ++i) {
|
||||
highbd_idct32x32_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct32x32_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct32x32_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct32x32_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
}
|
||||
} else {
|
||||
for (i = 0; i < 8; ++i) {
|
||||
highbd_idct32x32_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct32x32_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct32x32_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct32x32_1_add_neg_kernel(&dest, stride, dc);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/highbd_idct_neon.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/inv_txfm.h"
|
||||
|
||||
// res is in reverse row order
|
||||
static INLINE void highbd_idct4x4_1_add_kernel2(uint16_t **dest,
|
||||
const int stride,
|
||||
const int16x8_t res,
|
||||
const int16x8_t max) {
|
||||
const uint16x4_t a0 = vld1_u16(*dest);
|
||||
const uint16x4_t a1 = vld1_u16(*dest + stride);
|
||||
const int16x8_t a = vreinterpretq_s16_u16(vcombine_u16(a1, a0));
|
||||
// Note: In some profile tests, res is quite close to +/-32767.
|
||||
// We use saturating addition.
|
||||
const int16x8_t b = vqaddq_s16(res, a);
|
||||
const int16x8_t c = vminq_s16(b, max);
|
||||
const uint16x8_t d = vqshluq_n_s16(c, 0);
|
||||
vst1_u16(*dest, vget_high_u16(d));
|
||||
*dest += stride;
|
||||
vst1_u16(*dest, vget_low_u16(d));
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
void vpx_highbd_idct4x4_1_add_neon(const tran_low_t *input, uint16_t *dest,
|
||||
int stride, int bd) {
|
||||
const int16x8_t max = vdupq_n_s16((1 << bd) - 1);
|
||||
const tran_low_t out0 = HIGHBD_WRAPLOW(
|
||||
dct_const_round_shift(input[0] * (tran_high_t)cospi_16_64), bd);
|
||||
const tran_low_t out1 = HIGHBD_WRAPLOW(
|
||||
dct_const_round_shift(out0 * (tran_high_t)cospi_16_64), bd);
|
||||
const int16_t a1 = ROUND_POWER_OF_TWO(out1, 4);
|
||||
const int16x8_t dc = vdupq_n_s16(a1);
|
||||
|
||||
highbd_idct4x4_1_add_kernel1(&dest, stride, dc, max);
|
||||
highbd_idct4x4_1_add_kernel1(&dest, stride, dc, max);
|
||||
}
|
||||
|
||||
void vpx_highbd_idct4x4_16_add_neon(const tran_low_t *input, uint16_t *dest,
|
||||
int stride, int bd) {
|
||||
const int16x8_t max = vdupq_n_s16((1 << bd) - 1);
|
||||
int16x8_t a[2];
|
||||
int32x4_t c[4];
|
||||
|
||||
c[0] = vld1q_s32(input);
|
||||
c[1] = vld1q_s32(input + 4);
|
||||
c[2] = vld1q_s32(input + 8);
|
||||
c[3] = vld1q_s32(input + 12);
|
||||
|
||||
if (bd == 8) {
|
||||
// Rows
|
||||
a[0] = vcombine_s16(vmovn_s32(c[0]), vmovn_s32(c[1]));
|
||||
a[1] = vcombine_s16(vmovn_s32(c[2]), vmovn_s32(c[3]));
|
||||
transpose_idct4x4_16_bd8(a);
|
||||
|
||||
// Columns
|
||||
a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1]));
|
||||
transpose_idct4x4_16_bd8(a);
|
||||
a[0] = vrshrq_n_s16(a[0], 4);
|
||||
a[1] = vrshrq_n_s16(a[1], 4);
|
||||
} else {
|
||||
const int32x4_t cospis = vld1q_s32(kCospi32);
|
||||
|
||||
if (bd == 10) {
|
||||
idct4x4_16_kernel_bd10(cospis, c);
|
||||
idct4x4_16_kernel_bd10(cospis, c);
|
||||
} else {
|
||||
idct4x4_16_kernel_bd12(cospis, c);
|
||||
idct4x4_16_kernel_bd12(cospis, c);
|
||||
}
|
||||
a[0] = vcombine_s16(vqrshrn_n_s32(c[0], 4), vqrshrn_n_s32(c[1], 4));
|
||||
a[1] = vcombine_s16(vqrshrn_n_s32(c[3], 4), vqrshrn_n_s32(c[2], 4));
|
||||
}
|
||||
|
||||
highbd_idct4x4_1_add_kernel1(&dest, stride, a[0], max);
|
||||
highbd_idct4x4_1_add_kernel2(&dest, stride, a[1], max);
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/highbd_idct_neon.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/inv_txfm.h"
|
||||
|
||||
static INLINE void highbd_idct8x8_1_add_pos_kernel(uint16_t **dest,
|
||||
const int stride,
|
||||
const int16x8_t res,
|
||||
const int16x8_t max) {
|
||||
const uint16x8_t a = vld1q_u16(*dest);
|
||||
const int16x8_t b = vaddq_s16(res, vreinterpretq_s16_u16(a));
|
||||
const int16x8_t c = vminq_s16(b, max);
|
||||
vst1q_u16(*dest, vreinterpretq_u16_s16(c));
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void highbd_idct8x8_1_add_neg_kernel(uint16_t **dest,
|
||||
const int stride,
|
||||
const int16x8_t res) {
|
||||
const uint16x8_t a = vld1q_u16(*dest);
|
||||
const int16x8_t b = vaddq_s16(res, vreinterpretq_s16_u16(a));
|
||||
const uint16x8_t c = vqshluq_n_s16(b, 0);
|
||||
vst1q_u16(*dest, c);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
void vpx_highbd_idct8x8_1_add_neon(const tran_low_t *input, uint16_t *dest,
|
||||
int stride, int bd) {
|
||||
const tran_low_t out0 = HIGHBD_WRAPLOW(
|
||||
dct_const_round_shift(input[0] * (tran_high_t)cospi_16_64), bd);
|
||||
const tran_low_t out1 = HIGHBD_WRAPLOW(
|
||||
dct_const_round_shift(out0 * (tran_high_t)cospi_16_64), bd);
|
||||
const int16_t a1 = ROUND_POWER_OF_TWO(out1, 5);
|
||||
const int16x8_t dc = vdupq_n_s16(a1);
|
||||
|
||||
if (a1 >= 0) {
|
||||
const int16x8_t max = vdupq_n_s16((1 << bd) - 1);
|
||||
highbd_idct8x8_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct8x8_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct8x8_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct8x8_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct8x8_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct8x8_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct8x8_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
highbd_idct8x8_1_add_pos_kernel(&dest, stride, dc, max);
|
||||
} else {
|
||||
highbd_idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
highbd_idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_12_half1d_bd10(
|
||||
const int32x4_t cospis0, const int32x4_t cospis1, int32x4_t *const io0,
|
||||
int32x4_t *const io1, int32x4_t *const io2, int32x4_t *const io3,
|
||||
int32x4_t *const io4, int32x4_t *const io5, int32x4_t *const io6,
|
||||
int32x4_t *const io7) {
|
||||
int32x4_t step1[8], step2[8];
|
||||
|
||||
transpose_s32_4x4(io0, io1, io2, io3);
|
||||
|
||||
// stage 1
|
||||
step1[4] = vmulq_lane_s32(*io1, vget_high_s32(cospis1), 1);
|
||||
step1[5] = vmulq_lane_s32(*io3, vget_high_s32(cospis1), 0);
|
||||
step1[6] = vmulq_lane_s32(*io3, vget_low_s32(cospis1), 1);
|
||||
step1[7] = vmulq_lane_s32(*io1, vget_low_s32(cospis1), 0);
|
||||
step1[4] = vrshrq_n_s32(step1[4], DCT_CONST_BITS);
|
||||
step1[5] = vrshrq_n_s32(step1[5], DCT_CONST_BITS);
|
||||
step1[6] = vrshrq_n_s32(step1[6], DCT_CONST_BITS);
|
||||
step1[7] = vrshrq_n_s32(step1[7], DCT_CONST_BITS);
|
||||
|
||||
// stage 2
|
||||
step2[1] = vmulq_lane_s32(*io0, vget_high_s32(cospis0), 0);
|
||||
step2[2] = vmulq_lane_s32(*io2, vget_high_s32(cospis0), 1);
|
||||
step2[3] = vmulq_lane_s32(*io2, vget_low_s32(cospis0), 1);
|
||||
step2[1] = vrshrq_n_s32(step2[1], DCT_CONST_BITS);
|
||||
step2[2] = vrshrq_n_s32(step2[2], DCT_CONST_BITS);
|
||||
step2[3] = vrshrq_n_s32(step2[3], DCT_CONST_BITS);
|
||||
|
||||
step2[4] = vaddq_s32(step1[4], step1[5]);
|
||||
step2[5] = vsubq_s32(step1[4], step1[5]);
|
||||
step2[6] = vsubq_s32(step1[7], step1[6]);
|
||||
step2[7] = vaddq_s32(step1[7], step1[6]);
|
||||
|
||||
// stage 3
|
||||
step1[0] = vaddq_s32(step2[1], step2[3]);
|
||||
step1[1] = vaddq_s32(step2[1], step2[2]);
|
||||
step1[2] = vsubq_s32(step2[1], step2[2]);
|
||||
step1[3] = vsubq_s32(step2[1], step2[3]);
|
||||
|
||||
step1[6] = vmulq_lane_s32(step2[6], vget_high_s32(cospis0), 0);
|
||||
step1[5] = vmlsq_lane_s32(step1[6], step2[5], vget_high_s32(cospis0), 0);
|
||||
step1[6] = vmlaq_lane_s32(step1[6], step2[5], vget_high_s32(cospis0), 0);
|
||||
step1[5] = vrshrq_n_s32(step1[5], DCT_CONST_BITS);
|
||||
step1[6] = vrshrq_n_s32(step1[6], DCT_CONST_BITS);
|
||||
|
||||
// stage 4
|
||||
*io0 = vaddq_s32(step1[0], step2[7]);
|
||||
*io1 = vaddq_s32(step1[1], step1[6]);
|
||||
*io2 = vaddq_s32(step1[2], step1[5]);
|
||||
*io3 = vaddq_s32(step1[3], step2[4]);
|
||||
*io4 = vsubq_s32(step1[3], step2[4]);
|
||||
*io5 = vsubq_s32(step1[2], step1[5]);
|
||||
*io6 = vsubq_s32(step1[1], step1[6]);
|
||||
*io7 = vsubq_s32(step1[0], step2[7]);
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_12_half1d_bd12(
|
||||
const int32x4_t cospis0, const int32x4_t cospis1, int32x4_t *const io0,
|
||||
int32x4_t *const io1, int32x4_t *const io2, int32x4_t *const io3,
|
||||
int32x4_t *const io4, int32x4_t *const io5, int32x4_t *const io6,
|
||||
int32x4_t *const io7) {
|
||||
int32x2_t input1l, input1h, input3l, input3h;
|
||||
int32x2_t step1l[2], step1h[2];
|
||||
int32x4_t step1[8], step2[8];
|
||||
int64x2_t t64[8];
|
||||
int32x2_t t32[8];
|
||||
|
||||
transpose_s32_4x4(io0, io1, io2, io3);
|
||||
|
||||
// stage 1
|
||||
input1l = vget_low_s32(*io1);
|
||||
input1h = vget_high_s32(*io1);
|
||||
input3l = vget_low_s32(*io3);
|
||||
input3h = vget_high_s32(*io3);
|
||||
step1l[0] = vget_low_s32(*io0);
|
||||
step1h[0] = vget_high_s32(*io0);
|
||||
step1l[1] = vget_low_s32(*io2);
|
||||
step1h[1] = vget_high_s32(*io2);
|
||||
|
||||
t64[0] = vmull_lane_s32(input1l, vget_high_s32(cospis1), 1);
|
||||
t64[1] = vmull_lane_s32(input1h, vget_high_s32(cospis1), 1);
|
||||
t64[2] = vmull_lane_s32(input3l, vget_high_s32(cospis1), 0);
|
||||
t64[3] = vmull_lane_s32(input3h, vget_high_s32(cospis1), 0);
|
||||
t64[4] = vmull_lane_s32(input3l, vget_low_s32(cospis1), 1);
|
||||
t64[5] = vmull_lane_s32(input3h, vget_low_s32(cospis1), 1);
|
||||
t64[6] = vmull_lane_s32(input1l, vget_low_s32(cospis1), 0);
|
||||
t64[7] = vmull_lane_s32(input1h, vget_low_s32(cospis1), 0);
|
||||
t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS);
|
||||
t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS);
|
||||
t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS);
|
||||
t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS);
|
||||
t32[4] = vrshrn_n_s64(t64[4], DCT_CONST_BITS);
|
||||
t32[5] = vrshrn_n_s64(t64[5], DCT_CONST_BITS);
|
||||
t32[6] = vrshrn_n_s64(t64[6], DCT_CONST_BITS);
|
||||
t32[7] = vrshrn_n_s64(t64[7], DCT_CONST_BITS);
|
||||
step1[4] = vcombine_s32(t32[0], t32[1]);
|
||||
step1[5] = vcombine_s32(t32[2], t32[3]);
|
||||
step1[6] = vcombine_s32(t32[4], t32[5]);
|
||||
step1[7] = vcombine_s32(t32[6], t32[7]);
|
||||
|
||||
// stage 2
|
||||
t64[2] = vmull_lane_s32(step1l[0], vget_high_s32(cospis0), 0);
|
||||
t64[3] = vmull_lane_s32(step1h[0], vget_high_s32(cospis0), 0);
|
||||
t64[4] = vmull_lane_s32(step1l[1], vget_high_s32(cospis0), 1);
|
||||
t64[5] = vmull_lane_s32(step1h[1], vget_high_s32(cospis0), 1);
|
||||
t64[6] = vmull_lane_s32(step1l[1], vget_low_s32(cospis0), 1);
|
||||
t64[7] = vmull_lane_s32(step1h[1], vget_low_s32(cospis0), 1);
|
||||
t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS);
|
||||
t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS);
|
||||
t32[4] = vrshrn_n_s64(t64[4], DCT_CONST_BITS);
|
||||
t32[5] = vrshrn_n_s64(t64[5], DCT_CONST_BITS);
|
||||
t32[6] = vrshrn_n_s64(t64[6], DCT_CONST_BITS);
|
||||
t32[7] = vrshrn_n_s64(t64[7], DCT_CONST_BITS);
|
||||
step2[1] = vcombine_s32(t32[2], t32[3]);
|
||||
step2[2] = vcombine_s32(t32[4], t32[5]);
|
||||
step2[3] = vcombine_s32(t32[6], t32[7]);
|
||||
|
||||
step2[4] = vaddq_s32(step1[4], step1[5]);
|
||||
step2[5] = vsubq_s32(step1[4], step1[5]);
|
||||
step2[6] = vsubq_s32(step1[7], step1[6]);
|
||||
step2[7] = vaddq_s32(step1[7], step1[6]);
|
||||
|
||||
// stage 3
|
||||
step1[0] = vaddq_s32(step2[1], step2[3]);
|
||||
step1[1] = vaddq_s32(step2[1], step2[2]);
|
||||
step1[2] = vsubq_s32(step2[1], step2[2]);
|
||||
step1[3] = vsubq_s32(step2[1], step2[3]);
|
||||
|
||||
t64[2] = vmull_lane_s32(vget_low_s32(step2[6]), vget_high_s32(cospis0), 0);
|
||||
t64[3] = vmull_lane_s32(vget_high_s32(step2[6]), vget_high_s32(cospis0), 0);
|
||||
t64[0] =
|
||||
vmlsl_lane_s32(t64[2], vget_low_s32(step2[5]), vget_high_s32(cospis0), 0);
|
||||
t64[1] = vmlsl_lane_s32(t64[3], vget_high_s32(step2[5]),
|
||||
vget_high_s32(cospis0), 0);
|
||||
t64[2] =
|
||||
vmlal_lane_s32(t64[2], vget_low_s32(step2[5]), vget_high_s32(cospis0), 0);
|
||||
t64[3] = vmlal_lane_s32(t64[3], vget_high_s32(step2[5]),
|
||||
vget_high_s32(cospis0), 0);
|
||||
t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS);
|
||||
t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS);
|
||||
t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS);
|
||||
t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS);
|
||||
step1[5] = vcombine_s32(t32[0], t32[1]);
|
||||
step1[6] = vcombine_s32(t32[2], t32[3]);
|
||||
|
||||
// stage 4
|
||||
*io0 = vaddq_s32(step1[0], step2[7]);
|
||||
*io1 = vaddq_s32(step1[1], step1[6]);
|
||||
*io2 = vaddq_s32(step1[2], step1[5]);
|
||||
*io3 = vaddq_s32(step1[3], step2[4]);
|
||||
*io4 = vsubq_s32(step1[3], step2[4]);
|
||||
*io5 = vsubq_s32(step1[2], step1[5]);
|
||||
*io6 = vsubq_s32(step1[1], step1[6]);
|
||||
*io7 = vsubq_s32(step1[0], step2[7]);
|
||||
}
|
||||
|
||||
void vpx_highbd_idct8x8_12_add_neon(const tran_low_t *input, uint16_t *dest,
|
||||
int stride, int bd) {
|
||||
int32x4_t a[16];
|
||||
int16x8_t c[8];
|
||||
|
||||
a[0] = vld1q_s32(input);
|
||||
a[1] = vld1q_s32(input + 8);
|
||||
a[2] = vld1q_s32(input + 16);
|
||||
a[3] = vld1q_s32(input + 24);
|
||||
|
||||
if (bd == 8) {
|
||||
const int16x8_t cospis = vld1q_s16(kCospi);
|
||||
const int16x8_t cospisd = vaddq_s16(cospis, cospis);
|
||||
const int16x4_t cospis0 = vget_low_s16(cospis); // cospi 0, 8, 16, 24
|
||||
const int16x4_t cospisd0 = vget_low_s16(cospisd); // doubled 0, 8, 16, 24
|
||||
const int16x4_t cospisd1 = vget_high_s16(cospisd); // doubled 4, 12, 20, 28
|
||||
int16x4_t b[8];
|
||||
|
||||
b[0] = vmovn_s32(a[0]);
|
||||
b[1] = vmovn_s32(a[1]);
|
||||
b[2] = vmovn_s32(a[2]);
|
||||
b[3] = vmovn_s32(a[3]);
|
||||
|
||||
idct8x8_12_pass1_bd8(cospis0, cospisd0, cospisd1, b);
|
||||
idct8x8_12_pass2_bd8(cospis0, cospisd0, cospisd1, b, c);
|
||||
c[0] = vrshrq_n_s16(c[0], 5);
|
||||
c[1] = vrshrq_n_s16(c[1], 5);
|
||||
c[2] = vrshrq_n_s16(c[2], 5);
|
||||
c[3] = vrshrq_n_s16(c[3], 5);
|
||||
c[4] = vrshrq_n_s16(c[4], 5);
|
||||
c[5] = vrshrq_n_s16(c[5], 5);
|
||||
c[6] = vrshrq_n_s16(c[6], 5);
|
||||
c[7] = vrshrq_n_s16(c[7], 5);
|
||||
} else {
|
||||
const int32x4_t cospis0 = vld1q_s32(kCospi32); // cospi 0, 8, 16, 24
|
||||
const int32x4_t cospis1 = vld1q_s32(kCospi32 + 4); // cospi 4, 12, 20, 28
|
||||
|
||||
if (bd == 10) {
|
||||
idct8x8_12_half1d_bd10(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3],
|
||||
&a[4], &a[5], &a[6], &a[7]);
|
||||
idct8x8_12_half1d_bd10(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3],
|
||||
&a[8], &a[9], &a[10], &a[11]);
|
||||
idct8x8_12_half1d_bd10(cospis0, cospis1, &a[4], &a[5], &a[6], &a[7],
|
||||
&a[12], &a[13], &a[14], &a[15]);
|
||||
} else {
|
||||
idct8x8_12_half1d_bd12(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3],
|
||||
&a[4], &a[5], &a[6], &a[7]);
|
||||
idct8x8_12_half1d_bd12(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3],
|
||||
&a[8], &a[9], &a[10], &a[11]);
|
||||
idct8x8_12_half1d_bd12(cospis0, cospis1, &a[4], &a[5], &a[6], &a[7],
|
||||
&a[12], &a[13], &a[14], &a[15]);
|
||||
}
|
||||
c[0] = vcombine_s16(vrshrn_n_s32(a[0], 5), vrshrn_n_s32(a[4], 5));
|
||||
c[1] = vcombine_s16(vrshrn_n_s32(a[1], 5), vrshrn_n_s32(a[5], 5));
|
||||
c[2] = vcombine_s16(vrshrn_n_s32(a[2], 5), vrshrn_n_s32(a[6], 5));
|
||||
c[3] = vcombine_s16(vrshrn_n_s32(a[3], 5), vrshrn_n_s32(a[7], 5));
|
||||
c[4] = vcombine_s16(vrshrn_n_s32(a[8], 5), vrshrn_n_s32(a[12], 5));
|
||||
c[5] = vcombine_s16(vrshrn_n_s32(a[9], 5), vrshrn_n_s32(a[13], 5));
|
||||
c[6] = vcombine_s16(vrshrn_n_s32(a[10], 5), vrshrn_n_s32(a[14], 5));
|
||||
c[7] = vcombine_s16(vrshrn_n_s32(a[11], 5), vrshrn_n_s32(a[15], 5));
|
||||
}
|
||||
highbd_add8x8(c, dest, stride, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_idct8x8_64_add_neon(const tran_low_t *input, uint16_t *dest,
|
||||
int stride, int bd) {
|
||||
int32x4_t a[16];
|
||||
int16x8_t c[8];
|
||||
|
||||
a[0] = vld1q_s32(input);
|
||||
a[1] = vld1q_s32(input + 4);
|
||||
a[2] = vld1q_s32(input + 8);
|
||||
a[3] = vld1q_s32(input + 12);
|
||||
a[4] = vld1q_s32(input + 16);
|
||||
a[5] = vld1q_s32(input + 20);
|
||||
a[6] = vld1q_s32(input + 24);
|
||||
a[7] = vld1q_s32(input + 28);
|
||||
a[8] = vld1q_s32(input + 32);
|
||||
a[9] = vld1q_s32(input + 36);
|
||||
a[10] = vld1q_s32(input + 40);
|
||||
a[11] = vld1q_s32(input + 44);
|
||||
a[12] = vld1q_s32(input + 48);
|
||||
a[13] = vld1q_s32(input + 52);
|
||||
a[14] = vld1q_s32(input + 56);
|
||||
a[15] = vld1q_s32(input + 60);
|
||||
|
||||
if (bd == 8) {
|
||||
const int16x8_t cospis = vld1q_s16(kCospi);
|
||||
const int16x4_t cospis0 = vget_low_s16(cospis); // cospi 0, 8, 16, 24
|
||||
const int16x4_t cospis1 = vget_high_s16(cospis); // cospi 4, 12, 20, 28
|
||||
int16x8_t b[8];
|
||||
|
||||
b[0] = vcombine_s16(vmovn_s32(a[0]), vmovn_s32(a[1]));
|
||||
b[1] = vcombine_s16(vmovn_s32(a[2]), vmovn_s32(a[3]));
|
||||
b[2] = vcombine_s16(vmovn_s32(a[4]), vmovn_s32(a[5]));
|
||||
b[3] = vcombine_s16(vmovn_s32(a[6]), vmovn_s32(a[7]));
|
||||
b[4] = vcombine_s16(vmovn_s32(a[8]), vmovn_s32(a[9]));
|
||||
b[5] = vcombine_s16(vmovn_s32(a[10]), vmovn_s32(a[11]));
|
||||
b[6] = vcombine_s16(vmovn_s32(a[12]), vmovn_s32(a[13]));
|
||||
b[7] = vcombine_s16(vmovn_s32(a[14]), vmovn_s32(a[15]));
|
||||
|
||||
idct8x8_64_1d_bd8(cospis0, cospis1, b);
|
||||
idct8x8_64_1d_bd8(cospis0, cospis1, b);
|
||||
|
||||
c[0] = vrshrq_n_s16(b[0], 5);
|
||||
c[1] = vrshrq_n_s16(b[1], 5);
|
||||
c[2] = vrshrq_n_s16(b[2], 5);
|
||||
c[3] = vrshrq_n_s16(b[3], 5);
|
||||
c[4] = vrshrq_n_s16(b[4], 5);
|
||||
c[5] = vrshrq_n_s16(b[5], 5);
|
||||
c[6] = vrshrq_n_s16(b[6], 5);
|
||||
c[7] = vrshrq_n_s16(b[7], 5);
|
||||
} else {
|
||||
const int32x4_t cospis0 = vld1q_s32(kCospi32); // cospi 0, 8, 16, 24
|
||||
const int32x4_t cospis1 = vld1q_s32(kCospi32 + 4); // cospi 4, 12, 20, 28
|
||||
|
||||
if (bd == 10) {
|
||||
idct8x8_64_half1d_bd10(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3],
|
||||
&a[4], &a[5], &a[6], &a[7]);
|
||||
idct8x8_64_half1d_bd10(cospis0, cospis1, &a[8], &a[9], &a[10], &a[11],
|
||||
&a[12], &a[13], &a[14], &a[15]);
|
||||
idct8x8_64_half1d_bd10(cospis0, cospis1, &a[0], &a[8], &a[1], &a[9],
|
||||
&a[2], &a[10], &a[3], &a[11]);
|
||||
idct8x8_64_half1d_bd10(cospis0, cospis1, &a[4], &a[12], &a[5], &a[13],
|
||||
&a[6], &a[14], &a[7], &a[15]);
|
||||
} else {
|
||||
idct8x8_64_half1d_bd12(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3],
|
||||
&a[4], &a[5], &a[6], &a[7]);
|
||||
idct8x8_64_half1d_bd12(cospis0, cospis1, &a[8], &a[9], &a[10], &a[11],
|
||||
&a[12], &a[13], &a[14], &a[15]);
|
||||
idct8x8_64_half1d_bd12(cospis0, cospis1, &a[0], &a[8], &a[1], &a[9],
|
||||
&a[2], &a[10], &a[3], &a[11]);
|
||||
idct8x8_64_half1d_bd12(cospis0, cospis1, &a[4], &a[12], &a[5], &a[13],
|
||||
&a[6], &a[14], &a[7], &a[15]);
|
||||
}
|
||||
c[0] = vcombine_s16(vrshrn_n_s32(a[0], 5), vrshrn_n_s32(a[4], 5));
|
||||
c[1] = vcombine_s16(vrshrn_n_s32(a[8], 5), vrshrn_n_s32(a[12], 5));
|
||||
c[2] = vcombine_s16(vrshrn_n_s32(a[1], 5), vrshrn_n_s32(a[5], 5));
|
||||
c[3] = vcombine_s16(vrshrn_n_s32(a[9], 5), vrshrn_n_s32(a[13], 5));
|
||||
c[4] = vcombine_s16(vrshrn_n_s32(a[2], 5), vrshrn_n_s32(a[6], 5));
|
||||
c[5] = vcombine_s16(vrshrn_n_s32(a[10], 5), vrshrn_n_s32(a[14], 5));
|
||||
c[6] = vcombine_s16(vrshrn_n_s32(a[3], 5), vrshrn_n_s32(a[7], 5));
|
||||
c[7] = vcombine_s16(vrshrn_n_s32(a[11], 5), vrshrn_n_s32(a[15], 5));
|
||||
}
|
||||
highbd_add8x8(c, dest, stride, bd);
|
||||
}
|
||||
@@ -0,0 +1,474 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_ARM_HIGHBD_IDCT_NEON_H_
|
||||
#define VPX_VPX_DSP_ARM_HIGHBD_IDCT_NEON_H_
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/inv_txfm.h"
|
||||
|
||||
static INLINE void highbd_idct4x4_1_add_kernel1(uint16_t **dest,
|
||||
const int stride,
|
||||
const int16x8_t res,
|
||||
const int16x8_t max) {
|
||||
const uint16x4_t a0 = vld1_u16(*dest);
|
||||
const uint16x4_t a1 = vld1_u16(*dest + stride);
|
||||
const int16x8_t a = vreinterpretq_s16_u16(vcombine_u16(a0, a1));
|
||||
// Note: In some profile tests, res is quite close to +/-32767.
|
||||
// We use saturating addition.
|
||||
const int16x8_t b = vqaddq_s16(res, a);
|
||||
const int16x8_t c = vminq_s16(b, max);
|
||||
const uint16x8_t d = vqshluq_n_s16(c, 0);
|
||||
vst1_u16(*dest, vget_low_u16(d));
|
||||
*dest += stride;
|
||||
vst1_u16(*dest, vget_high_u16(d));
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void idct4x4_16_kernel_bd10(const int32x4_t cospis,
|
||||
int32x4_t *const a) {
|
||||
int32x4_t b0, b1, b2, b3;
|
||||
|
||||
transpose_s32_4x4(&a[0], &a[1], &a[2], &a[3]);
|
||||
b0 = vaddq_s32(a[0], a[2]);
|
||||
b1 = vsubq_s32(a[0], a[2]);
|
||||
b0 = vmulq_lane_s32(b0, vget_high_s32(cospis), 0);
|
||||
b1 = vmulq_lane_s32(b1, vget_high_s32(cospis), 0);
|
||||
b2 = vmulq_lane_s32(a[1], vget_high_s32(cospis), 1);
|
||||
b3 = vmulq_lane_s32(a[1], vget_low_s32(cospis), 1);
|
||||
b2 = vmlsq_lane_s32(b2, a[3], vget_low_s32(cospis), 1);
|
||||
b3 = vmlaq_lane_s32(b3, a[3], vget_high_s32(cospis), 1);
|
||||
b0 = vrshrq_n_s32(b0, DCT_CONST_BITS);
|
||||
b1 = vrshrq_n_s32(b1, DCT_CONST_BITS);
|
||||
b2 = vrshrq_n_s32(b2, DCT_CONST_BITS);
|
||||
b3 = vrshrq_n_s32(b3, DCT_CONST_BITS);
|
||||
a[0] = vaddq_s32(b0, b3);
|
||||
a[1] = vaddq_s32(b1, b2);
|
||||
a[2] = vsubq_s32(b1, b2);
|
||||
a[3] = vsubq_s32(b0, b3);
|
||||
}
|
||||
|
||||
static INLINE void idct4x4_16_kernel_bd12(const int32x4_t cospis,
|
||||
int32x4_t *const a) {
|
||||
int32x4_t b0, b1, b2, b3;
|
||||
int64x2_t c[12];
|
||||
|
||||
transpose_s32_4x4(&a[0], &a[1], &a[2], &a[3]);
|
||||
b0 = vaddq_s32(a[0], a[2]);
|
||||
b1 = vsubq_s32(a[0], a[2]);
|
||||
c[0] = vmull_lane_s32(vget_low_s32(b0), vget_high_s32(cospis), 0);
|
||||
c[1] = vmull_lane_s32(vget_high_s32(b0), vget_high_s32(cospis), 0);
|
||||
c[2] = vmull_lane_s32(vget_low_s32(b1), vget_high_s32(cospis), 0);
|
||||
c[3] = vmull_lane_s32(vget_high_s32(b1), vget_high_s32(cospis), 0);
|
||||
c[4] = vmull_lane_s32(vget_low_s32(a[1]), vget_high_s32(cospis), 1);
|
||||
c[5] = vmull_lane_s32(vget_high_s32(a[1]), vget_high_s32(cospis), 1);
|
||||
c[6] = vmull_lane_s32(vget_low_s32(a[1]), vget_low_s32(cospis), 1);
|
||||
c[7] = vmull_lane_s32(vget_high_s32(a[1]), vget_low_s32(cospis), 1);
|
||||
c[8] = vmull_lane_s32(vget_low_s32(a[3]), vget_low_s32(cospis), 1);
|
||||
c[9] = vmull_lane_s32(vget_high_s32(a[3]), vget_low_s32(cospis), 1);
|
||||
c[10] = vmull_lane_s32(vget_low_s32(a[3]), vget_high_s32(cospis), 1);
|
||||
c[11] = vmull_lane_s32(vget_high_s32(a[3]), vget_high_s32(cospis), 1);
|
||||
c[4] = vsubq_s64(c[4], c[8]);
|
||||
c[5] = vsubq_s64(c[5], c[9]);
|
||||
c[6] = vaddq_s64(c[6], c[10]);
|
||||
c[7] = vaddq_s64(c[7], c[11]);
|
||||
b0 = vcombine_s32(vrshrn_n_s64(c[0], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(c[1], DCT_CONST_BITS));
|
||||
b1 = vcombine_s32(vrshrn_n_s64(c[2], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(c[3], DCT_CONST_BITS));
|
||||
b2 = vcombine_s32(vrshrn_n_s64(c[4], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(c[5], DCT_CONST_BITS));
|
||||
b3 = vcombine_s32(vrshrn_n_s64(c[6], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(c[7], DCT_CONST_BITS));
|
||||
a[0] = vaddq_s32(b0, b3);
|
||||
a[1] = vaddq_s32(b1, b2);
|
||||
a[2] = vsubq_s32(b1, b2);
|
||||
a[3] = vsubq_s32(b0, b3);
|
||||
}
|
||||
|
||||
static INLINE void highbd_add8x8(int16x8_t *const a, uint16_t *dest,
|
||||
const int stride, const int bd) {
|
||||
const int16x8_t max = vdupq_n_s16((1 << bd) - 1);
|
||||
const uint16_t *dst = dest;
|
||||
uint16x8_t d0, d1, d2, d3, d4, d5, d6, d7;
|
||||
uint16x8_t d0_u16, d1_u16, d2_u16, d3_u16, d4_u16, d5_u16, d6_u16, d7_u16;
|
||||
int16x8_t d0_s16, d1_s16, d2_s16, d3_s16, d4_s16, d5_s16, d6_s16, d7_s16;
|
||||
|
||||
d0 = vld1q_u16(dst);
|
||||
dst += stride;
|
||||
d1 = vld1q_u16(dst);
|
||||
dst += stride;
|
||||
d2 = vld1q_u16(dst);
|
||||
dst += stride;
|
||||
d3 = vld1q_u16(dst);
|
||||
dst += stride;
|
||||
d4 = vld1q_u16(dst);
|
||||
dst += stride;
|
||||
d5 = vld1q_u16(dst);
|
||||
dst += stride;
|
||||
d6 = vld1q_u16(dst);
|
||||
dst += stride;
|
||||
d7 = vld1q_u16(dst);
|
||||
|
||||
d0_s16 = vqaddq_s16(a[0], vreinterpretq_s16_u16(d0));
|
||||
d1_s16 = vqaddq_s16(a[1], vreinterpretq_s16_u16(d1));
|
||||
d2_s16 = vqaddq_s16(a[2], vreinterpretq_s16_u16(d2));
|
||||
d3_s16 = vqaddq_s16(a[3], vreinterpretq_s16_u16(d3));
|
||||
d4_s16 = vqaddq_s16(a[4], vreinterpretq_s16_u16(d4));
|
||||
d5_s16 = vqaddq_s16(a[5], vreinterpretq_s16_u16(d5));
|
||||
d6_s16 = vqaddq_s16(a[6], vreinterpretq_s16_u16(d6));
|
||||
d7_s16 = vqaddq_s16(a[7], vreinterpretq_s16_u16(d7));
|
||||
|
||||
d0_s16 = vminq_s16(d0_s16, max);
|
||||
d1_s16 = vminq_s16(d1_s16, max);
|
||||
d2_s16 = vminq_s16(d2_s16, max);
|
||||
d3_s16 = vminq_s16(d3_s16, max);
|
||||
d4_s16 = vminq_s16(d4_s16, max);
|
||||
d5_s16 = vminq_s16(d5_s16, max);
|
||||
d6_s16 = vminq_s16(d6_s16, max);
|
||||
d7_s16 = vminq_s16(d7_s16, max);
|
||||
d0_u16 = vqshluq_n_s16(d0_s16, 0);
|
||||
d1_u16 = vqshluq_n_s16(d1_s16, 0);
|
||||
d2_u16 = vqshluq_n_s16(d2_s16, 0);
|
||||
d3_u16 = vqshluq_n_s16(d3_s16, 0);
|
||||
d4_u16 = vqshluq_n_s16(d4_s16, 0);
|
||||
d5_u16 = vqshluq_n_s16(d5_s16, 0);
|
||||
d6_u16 = vqshluq_n_s16(d6_s16, 0);
|
||||
d7_u16 = vqshluq_n_s16(d7_s16, 0);
|
||||
|
||||
vst1q_u16(dest, d0_u16);
|
||||
dest += stride;
|
||||
vst1q_u16(dest, d1_u16);
|
||||
dest += stride;
|
||||
vst1q_u16(dest, d2_u16);
|
||||
dest += stride;
|
||||
vst1q_u16(dest, d3_u16);
|
||||
dest += stride;
|
||||
vst1q_u16(dest, d4_u16);
|
||||
dest += stride;
|
||||
vst1q_u16(dest, d5_u16);
|
||||
dest += stride;
|
||||
vst1q_u16(dest, d6_u16);
|
||||
dest += stride;
|
||||
vst1q_u16(dest, d7_u16);
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_64_half1d_bd10(
|
||||
const int32x4_t cospis0, const int32x4_t cospis1, int32x4_t *const io0,
|
||||
int32x4_t *const io1, int32x4_t *const io2, int32x4_t *const io3,
|
||||
int32x4_t *const io4, int32x4_t *const io5, int32x4_t *const io6,
|
||||
int32x4_t *const io7) {
|
||||
int32x4_t step1[8], step2[8];
|
||||
|
||||
transpose_s32_8x4(io0, io1, io2, io3, io4, io5, io6, io7);
|
||||
|
||||
// stage 1
|
||||
step1[4] = vmulq_lane_s32(*io1, vget_high_s32(cospis1), 1);
|
||||
step1[5] = vmulq_lane_s32(*io3, vget_high_s32(cospis1), 0);
|
||||
step1[6] = vmulq_lane_s32(*io3, vget_low_s32(cospis1), 1);
|
||||
step1[7] = vmulq_lane_s32(*io1, vget_low_s32(cospis1), 0);
|
||||
|
||||
step1[4] = vmlsq_lane_s32(step1[4], *io7, vget_low_s32(cospis1), 0);
|
||||
step1[5] = vmlaq_lane_s32(step1[5], *io5, vget_low_s32(cospis1), 1);
|
||||
step1[6] = vmlsq_lane_s32(step1[6], *io5, vget_high_s32(cospis1), 0);
|
||||
step1[7] = vmlaq_lane_s32(step1[7], *io7, vget_high_s32(cospis1), 1);
|
||||
|
||||
step1[4] = vrshrq_n_s32(step1[4], DCT_CONST_BITS);
|
||||
step1[5] = vrshrq_n_s32(step1[5], DCT_CONST_BITS);
|
||||
step1[6] = vrshrq_n_s32(step1[6], DCT_CONST_BITS);
|
||||
step1[7] = vrshrq_n_s32(step1[7], DCT_CONST_BITS);
|
||||
|
||||
// stage 2
|
||||
step2[1] = vmulq_lane_s32(*io0, vget_high_s32(cospis0), 0);
|
||||
step2[2] = vmulq_lane_s32(*io2, vget_high_s32(cospis0), 1);
|
||||
step2[3] = vmulq_lane_s32(*io2, vget_low_s32(cospis0), 1);
|
||||
|
||||
step2[0] = vmlaq_lane_s32(step2[1], *io4, vget_high_s32(cospis0), 0);
|
||||
step2[1] = vmlsq_lane_s32(step2[1], *io4, vget_high_s32(cospis0), 0);
|
||||
step2[2] = vmlsq_lane_s32(step2[2], *io6, vget_low_s32(cospis0), 1);
|
||||
step2[3] = vmlaq_lane_s32(step2[3], *io6, vget_high_s32(cospis0), 1);
|
||||
|
||||
step2[0] = vrshrq_n_s32(step2[0], DCT_CONST_BITS);
|
||||
step2[1] = vrshrq_n_s32(step2[1], DCT_CONST_BITS);
|
||||
step2[2] = vrshrq_n_s32(step2[2], DCT_CONST_BITS);
|
||||
step2[3] = vrshrq_n_s32(step2[3], DCT_CONST_BITS);
|
||||
|
||||
step2[4] = vaddq_s32(step1[4], step1[5]);
|
||||
step2[5] = vsubq_s32(step1[4], step1[5]);
|
||||
step2[6] = vsubq_s32(step1[7], step1[6]);
|
||||
step2[7] = vaddq_s32(step1[7], step1[6]);
|
||||
|
||||
// stage 3
|
||||
step1[0] = vaddq_s32(step2[0], step2[3]);
|
||||
step1[1] = vaddq_s32(step2[1], step2[2]);
|
||||
step1[2] = vsubq_s32(step2[1], step2[2]);
|
||||
step1[3] = vsubq_s32(step2[0], step2[3]);
|
||||
|
||||
step1[6] = vmulq_lane_s32(step2[6], vget_high_s32(cospis0), 0);
|
||||
step1[5] = vmlsq_lane_s32(step1[6], step2[5], vget_high_s32(cospis0), 0);
|
||||
step1[6] = vmlaq_lane_s32(step1[6], step2[5], vget_high_s32(cospis0), 0);
|
||||
step1[5] = vrshrq_n_s32(step1[5], DCT_CONST_BITS);
|
||||
step1[6] = vrshrq_n_s32(step1[6], DCT_CONST_BITS);
|
||||
|
||||
// stage 4
|
||||
*io0 = vaddq_s32(step1[0], step2[7]);
|
||||
*io1 = vaddq_s32(step1[1], step1[6]);
|
||||
*io2 = vaddq_s32(step1[2], step1[5]);
|
||||
*io3 = vaddq_s32(step1[3], step2[4]);
|
||||
*io4 = vsubq_s32(step1[3], step2[4]);
|
||||
*io5 = vsubq_s32(step1[2], step1[5]);
|
||||
*io6 = vsubq_s32(step1[1], step1[6]);
|
||||
*io7 = vsubq_s32(step1[0], step2[7]);
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_64_half1d_bd12(
|
||||
const int32x4_t cospis0, const int32x4_t cospis1, int32x4_t *const io0,
|
||||
int32x4_t *const io1, int32x4_t *const io2, int32x4_t *const io3,
|
||||
int32x4_t *const io4, int32x4_t *const io5, int32x4_t *const io6,
|
||||
int32x4_t *const io7) {
|
||||
int32x2_t input1l, input1h, input3l, input3h, input5l, input5h, input7l,
|
||||
input7h;
|
||||
int32x2_t step1l[4], step1h[4];
|
||||
int32x4_t step1[8], step2[8];
|
||||
int64x2_t t64[8];
|
||||
int32x2_t t32[8];
|
||||
|
||||
transpose_s32_8x4(io0, io1, io2, io3, io4, io5, io6, io7);
|
||||
|
||||
// stage 1
|
||||
input1l = vget_low_s32(*io1);
|
||||
input1h = vget_high_s32(*io1);
|
||||
input3l = vget_low_s32(*io3);
|
||||
input3h = vget_high_s32(*io3);
|
||||
input5l = vget_low_s32(*io5);
|
||||
input5h = vget_high_s32(*io5);
|
||||
input7l = vget_low_s32(*io7);
|
||||
input7h = vget_high_s32(*io7);
|
||||
step1l[0] = vget_low_s32(*io0);
|
||||
step1h[0] = vget_high_s32(*io0);
|
||||
step1l[1] = vget_low_s32(*io2);
|
||||
step1h[1] = vget_high_s32(*io2);
|
||||
step1l[2] = vget_low_s32(*io4);
|
||||
step1h[2] = vget_high_s32(*io4);
|
||||
step1l[3] = vget_low_s32(*io6);
|
||||
step1h[3] = vget_high_s32(*io6);
|
||||
|
||||
t64[0] = vmull_lane_s32(input1l, vget_high_s32(cospis1), 1);
|
||||
t64[1] = vmull_lane_s32(input1h, vget_high_s32(cospis1), 1);
|
||||
t64[2] = vmull_lane_s32(input3l, vget_high_s32(cospis1), 0);
|
||||
t64[3] = vmull_lane_s32(input3h, vget_high_s32(cospis1), 0);
|
||||
t64[4] = vmull_lane_s32(input3l, vget_low_s32(cospis1), 1);
|
||||
t64[5] = vmull_lane_s32(input3h, vget_low_s32(cospis1), 1);
|
||||
t64[6] = vmull_lane_s32(input1l, vget_low_s32(cospis1), 0);
|
||||
t64[7] = vmull_lane_s32(input1h, vget_low_s32(cospis1), 0);
|
||||
t64[0] = vmlsl_lane_s32(t64[0], input7l, vget_low_s32(cospis1), 0);
|
||||
t64[1] = vmlsl_lane_s32(t64[1], input7h, vget_low_s32(cospis1), 0);
|
||||
t64[2] = vmlal_lane_s32(t64[2], input5l, vget_low_s32(cospis1), 1);
|
||||
t64[3] = vmlal_lane_s32(t64[3], input5h, vget_low_s32(cospis1), 1);
|
||||
t64[4] = vmlsl_lane_s32(t64[4], input5l, vget_high_s32(cospis1), 0);
|
||||
t64[5] = vmlsl_lane_s32(t64[5], input5h, vget_high_s32(cospis1), 0);
|
||||
t64[6] = vmlal_lane_s32(t64[6], input7l, vget_high_s32(cospis1), 1);
|
||||
t64[7] = vmlal_lane_s32(t64[7], input7h, vget_high_s32(cospis1), 1);
|
||||
t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS);
|
||||
t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS);
|
||||
t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS);
|
||||
t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS);
|
||||
t32[4] = vrshrn_n_s64(t64[4], DCT_CONST_BITS);
|
||||
t32[5] = vrshrn_n_s64(t64[5], DCT_CONST_BITS);
|
||||
t32[6] = vrshrn_n_s64(t64[6], DCT_CONST_BITS);
|
||||
t32[7] = vrshrn_n_s64(t64[7], DCT_CONST_BITS);
|
||||
step1[4] = vcombine_s32(t32[0], t32[1]);
|
||||
step1[5] = vcombine_s32(t32[2], t32[3]);
|
||||
step1[6] = vcombine_s32(t32[4], t32[5]);
|
||||
step1[7] = vcombine_s32(t32[6], t32[7]);
|
||||
|
||||
// stage 2
|
||||
t64[2] = vmull_lane_s32(step1l[0], vget_high_s32(cospis0), 0);
|
||||
t64[3] = vmull_lane_s32(step1h[0], vget_high_s32(cospis0), 0);
|
||||
t64[4] = vmull_lane_s32(step1l[1], vget_high_s32(cospis0), 1);
|
||||
t64[5] = vmull_lane_s32(step1h[1], vget_high_s32(cospis0), 1);
|
||||
t64[6] = vmull_lane_s32(step1l[1], vget_low_s32(cospis0), 1);
|
||||
t64[7] = vmull_lane_s32(step1h[1], vget_low_s32(cospis0), 1);
|
||||
t64[0] = vmlal_lane_s32(t64[2], step1l[2], vget_high_s32(cospis0), 0);
|
||||
t64[1] = vmlal_lane_s32(t64[3], step1h[2], vget_high_s32(cospis0), 0);
|
||||
t64[2] = vmlsl_lane_s32(t64[2], step1l[2], vget_high_s32(cospis0), 0);
|
||||
t64[3] = vmlsl_lane_s32(t64[3], step1h[2], vget_high_s32(cospis0), 0);
|
||||
t64[4] = vmlsl_lane_s32(t64[4], step1l[3], vget_low_s32(cospis0), 1);
|
||||
t64[5] = vmlsl_lane_s32(t64[5], step1h[3], vget_low_s32(cospis0), 1);
|
||||
t64[6] = vmlal_lane_s32(t64[6], step1l[3], vget_high_s32(cospis0), 1);
|
||||
t64[7] = vmlal_lane_s32(t64[7], step1h[3], vget_high_s32(cospis0), 1);
|
||||
t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS);
|
||||
t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS);
|
||||
t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS);
|
||||
t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS);
|
||||
t32[4] = vrshrn_n_s64(t64[4], DCT_CONST_BITS);
|
||||
t32[5] = vrshrn_n_s64(t64[5], DCT_CONST_BITS);
|
||||
t32[6] = vrshrn_n_s64(t64[6], DCT_CONST_BITS);
|
||||
t32[7] = vrshrn_n_s64(t64[7], DCT_CONST_BITS);
|
||||
step2[0] = vcombine_s32(t32[0], t32[1]);
|
||||
step2[1] = vcombine_s32(t32[2], t32[3]);
|
||||
step2[2] = vcombine_s32(t32[4], t32[5]);
|
||||
step2[3] = vcombine_s32(t32[6], t32[7]);
|
||||
|
||||
step2[4] = vaddq_s32(step1[4], step1[5]);
|
||||
step2[5] = vsubq_s32(step1[4], step1[5]);
|
||||
step2[6] = vsubq_s32(step1[7], step1[6]);
|
||||
step2[7] = vaddq_s32(step1[7], step1[6]);
|
||||
|
||||
// stage 3
|
||||
step1[0] = vaddq_s32(step2[0], step2[3]);
|
||||
step1[1] = vaddq_s32(step2[1], step2[2]);
|
||||
step1[2] = vsubq_s32(step2[1], step2[2]);
|
||||
step1[3] = vsubq_s32(step2[0], step2[3]);
|
||||
|
||||
t64[2] = vmull_lane_s32(vget_low_s32(step2[6]), vget_high_s32(cospis0), 0);
|
||||
t64[3] = vmull_lane_s32(vget_high_s32(step2[6]), vget_high_s32(cospis0), 0);
|
||||
t64[0] =
|
||||
vmlsl_lane_s32(t64[2], vget_low_s32(step2[5]), vget_high_s32(cospis0), 0);
|
||||
t64[1] = vmlsl_lane_s32(t64[3], vget_high_s32(step2[5]),
|
||||
vget_high_s32(cospis0), 0);
|
||||
t64[2] =
|
||||
vmlal_lane_s32(t64[2], vget_low_s32(step2[5]), vget_high_s32(cospis0), 0);
|
||||
t64[3] = vmlal_lane_s32(t64[3], vget_high_s32(step2[5]),
|
||||
vget_high_s32(cospis0), 0);
|
||||
t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS);
|
||||
t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS);
|
||||
t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS);
|
||||
t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS);
|
||||
step1[5] = vcombine_s32(t32[0], t32[1]);
|
||||
step1[6] = vcombine_s32(t32[2], t32[3]);
|
||||
|
||||
// stage 4
|
||||
*io0 = vaddq_s32(step1[0], step2[7]);
|
||||
*io1 = vaddq_s32(step1[1], step1[6]);
|
||||
*io2 = vaddq_s32(step1[2], step1[5]);
|
||||
*io3 = vaddq_s32(step1[3], step2[4]);
|
||||
*io4 = vsubq_s32(step1[3], step2[4]);
|
||||
*io5 = vsubq_s32(step1[2], step1[5]);
|
||||
*io6 = vsubq_s32(step1[1], step1[6]);
|
||||
*io7 = vsubq_s32(step1[0], step2[7]);
|
||||
}
|
||||
|
||||
static INLINE void highbd_idct16x16_store_pass1(const int32x4x2_t *const out,
|
||||
int32_t *output) {
|
||||
// Save the result into output
|
||||
vst1q_s32(output + 0, out[0].val[0]);
|
||||
vst1q_s32(output + 4, out[0].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[1].val[0]);
|
||||
vst1q_s32(output + 4, out[1].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[2].val[0]);
|
||||
vst1q_s32(output + 4, out[2].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[3].val[0]);
|
||||
vst1q_s32(output + 4, out[3].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[4].val[0]);
|
||||
vst1q_s32(output + 4, out[4].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[5].val[0]);
|
||||
vst1q_s32(output + 4, out[5].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[6].val[0]);
|
||||
vst1q_s32(output + 4, out[6].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[7].val[0]);
|
||||
vst1q_s32(output + 4, out[7].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[8].val[0]);
|
||||
vst1q_s32(output + 4, out[8].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[9].val[0]);
|
||||
vst1q_s32(output + 4, out[9].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[10].val[0]);
|
||||
vst1q_s32(output + 4, out[10].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[11].val[0]);
|
||||
vst1q_s32(output + 4, out[11].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[12].val[0]);
|
||||
vst1q_s32(output + 4, out[12].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[13].val[0]);
|
||||
vst1q_s32(output + 4, out[13].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[14].val[0]);
|
||||
vst1q_s32(output + 4, out[14].val[1]);
|
||||
output += 16;
|
||||
vst1q_s32(output + 0, out[15].val[0]);
|
||||
vst1q_s32(output + 4, out[15].val[1]);
|
||||
}
|
||||
|
||||
static INLINE void highbd_idct16x16_add_store(const int32x4x2_t *const out,
|
||||
uint16_t *dest, const int stride,
|
||||
const int bd) {
|
||||
// Add the result to dest
|
||||
const int16x8_t max = vdupq_n_s16((1 << bd) - 1);
|
||||
int16x8_t o[16];
|
||||
o[0] = vcombine_s16(vrshrn_n_s32(out[0].val[0], 6),
|
||||
vrshrn_n_s32(out[0].val[1], 6));
|
||||
o[1] = vcombine_s16(vrshrn_n_s32(out[1].val[0], 6),
|
||||
vrshrn_n_s32(out[1].val[1], 6));
|
||||
o[2] = vcombine_s16(vrshrn_n_s32(out[2].val[0], 6),
|
||||
vrshrn_n_s32(out[2].val[1], 6));
|
||||
o[3] = vcombine_s16(vrshrn_n_s32(out[3].val[0], 6),
|
||||
vrshrn_n_s32(out[3].val[1], 6));
|
||||
o[4] = vcombine_s16(vrshrn_n_s32(out[4].val[0], 6),
|
||||
vrshrn_n_s32(out[4].val[1], 6));
|
||||
o[5] = vcombine_s16(vrshrn_n_s32(out[5].val[0], 6),
|
||||
vrshrn_n_s32(out[5].val[1], 6));
|
||||
o[6] = vcombine_s16(vrshrn_n_s32(out[6].val[0], 6),
|
||||
vrshrn_n_s32(out[6].val[1], 6));
|
||||
o[7] = vcombine_s16(vrshrn_n_s32(out[7].val[0], 6),
|
||||
vrshrn_n_s32(out[7].val[1], 6));
|
||||
o[8] = vcombine_s16(vrshrn_n_s32(out[8].val[0], 6),
|
||||
vrshrn_n_s32(out[8].val[1], 6));
|
||||
o[9] = vcombine_s16(vrshrn_n_s32(out[9].val[0], 6),
|
||||
vrshrn_n_s32(out[9].val[1], 6));
|
||||
o[10] = vcombine_s16(vrshrn_n_s32(out[10].val[0], 6),
|
||||
vrshrn_n_s32(out[10].val[1], 6));
|
||||
o[11] = vcombine_s16(vrshrn_n_s32(out[11].val[0], 6),
|
||||
vrshrn_n_s32(out[11].val[1], 6));
|
||||
o[12] = vcombine_s16(vrshrn_n_s32(out[12].val[0], 6),
|
||||
vrshrn_n_s32(out[12].val[1], 6));
|
||||
o[13] = vcombine_s16(vrshrn_n_s32(out[13].val[0], 6),
|
||||
vrshrn_n_s32(out[13].val[1], 6));
|
||||
o[14] = vcombine_s16(vrshrn_n_s32(out[14].val[0], 6),
|
||||
vrshrn_n_s32(out[14].val[1], 6));
|
||||
o[15] = vcombine_s16(vrshrn_n_s32(out[15].val[0], 6),
|
||||
vrshrn_n_s32(out[15].val[1], 6));
|
||||
highbd_idct16x16_add8x1(o[0], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[1], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[2], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[3], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[4], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[5], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[6], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[7], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[8], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[9], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[10], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[11], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[12], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[13], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[14], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(o[15], max, &dest, stride);
|
||||
}
|
||||
|
||||
void vpx_highbd_idct16x16_256_add_half1d(const int32_t *input, int32_t *output,
|
||||
uint16_t *dest, const int stride,
|
||||
const int bd);
|
||||
|
||||
#endif // VPX_VPX_DSP_ARM_HIGHBD_IDCT_NEON_H_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,761 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
|
||||
static INLINE void load_thresh(const uint8_t *blimit, const uint8_t *limit,
|
||||
const uint8_t *thresh, uint16x8_t *blimit_vec,
|
||||
uint16x8_t *limit_vec, uint16x8_t *thresh_vec,
|
||||
const int bd) {
|
||||
const int16x8_t shift = vdupq_n_s16(bd - 8);
|
||||
*blimit_vec = vmovl_u8(vld1_dup_u8(blimit));
|
||||
*limit_vec = vmovl_u8(vld1_dup_u8(limit));
|
||||
*thresh_vec = vmovl_u8(vld1_dup_u8(thresh));
|
||||
*blimit_vec = vshlq_u16(*blimit_vec, shift);
|
||||
*limit_vec = vshlq_u16(*limit_vec, shift);
|
||||
*thresh_vec = vshlq_u16(*thresh_vec, shift);
|
||||
}
|
||||
|
||||
// Here flat is 128-bit long, with each 16-bit chunk being a mask of
|
||||
// a pixel. When used to control filter branches, we only detect whether it is
|
||||
// all 0s or all 1s. We pairwise add flat to a 32-bit long number flat_status.
|
||||
// flat equals 0 if and only if flat_status equals 0.
|
||||
// flat equals -1 (all 1s) if and only if flat_status equals -4. (This is true
|
||||
// because each mask occupies more than 1 bit.)
|
||||
static INLINE uint32_t calc_flat_status(const uint16x8_t flat) {
|
||||
const uint64x1_t t0 = vadd_u64(vreinterpret_u64_u16(vget_low_u16(flat)),
|
||||
vreinterpret_u64_u16(vget_high_u16(flat)));
|
||||
const uint64x1_t t1 = vpaddl_u32(vreinterpret_u32_u64(t0));
|
||||
return vget_lane_u32(vreinterpret_u32_u64(t1), 0);
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t
|
||||
filter_hev_mask4(const uint16x8_t limit, const uint16x8_t blimit,
|
||||
const uint16x8_t thresh, const uint16x8_t p3,
|
||||
const uint16x8_t p2, const uint16x8_t p1, const uint16x8_t p0,
|
||||
const uint16x8_t q0, const uint16x8_t q1, const uint16x8_t q2,
|
||||
const uint16x8_t q3, uint16x8_t *hev, uint16x8_t *mask) {
|
||||
uint16x8_t max, t0, t1;
|
||||
|
||||
max = vabdq_u16(p1, p0);
|
||||
max = vmaxq_u16(max, vabdq_u16(q1, q0));
|
||||
*hev = vcgtq_u16(max, thresh);
|
||||
*mask = vmaxq_u16(max, vabdq_u16(p3, p2));
|
||||
*mask = vmaxq_u16(*mask, vabdq_u16(p2, p1));
|
||||
*mask = vmaxq_u16(*mask, vabdq_u16(q2, q1));
|
||||
*mask = vmaxq_u16(*mask, vabdq_u16(q3, q2));
|
||||
t0 = vabdq_u16(p0, q0);
|
||||
t1 = vabdq_u16(p1, q1);
|
||||
t0 = vaddq_u16(t0, t0);
|
||||
t1 = vshrq_n_u16(t1, 1);
|
||||
t0 = vaddq_u16(t0, t1);
|
||||
*mask = vcleq_u16(*mask, limit);
|
||||
t0 = vcleq_u16(t0, blimit);
|
||||
*mask = vandq_u16(*mask, t0);
|
||||
|
||||
return max;
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t filter_flat_hev_mask(
|
||||
const uint16x8_t limit, const uint16x8_t blimit, const uint16x8_t thresh,
|
||||
const uint16x8_t p3, const uint16x8_t p2, const uint16x8_t p1,
|
||||
const uint16x8_t p0, const uint16x8_t q0, const uint16x8_t q1,
|
||||
const uint16x8_t q2, const uint16x8_t q3, uint16x8_t *flat,
|
||||
uint32_t *flat_status, uint16x8_t *hev, const int bd) {
|
||||
uint16x8_t mask;
|
||||
const uint16x8_t max = filter_hev_mask4(limit, blimit, thresh, p3, p2, p1, p0,
|
||||
q0, q1, q2, q3, hev, &mask);
|
||||
*flat = vmaxq_u16(max, vabdq_u16(p2, p0));
|
||||
*flat = vmaxq_u16(*flat, vabdq_u16(q2, q0));
|
||||
*flat = vmaxq_u16(*flat, vabdq_u16(p3, p0));
|
||||
*flat = vmaxq_u16(*flat, vabdq_u16(q3, q0));
|
||||
*flat = vcleq_u16(*flat, vdupq_n_u16(1 << (bd - 8))); /* flat_mask4() */
|
||||
*flat = vandq_u16(*flat, mask);
|
||||
*flat_status = calc_flat_status(*flat);
|
||||
|
||||
return mask;
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t flat_mask5(const uint16x8_t p4, const uint16x8_t p3,
|
||||
const uint16x8_t p2, const uint16x8_t p1,
|
||||
const uint16x8_t p0, const uint16x8_t q0,
|
||||
const uint16x8_t q1, const uint16x8_t q2,
|
||||
const uint16x8_t q3, const uint16x8_t q4,
|
||||
const uint16x8_t flat,
|
||||
uint32_t *flat2_status, const int bd) {
|
||||
uint16x8_t flat2 = vabdq_u16(p4, p0);
|
||||
flat2 = vmaxq_u16(flat2, vabdq_u16(p3, p0));
|
||||
flat2 = vmaxq_u16(flat2, vabdq_u16(p2, p0));
|
||||
flat2 = vmaxq_u16(flat2, vabdq_u16(p1, p0));
|
||||
flat2 = vmaxq_u16(flat2, vabdq_u16(q1, q0));
|
||||
flat2 = vmaxq_u16(flat2, vabdq_u16(q2, q0));
|
||||
flat2 = vmaxq_u16(flat2, vabdq_u16(q3, q0));
|
||||
flat2 = vmaxq_u16(flat2, vabdq_u16(q4, q0));
|
||||
flat2 = vcleq_u16(flat2, vdupq_n_u16(1 << (bd - 8)));
|
||||
flat2 = vandq_u16(flat2, flat);
|
||||
*flat2_status = calc_flat_status(flat2);
|
||||
|
||||
return flat2;
|
||||
}
|
||||
|
||||
static INLINE int16x8_t flip_sign(const uint16x8_t v, const int bd) {
|
||||
const uint16x8_t offset = vdupq_n_u16(0x80 << (bd - 8));
|
||||
return vreinterpretq_s16_u16(vsubq_u16(v, offset));
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t flip_sign_back(const int16x8_t v, const int bd) {
|
||||
const int16x8_t offset = vdupq_n_s16(0x80 << (bd - 8));
|
||||
return vreinterpretq_u16_s16(vaddq_s16(v, offset));
|
||||
}
|
||||
|
||||
static INLINE void filter_update(const uint16x8_t sub0, const uint16x8_t sub1,
|
||||
const uint16x8_t add0, const uint16x8_t add1,
|
||||
uint16x8_t *sum) {
|
||||
*sum = vsubq_u16(*sum, sub0);
|
||||
*sum = vsubq_u16(*sum, sub1);
|
||||
*sum = vaddq_u16(*sum, add0);
|
||||
*sum = vaddq_u16(*sum, add1);
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t calc_7_tap_filter_kernel(const uint16x8_t sub0,
|
||||
const uint16x8_t sub1,
|
||||
const uint16x8_t add0,
|
||||
const uint16x8_t add1,
|
||||
uint16x8_t *sum) {
|
||||
filter_update(sub0, sub1, add0, add1, sum);
|
||||
return vrshrq_n_u16(*sum, 3);
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t apply_15_tap_filter_kernel(
|
||||
const uint16x8_t flat, const uint16x8_t sub0, const uint16x8_t sub1,
|
||||
const uint16x8_t add0, const uint16x8_t add1, const uint16x8_t in,
|
||||
uint16x8_t *sum) {
|
||||
filter_update(sub0, sub1, add0, add1, sum);
|
||||
return vbslq_u16(flat, vrshrq_n_u16(*sum, 4), in);
|
||||
}
|
||||
|
||||
// 7-tap filter [1, 1, 1, 2, 1, 1, 1]
|
||||
static INLINE void calc_7_tap_filter(const uint16x8_t p3, const uint16x8_t p2,
|
||||
const uint16x8_t p1, const uint16x8_t p0,
|
||||
const uint16x8_t q0, const uint16x8_t q1,
|
||||
const uint16x8_t q2, const uint16x8_t q3,
|
||||
uint16x8_t *op2, uint16x8_t *op1,
|
||||
uint16x8_t *op0, uint16x8_t *oq0,
|
||||
uint16x8_t *oq1, uint16x8_t *oq2) {
|
||||
uint16x8_t sum;
|
||||
sum = vaddq_u16(p3, p3); // 2*p3
|
||||
sum = vaddq_u16(sum, p3); // 3*p3
|
||||
sum = vaddq_u16(sum, p2); // 3*p3+p2
|
||||
sum = vaddq_u16(sum, p2); // 3*p3+2*p2
|
||||
sum = vaddq_u16(sum, p1); // 3*p3+2*p2+p1
|
||||
sum = vaddq_u16(sum, p0); // 3*p3+2*p2+p1+p0
|
||||
sum = vaddq_u16(sum, q0); // 3*p3+2*p2+p1+p0+q0
|
||||
*op2 = vrshrq_n_u16(sum, 3);
|
||||
*op1 = calc_7_tap_filter_kernel(p3, p2, p1, q1, &sum);
|
||||
*op0 = calc_7_tap_filter_kernel(p3, p1, p0, q2, &sum);
|
||||
*oq0 = calc_7_tap_filter_kernel(p3, p0, q0, q3, &sum);
|
||||
*oq1 = calc_7_tap_filter_kernel(p2, q0, q1, q3, &sum);
|
||||
*oq2 = calc_7_tap_filter_kernel(p1, q1, q2, q3, &sum);
|
||||
}
|
||||
|
||||
static INLINE void apply_7_tap_filter(const uint16x8_t flat,
|
||||
const uint16x8_t p3, const uint16x8_t p2,
|
||||
const uint16x8_t p1, const uint16x8_t p0,
|
||||
const uint16x8_t q0, const uint16x8_t q1,
|
||||
const uint16x8_t q2, const uint16x8_t q3,
|
||||
uint16x8_t *op2, uint16x8_t *op1,
|
||||
uint16x8_t *op0, uint16x8_t *oq0,
|
||||
uint16x8_t *oq1, uint16x8_t *oq2) {
|
||||
uint16x8_t tp1, tp0, tq0, tq1;
|
||||
calc_7_tap_filter(p3, p2, p1, p0, q0, q1, q2, q3, op2, &tp1, &tp0, &tq0, &tq1,
|
||||
oq2);
|
||||
*op2 = vbslq_u16(flat, *op2, p2);
|
||||
*op1 = vbslq_u16(flat, tp1, *op1);
|
||||
*op0 = vbslq_u16(flat, tp0, *op0);
|
||||
*oq0 = vbslq_u16(flat, tq0, *oq0);
|
||||
*oq1 = vbslq_u16(flat, tq1, *oq1);
|
||||
*oq2 = vbslq_u16(flat, *oq2, q2);
|
||||
}
|
||||
|
||||
// 15-tap filter [1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1]
|
||||
static INLINE void apply_15_tap_filter(
|
||||
const uint16x8_t flat2, const uint16x8_t p7, const uint16x8_t p6,
|
||||
const uint16x8_t p5, const uint16x8_t p4, const uint16x8_t p3,
|
||||
const uint16x8_t p2, const uint16x8_t p1, const uint16x8_t p0,
|
||||
const uint16x8_t q0, const uint16x8_t q1, const uint16x8_t q2,
|
||||
const uint16x8_t q3, const uint16x8_t q4, const uint16x8_t q5,
|
||||
const uint16x8_t q6, const uint16x8_t q7, uint16x8_t *op6, uint16x8_t *op5,
|
||||
uint16x8_t *op4, uint16x8_t *op3, uint16x8_t *op2, uint16x8_t *op1,
|
||||
uint16x8_t *op0, uint16x8_t *oq0, uint16x8_t *oq1, uint16x8_t *oq2,
|
||||
uint16x8_t *oq3, uint16x8_t *oq4, uint16x8_t *oq5, uint16x8_t *oq6) {
|
||||
uint16x8_t sum;
|
||||
sum = vshlq_n_u16(p7, 3); // 8*p7
|
||||
sum = vsubq_u16(sum, p7); // 7*p7
|
||||
sum = vaddq_u16(sum, p6); // 7*p7+p6
|
||||
sum = vaddq_u16(sum, p6); // 7*p7+2*p6
|
||||
sum = vaddq_u16(sum, p5); // 7*p7+2*p6+p5
|
||||
sum = vaddq_u16(sum, p4); // 7*p7+2*p6+p5+p4
|
||||
sum = vaddq_u16(sum, p3); // 7*p7+2*p6+p5+p4+p3
|
||||
sum = vaddq_u16(sum, p2); // 7*p7+2*p6+p5+p4+p3+p2
|
||||
sum = vaddq_u16(sum, p1); // 7*p7+2*p6+p5+p4+p3+p2+p1
|
||||
sum = vaddq_u16(sum, p0); // 7*p7+2*p6+p5+p4+p3+p2+p1+p0
|
||||
sum = vaddq_u16(sum, q0); // 7*p7+2*p6+p5+p4+p3+p2+p1+p0+q0
|
||||
*op6 = vbslq_u16(flat2, vrshrq_n_u16(sum, 4), p6);
|
||||
*op5 = apply_15_tap_filter_kernel(flat2, p7, p6, p5, q1, p5, &sum);
|
||||
*op4 = apply_15_tap_filter_kernel(flat2, p7, p5, p4, q2, p4, &sum);
|
||||
*op3 = apply_15_tap_filter_kernel(flat2, p7, p4, p3, q3, p3, &sum);
|
||||
*op2 = apply_15_tap_filter_kernel(flat2, p7, p3, p2, q4, *op2, &sum);
|
||||
*op1 = apply_15_tap_filter_kernel(flat2, p7, p2, p1, q5, *op1, &sum);
|
||||
*op0 = apply_15_tap_filter_kernel(flat2, p7, p1, p0, q6, *op0, &sum);
|
||||
*oq0 = apply_15_tap_filter_kernel(flat2, p7, p0, q0, q7, *oq0, &sum);
|
||||
*oq1 = apply_15_tap_filter_kernel(flat2, p6, q0, q1, q7, *oq1, &sum);
|
||||
*oq2 = apply_15_tap_filter_kernel(flat2, p5, q1, q2, q7, *oq2, &sum);
|
||||
*oq3 = apply_15_tap_filter_kernel(flat2, p4, q2, q3, q7, q3, &sum);
|
||||
*oq4 = apply_15_tap_filter_kernel(flat2, p3, q3, q4, q7, q4, &sum);
|
||||
*oq5 = apply_15_tap_filter_kernel(flat2, p2, q4, q5, q7, q5, &sum);
|
||||
*oq6 = apply_15_tap_filter_kernel(flat2, p1, q5, q6, q7, q6, &sum);
|
||||
}
|
||||
|
||||
static INLINE void filter4(const uint16x8_t mask, const uint16x8_t hev,
|
||||
const uint16x8_t p1, const uint16x8_t p0,
|
||||
const uint16x8_t q0, const uint16x8_t q1,
|
||||
uint16x8_t *op1, uint16x8_t *op0, uint16x8_t *oq0,
|
||||
uint16x8_t *oq1, const int bd) {
|
||||
const int16x8_t max = vdupq_n_s16((1 << (bd - 1)) - 1);
|
||||
const int16x8_t min = vdupq_n_s16((int16_t)(((uint32_t)-1) << (bd - 1)));
|
||||
int16x8_t filter, filter1, filter2, t;
|
||||
int16x8_t ps1 = flip_sign(p1, bd);
|
||||
int16x8_t ps0 = flip_sign(p0, bd);
|
||||
int16x8_t qs0 = flip_sign(q0, bd);
|
||||
int16x8_t qs1 = flip_sign(q1, bd);
|
||||
|
||||
/* add outer taps if we have high edge variance */
|
||||
filter = vsubq_s16(ps1, qs1);
|
||||
filter = vmaxq_s16(filter, min);
|
||||
filter = vminq_s16(filter, max);
|
||||
filter = vandq_s16(filter, vreinterpretq_s16_u16(hev));
|
||||
t = vsubq_s16(qs0, ps0);
|
||||
|
||||
/* inner taps */
|
||||
filter = vaddq_s16(filter, t);
|
||||
filter = vaddq_s16(filter, t);
|
||||
filter = vaddq_s16(filter, t);
|
||||
filter = vmaxq_s16(filter, min);
|
||||
filter = vminq_s16(filter, max);
|
||||
filter = vandq_s16(filter, vreinterpretq_s16_u16(mask));
|
||||
|
||||
/* save bottom 3 bits so that we round one side +4 and the other +3 */
|
||||
/* if it equals 4 we'll set it to adjust by -1 to account for the fact */
|
||||
/* we'd round it by 3 the other way */
|
||||
t = vaddq_s16(filter, vdupq_n_s16(4));
|
||||
t = vminq_s16(t, max);
|
||||
filter1 = vshrq_n_s16(t, 3);
|
||||
t = vaddq_s16(filter, vdupq_n_s16(3));
|
||||
t = vminq_s16(t, max);
|
||||
filter2 = vshrq_n_s16(t, 3);
|
||||
|
||||
qs0 = vsubq_s16(qs0, filter1);
|
||||
qs0 = vmaxq_s16(qs0, min);
|
||||
qs0 = vminq_s16(qs0, max);
|
||||
ps0 = vaddq_s16(ps0, filter2);
|
||||
ps0 = vmaxq_s16(ps0, min);
|
||||
ps0 = vminq_s16(ps0, max);
|
||||
*oq0 = flip_sign_back(qs0, bd);
|
||||
*op0 = flip_sign_back(ps0, bd);
|
||||
|
||||
/* outer tap adjustments */
|
||||
filter = vrshrq_n_s16(filter1, 1);
|
||||
filter = vbicq_s16(filter, vreinterpretq_s16_u16(hev));
|
||||
|
||||
qs1 = vsubq_s16(qs1, filter);
|
||||
qs1 = vmaxq_s16(qs1, min);
|
||||
qs1 = vminq_s16(qs1, max);
|
||||
ps1 = vaddq_s16(ps1, filter);
|
||||
ps1 = vmaxq_s16(ps1, min);
|
||||
ps1 = vminq_s16(ps1, max);
|
||||
*oq1 = flip_sign_back(qs1, bd);
|
||||
*op1 = flip_sign_back(ps1, bd);
|
||||
}
|
||||
|
||||
static INLINE void filter8(const uint16x8_t mask, const uint16x8_t flat,
|
||||
const uint32_t flat_status, const uint16x8_t hev,
|
||||
const uint16x8_t p3, const uint16x8_t p2,
|
||||
const uint16x8_t p1, const uint16x8_t p0,
|
||||
const uint16x8_t q0, const uint16x8_t q1,
|
||||
const uint16x8_t q2, const uint16x8_t q3,
|
||||
uint16x8_t *op2, uint16x8_t *op1, uint16x8_t *op0,
|
||||
uint16x8_t *oq0, uint16x8_t *oq1, uint16x8_t *oq2,
|
||||
const int bd) {
|
||||
if (flat_status != (uint32_t)-4) {
|
||||
filter4(mask, hev, p1, p0, q0, q1, op1, op0, oq0, oq1, bd);
|
||||
*op2 = p2;
|
||||
*oq2 = q2;
|
||||
if (flat_status) {
|
||||
apply_7_tap_filter(flat, p3, p2, p1, p0, q0, q1, q2, q3, op2, op1, op0,
|
||||
oq0, oq1, oq2);
|
||||
}
|
||||
} else {
|
||||
calc_7_tap_filter(p3, p2, p1, p0, q0, q1, q2, q3, op2, op1, op0, oq0, oq1,
|
||||
oq2);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void filter16(
|
||||
const uint16x8_t mask, const uint16x8_t flat, const uint32_t flat_status,
|
||||
const uint16x8_t flat2, const uint32_t flat2_status, const uint16x8_t hev,
|
||||
const uint16x8_t p7, const uint16x8_t p6, const uint16x8_t p5,
|
||||
const uint16x8_t p4, const uint16x8_t p3, const uint16x8_t p2,
|
||||
const uint16x8_t p1, const uint16x8_t p0, const uint16x8_t q0,
|
||||
const uint16x8_t q1, const uint16x8_t q2, const uint16x8_t q3,
|
||||
const uint16x8_t q4, const uint16x8_t q5, const uint16x8_t q6,
|
||||
const uint16x8_t q7, uint16x8_t *op6, uint16x8_t *op5, uint16x8_t *op4,
|
||||
uint16x8_t *op3, uint16x8_t *op2, uint16x8_t *op1, uint16x8_t *op0,
|
||||
uint16x8_t *oq0, uint16x8_t *oq1, uint16x8_t *oq2, uint16x8_t *oq3,
|
||||
uint16x8_t *oq4, uint16x8_t *oq5, uint16x8_t *oq6, const int bd) {
|
||||
if (flat_status != (uint32_t)-4) {
|
||||
filter4(mask, hev, p1, p0, q0, q1, op1, op0, oq0, oq1, bd);
|
||||
}
|
||||
|
||||
if (flat_status) {
|
||||
*op2 = p2;
|
||||
*oq2 = q2;
|
||||
if (flat2_status != (uint32_t)-4) {
|
||||
apply_7_tap_filter(flat, p3, p2, p1, p0, q0, q1, q2, q3, op2, op1, op0,
|
||||
oq0, oq1, oq2);
|
||||
}
|
||||
if (flat2_status) {
|
||||
apply_15_tap_filter(flat2, p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3,
|
||||
q4, q5, q6, q7, op6, op5, op4, op3, op2, op1, op0,
|
||||
oq0, oq1, oq2, oq3, oq4, oq5, oq6);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void load_8x8(const uint16_t *s, const int p, uint16x8_t *p3,
|
||||
uint16x8_t *p2, uint16x8_t *p1, uint16x8_t *p0,
|
||||
uint16x8_t *q0, uint16x8_t *q1, uint16x8_t *q2,
|
||||
uint16x8_t *q3) {
|
||||
*p3 = vld1q_u16(s);
|
||||
s += p;
|
||||
*p2 = vld1q_u16(s);
|
||||
s += p;
|
||||
*p1 = vld1q_u16(s);
|
||||
s += p;
|
||||
*p0 = vld1q_u16(s);
|
||||
s += p;
|
||||
*q0 = vld1q_u16(s);
|
||||
s += p;
|
||||
*q1 = vld1q_u16(s);
|
||||
s += p;
|
||||
*q2 = vld1q_u16(s);
|
||||
s += p;
|
||||
*q3 = vld1q_u16(s);
|
||||
}
|
||||
|
||||
static INLINE void load_8x16(const uint16_t *s, const int p, uint16x8_t *s0,
|
||||
uint16x8_t *s1, uint16x8_t *s2, uint16x8_t *s3,
|
||||
uint16x8_t *s4, uint16x8_t *s5, uint16x8_t *s6,
|
||||
uint16x8_t *s7, uint16x8_t *s8, uint16x8_t *s9,
|
||||
uint16x8_t *s10, uint16x8_t *s11, uint16x8_t *s12,
|
||||
uint16x8_t *s13, uint16x8_t *s14,
|
||||
uint16x8_t *s15) {
|
||||
*s0 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s1 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s2 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s3 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s4 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s5 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s6 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s7 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s8 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s9 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s10 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s11 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s12 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s13 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s14 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s15 = vld1q_u16(s);
|
||||
}
|
||||
|
||||
static INLINE void store_8x4(uint16_t *s, const int p, const uint16x8_t s0,
|
||||
const uint16x8_t s1, const uint16x8_t s2,
|
||||
const uint16x8_t s3) {
|
||||
vst1q_u16(s, s0);
|
||||
s += p;
|
||||
vst1q_u16(s, s1);
|
||||
s += p;
|
||||
vst1q_u16(s, s2);
|
||||
s += p;
|
||||
vst1q_u16(s, s3);
|
||||
}
|
||||
|
||||
static INLINE void store_8x6(uint16_t *s, const int p, const uint16x8_t s0,
|
||||
const uint16x8_t s1, const uint16x8_t s2,
|
||||
const uint16x8_t s3, const uint16x8_t s4,
|
||||
const uint16x8_t s5) {
|
||||
vst1q_u16(s, s0);
|
||||
s += p;
|
||||
vst1q_u16(s, s1);
|
||||
s += p;
|
||||
vst1q_u16(s, s2);
|
||||
s += p;
|
||||
vst1q_u16(s, s3);
|
||||
s += p;
|
||||
vst1q_u16(s, s4);
|
||||
s += p;
|
||||
vst1q_u16(s, s5);
|
||||
}
|
||||
|
||||
static INLINE void store_4x8(uint16_t *s, const int p, const uint16x8_t p1,
|
||||
const uint16x8_t p0, const uint16x8_t q0,
|
||||
const uint16x8_t q1) {
|
||||
uint16x8x4_t o;
|
||||
|
||||
o.val[0] = p1;
|
||||
o.val[1] = p0;
|
||||
o.val[2] = q0;
|
||||
o.val[3] = q1;
|
||||
vst4q_lane_u16(s, o, 0);
|
||||
s += p;
|
||||
vst4q_lane_u16(s, o, 1);
|
||||
s += p;
|
||||
vst4q_lane_u16(s, o, 2);
|
||||
s += p;
|
||||
vst4q_lane_u16(s, o, 3);
|
||||
s += p;
|
||||
vst4q_lane_u16(s, o, 4);
|
||||
s += p;
|
||||
vst4q_lane_u16(s, o, 5);
|
||||
s += p;
|
||||
vst4q_lane_u16(s, o, 6);
|
||||
s += p;
|
||||
vst4q_lane_u16(s, o, 7);
|
||||
}
|
||||
|
||||
static INLINE void store_6x8(uint16_t *s, const int p, const uint16x8_t s0,
|
||||
const uint16x8_t s1, const uint16x8_t s2,
|
||||
const uint16x8_t s3, const uint16x8_t s4,
|
||||
const uint16x8_t s5) {
|
||||
uint16x8x3_t o0, o1;
|
||||
|
||||
o0.val[0] = s0;
|
||||
o0.val[1] = s1;
|
||||
o0.val[2] = s2;
|
||||
o1.val[0] = s3;
|
||||
o1.val[1] = s4;
|
||||
o1.val[2] = s5;
|
||||
vst3q_lane_u16(s - 3, o0, 0);
|
||||
vst3q_lane_u16(s + 0, o1, 0);
|
||||
s += p;
|
||||
vst3q_lane_u16(s - 3, o0, 1);
|
||||
vst3q_lane_u16(s + 0, o1, 1);
|
||||
s += p;
|
||||
vst3q_lane_u16(s - 3, o0, 2);
|
||||
vst3q_lane_u16(s + 0, o1, 2);
|
||||
s += p;
|
||||
vst3q_lane_u16(s - 3, o0, 3);
|
||||
vst3q_lane_u16(s + 0, o1, 3);
|
||||
s += p;
|
||||
vst3q_lane_u16(s - 3, o0, 4);
|
||||
vst3q_lane_u16(s + 0, o1, 4);
|
||||
s += p;
|
||||
vst3q_lane_u16(s - 3, o0, 5);
|
||||
vst3q_lane_u16(s + 0, o1, 5);
|
||||
s += p;
|
||||
vst3q_lane_u16(s - 3, o0, 6);
|
||||
vst3q_lane_u16(s + 0, o1, 6);
|
||||
s += p;
|
||||
vst3q_lane_u16(s - 3, o0, 7);
|
||||
vst3q_lane_u16(s + 0, o1, 7);
|
||||
}
|
||||
|
||||
static INLINE void store_7x8(uint16_t *s, const int p, const uint16x8_t s0,
|
||||
const uint16x8_t s1, const uint16x8_t s2,
|
||||
const uint16x8_t s3, const uint16x8_t s4,
|
||||
const uint16x8_t s5, const uint16x8_t s6) {
|
||||
uint16x8x4_t o0;
|
||||
uint16x8x3_t o1;
|
||||
|
||||
o0.val[0] = s0;
|
||||
o0.val[1] = s1;
|
||||
o0.val[2] = s2;
|
||||
o0.val[3] = s3;
|
||||
o1.val[0] = s4;
|
||||
o1.val[1] = s5;
|
||||
o1.val[2] = s6;
|
||||
vst4q_lane_u16(s - 4, o0, 0);
|
||||
vst3q_lane_u16(s + 0, o1, 0);
|
||||
s += p;
|
||||
vst4q_lane_u16(s - 4, o0, 1);
|
||||
vst3q_lane_u16(s + 0, o1, 1);
|
||||
s += p;
|
||||
vst4q_lane_u16(s - 4, o0, 2);
|
||||
vst3q_lane_u16(s + 0, o1, 2);
|
||||
s += p;
|
||||
vst4q_lane_u16(s - 4, o0, 3);
|
||||
vst3q_lane_u16(s + 0, o1, 3);
|
||||
s += p;
|
||||
vst4q_lane_u16(s - 4, o0, 4);
|
||||
vst3q_lane_u16(s + 0, o1, 4);
|
||||
s += p;
|
||||
vst4q_lane_u16(s - 4, o0, 5);
|
||||
vst3q_lane_u16(s + 0, o1, 5);
|
||||
s += p;
|
||||
vst4q_lane_u16(s - 4, o0, 6);
|
||||
vst3q_lane_u16(s + 0, o1, 6);
|
||||
s += p;
|
||||
vst4q_lane_u16(s - 4, o0, 7);
|
||||
vst3q_lane_u16(s + 0, o1, 7);
|
||||
}
|
||||
|
||||
static INLINE void store_8x14(uint16_t *s, const int p, const uint16x8_t p6,
|
||||
const uint16x8_t p5, const uint16x8_t p4,
|
||||
const uint16x8_t p3, const uint16x8_t p2,
|
||||
const uint16x8_t p1, const uint16x8_t p0,
|
||||
const uint16x8_t q0, const uint16x8_t q1,
|
||||
const uint16x8_t q2, const uint16x8_t q3,
|
||||
const uint16x8_t q4, const uint16x8_t q5,
|
||||
const uint16x8_t q6, const uint32_t flat_status,
|
||||
const uint32_t flat2_status) {
|
||||
if (flat_status) {
|
||||
if (flat2_status) {
|
||||
vst1q_u16(s - 7 * p, p6);
|
||||
vst1q_u16(s - 6 * p, p5);
|
||||
vst1q_u16(s - 5 * p, p4);
|
||||
vst1q_u16(s - 4 * p, p3);
|
||||
vst1q_u16(s + 3 * p, q3);
|
||||
vst1q_u16(s + 4 * p, q4);
|
||||
vst1q_u16(s + 5 * p, q5);
|
||||
vst1q_u16(s + 6 * p, q6);
|
||||
}
|
||||
vst1q_u16(s - 3 * p, p2);
|
||||
vst1q_u16(s + 2 * p, q2);
|
||||
}
|
||||
vst1q_u16(s - 2 * p, p1);
|
||||
vst1q_u16(s - 1 * p, p0);
|
||||
vst1q_u16(s + 0 * p, q0);
|
||||
vst1q_u16(s + 1 * p, q1);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_horizontal_4_neon(uint16_t *s, int p, const uint8_t *blimit,
|
||||
const uint8_t *limit,
|
||||
const uint8_t *thresh, int bd) {
|
||||
uint16x8_t blimit_vec, limit_vec, thresh_vec, p3, p2, p1, p0, q0, q1, q2, q3,
|
||||
mask, hev;
|
||||
|
||||
load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
|
||||
load_8x8(s - 4 * p, p, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3);
|
||||
filter_hev_mask4(limit_vec, blimit_vec, thresh_vec, p3, p2, p1, p0, q0, q1,
|
||||
q2, q3, &hev, &mask);
|
||||
filter4(mask, hev, p1, p0, q0, q1, &p1, &p0, &q0, &q1, bd);
|
||||
store_8x4(s - 2 * p, p, p1, p0, q0, q1);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_horizontal_4_dual_neon(
|
||||
uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0,
|
||||
const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1,
|
||||
const uint8_t *thresh1, int bd) {
|
||||
vpx_highbd_lpf_horizontal_4_neon(s, p, blimit0, limit0, thresh0, bd);
|
||||
vpx_highbd_lpf_horizontal_4_neon(s + 8, p, blimit1, limit1, thresh1, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_vertical_4_neon(uint16_t *s, int p, const uint8_t *blimit,
|
||||
const uint8_t *limit, const uint8_t *thresh,
|
||||
int bd) {
|
||||
uint16x8_t blimit_vec, limit_vec, thresh_vec, p3, p2, p1, p0, q0, q1, q2, q3,
|
||||
mask, hev;
|
||||
|
||||
load_8x8(s - 4, p, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3);
|
||||
transpose_s16_8x8((int16x8_t *)&p3, (int16x8_t *)&p2, (int16x8_t *)&p1,
|
||||
(int16x8_t *)&p0, (int16x8_t *)&q0, (int16x8_t *)&q1,
|
||||
(int16x8_t *)&q2, (int16x8_t *)&q3);
|
||||
load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
|
||||
filter_hev_mask4(limit_vec, blimit_vec, thresh_vec, p3, p2, p1, p0, q0, q1,
|
||||
q2, q3, &hev, &mask);
|
||||
filter4(mask, hev, p1, p0, q0, q1, &p1, &p0, &q0, &q1, bd);
|
||||
store_4x8(s - 2, p, p1, p0, q0, q1);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_vertical_4_dual_neon(
|
||||
uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0,
|
||||
const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1,
|
||||
const uint8_t *thresh1, int bd) {
|
||||
vpx_highbd_lpf_vertical_4_neon(s, p, blimit0, limit0, thresh0, bd);
|
||||
vpx_highbd_lpf_vertical_4_neon(s + 8 * p, p, blimit1, limit1, thresh1, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_horizontal_8_neon(uint16_t *s, int p, const uint8_t *blimit,
|
||||
const uint8_t *limit,
|
||||
const uint8_t *thresh, int bd) {
|
||||
uint16x8_t blimit_vec, limit_vec, thresh_vec, p3, p2, p1, p0, q0, q1, q2, q3,
|
||||
op2, op1, op0, oq0, oq1, oq2, mask, flat, hev;
|
||||
uint32_t flat_status;
|
||||
|
||||
load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
|
||||
load_8x8(s - 4 * p, p, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3);
|
||||
mask = filter_flat_hev_mask(limit_vec, blimit_vec, thresh_vec, p3, p2, p1, p0,
|
||||
q0, q1, q2, q3, &flat, &flat_status, &hev, bd);
|
||||
filter8(mask, flat, flat_status, hev, p3, p2, p1, p0, q0, q1, q2, q3, &op2,
|
||||
&op1, &op0, &oq0, &oq1, &oq2, bd);
|
||||
store_8x6(s - 3 * p, p, op2, op1, op0, oq0, oq1, oq2);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_horizontal_8_dual_neon(
|
||||
uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0,
|
||||
const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1,
|
||||
const uint8_t *thresh1, int bd) {
|
||||
vpx_highbd_lpf_horizontal_8_neon(s, p, blimit0, limit0, thresh0, bd);
|
||||
vpx_highbd_lpf_horizontal_8_neon(s + 8, p, blimit1, limit1, thresh1, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_vertical_8_neon(uint16_t *s, int p, const uint8_t *blimit,
|
||||
const uint8_t *limit, const uint8_t *thresh,
|
||||
int bd) {
|
||||
uint16x8_t blimit_vec, limit_vec, thresh_vec, p3, p2, p1, p0, q0, q1, q2, q3,
|
||||
op2, op1, op0, oq0, oq1, oq2, mask, flat, hev;
|
||||
uint32_t flat_status;
|
||||
|
||||
load_8x8(s - 4, p, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3);
|
||||
transpose_s16_8x8((int16x8_t *)&p3, (int16x8_t *)&p2, (int16x8_t *)&p1,
|
||||
(int16x8_t *)&p0, (int16x8_t *)&q0, (int16x8_t *)&q1,
|
||||
(int16x8_t *)&q2, (int16x8_t *)&q3);
|
||||
load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
|
||||
mask = filter_flat_hev_mask(limit_vec, blimit_vec, thresh_vec, p3, p2, p1, p0,
|
||||
q0, q1, q2, q3, &flat, &flat_status, &hev, bd);
|
||||
filter8(mask, flat, flat_status, hev, p3, p2, p1, p0, q0, q1, q2, q3, &op2,
|
||||
&op1, &op0, &oq0, &oq1, &oq2, bd);
|
||||
// Note: store_6x8() is faster than transpose + store_8x8().
|
||||
store_6x8(s, p, op2, op1, op0, oq0, oq1, oq2);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_vertical_8_dual_neon(
|
||||
uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0,
|
||||
const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1,
|
||||
const uint8_t *thresh1, int bd) {
|
||||
vpx_highbd_lpf_vertical_8_neon(s, p, blimit0, limit0, thresh0, bd);
|
||||
vpx_highbd_lpf_vertical_8_neon(s + 8 * p, p, blimit1, limit1, thresh1, bd);
|
||||
}
|
||||
|
||||
static void lpf_horizontal_16_kernel(uint16_t *s, int p,
|
||||
const uint16x8_t blimit_vec,
|
||||
const uint16x8_t limit_vec,
|
||||
const uint16x8_t thresh_vec,
|
||||
const int bd) {
|
||||
uint16x8_t mask, flat, flat2, hev, p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2,
|
||||
q3, q4, q5, q6, q7, op6, op5, op4, op3, op2, op1, op0, oq0, oq1, oq2, oq3,
|
||||
oq4, oq5, oq6;
|
||||
uint32_t flat_status, flat2_status;
|
||||
|
||||
load_8x16(s - 8 * p, p, &p7, &p6, &p5, &p4, &p3, &p2, &p1, &p0, &q0, &q1, &q2,
|
||||
&q3, &q4, &q5, &q6, &q7);
|
||||
mask = filter_flat_hev_mask(limit_vec, blimit_vec, thresh_vec, p3, p2, p1, p0,
|
||||
q0, q1, q2, q3, &flat, &flat_status, &hev, bd);
|
||||
flat2 = flat_mask5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat,
|
||||
&flat2_status, bd);
|
||||
filter16(mask, flat, flat_status, flat2, flat2_status, hev, p7, p6, p5, p4,
|
||||
p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7, &op6, &op5, &op4,
|
||||
&op3, &op2, &op1, &op0, &oq0, &oq1, &oq2, &oq3, &oq4, &oq5, &oq6,
|
||||
bd);
|
||||
store_8x14(s, p, op6, op5, op4, op3, op2, op1, op0, oq0, oq1, oq2, oq3, oq4,
|
||||
oq5, oq6, flat_status, flat2_status);
|
||||
}
|
||||
|
||||
static void lpf_vertical_16_kernel(uint16_t *s, int p,
|
||||
const uint16x8_t blimit_vec,
|
||||
const uint16x8_t limit_vec,
|
||||
const uint16x8_t thresh_vec, const int bd) {
|
||||
uint16x8_t mask, flat, flat2, hev, p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2,
|
||||
q3, q4, q5, q6, q7, op6, op5, op4, op3, op2, op1, op0, oq0, oq1, oq2, oq3,
|
||||
oq4, oq5, oq6;
|
||||
uint32_t flat_status, flat2_status;
|
||||
|
||||
load_8x8(s - 8, p, &p7, &p6, &p5, &p4, &p3, &p2, &p1, &p0);
|
||||
transpose_s16_8x8((int16x8_t *)&p7, (int16x8_t *)&p6, (int16x8_t *)&p5,
|
||||
(int16x8_t *)&p4, (int16x8_t *)&p3, (int16x8_t *)&p2,
|
||||
(int16x8_t *)&p1, (int16x8_t *)&p0);
|
||||
load_8x8(s, p, &q0, &q1, &q2, &q3, &q4, &q5, &q6, &q7);
|
||||
transpose_s16_8x8((int16x8_t *)&q0, (int16x8_t *)&q1, (int16x8_t *)&q2,
|
||||
(int16x8_t *)&q3, (int16x8_t *)&q4, (int16x8_t *)&q5,
|
||||
(int16x8_t *)&q6, (int16x8_t *)&q7);
|
||||
mask = filter_flat_hev_mask(limit_vec, blimit_vec, thresh_vec, p3, p2, p1, p0,
|
||||
q0, q1, q2, q3, &flat, &flat_status, &hev, bd);
|
||||
flat2 = flat_mask5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat,
|
||||
&flat2_status, bd);
|
||||
filter16(mask, flat, flat_status, flat2, flat2_status, hev, p7, p6, p5, p4,
|
||||
p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7, &op6, &op5, &op4,
|
||||
&op3, &op2, &op1, &op0, &oq0, &oq1, &oq2, &oq3, &oq4, &oq5, &oq6,
|
||||
bd);
|
||||
if (flat_status) {
|
||||
if (flat2_status) {
|
||||
store_7x8(s - 3, p, op6, op5, op4, op3, op2, op1, op0);
|
||||
store_7x8(s + 4, p, oq0, oq1, oq2, oq3, oq4, oq5, oq6);
|
||||
} else {
|
||||
// Note: store_6x8() is faster than transpose + store_8x8().
|
||||
store_6x8(s, p, op2, op1, op0, oq0, oq1, oq2);
|
||||
}
|
||||
} else {
|
||||
store_4x8(s - 2, p, op1, op0, oq0, oq1);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_horizontal_16_neon(uint16_t *s, int p,
|
||||
const uint8_t *blimit,
|
||||
const uint8_t *limit,
|
||||
const uint8_t *thresh, int bd) {
|
||||
uint16x8_t blimit_vec, limit_vec, thresh_vec;
|
||||
load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
|
||||
lpf_horizontal_16_kernel(s, p, blimit_vec, limit_vec, thresh_vec, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_horizontal_16_dual_neon(uint16_t *s, int p,
|
||||
const uint8_t *blimit,
|
||||
const uint8_t *limit,
|
||||
const uint8_t *thresh, int bd) {
|
||||
uint16x8_t blimit_vec, limit_vec, thresh_vec;
|
||||
load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
|
||||
lpf_horizontal_16_kernel(s, p, blimit_vec, limit_vec, thresh_vec, bd);
|
||||
lpf_horizontal_16_kernel(s + 8, p, blimit_vec, limit_vec, thresh_vec, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_vertical_16_neon(uint16_t *s, int p, const uint8_t *blimit,
|
||||
const uint8_t *limit,
|
||||
const uint8_t *thresh, int bd) {
|
||||
uint16x8_t blimit_vec, limit_vec, thresh_vec;
|
||||
load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
|
||||
lpf_vertical_16_kernel(s, p, blimit_vec, limit_vec, thresh_vec, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_lpf_vertical_16_dual_neon(uint16_t *s, int p,
|
||||
const uint8_t *blimit,
|
||||
const uint8_t *limit,
|
||||
const uint8_t *thresh, int bd) {
|
||||
uint16x8_t blimit_vec, limit_vec, thresh_vec;
|
||||
load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
|
||||
lpf_vertical_16_kernel(s, p, blimit_vec, limit_vec, thresh_vec, bd);
|
||||
lpf_vertical_16_kernel(s + 8 * p, p, blimit_vec, limit_vec, thresh_vec, bd);
|
||||
}
|
||||
@@ -0,0 +1,931 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
|
||||
static INLINE void load_4x4(const int16_t *s, const ptrdiff_t p,
|
||||
int16x4_t *const s0, int16x4_t *const s1,
|
||||
int16x4_t *const s2, int16x4_t *const s3) {
|
||||
*s0 = vld1_s16(s);
|
||||
s += p;
|
||||
*s1 = vld1_s16(s);
|
||||
s += p;
|
||||
*s2 = vld1_s16(s);
|
||||
s += p;
|
||||
*s3 = vld1_s16(s);
|
||||
}
|
||||
|
||||
static INLINE void load_8x4(const uint16_t *s, const ptrdiff_t p,
|
||||
uint16x8_t *const s0, uint16x8_t *const s1,
|
||||
uint16x8_t *const s2, uint16x8_t *const s3) {
|
||||
*s0 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s1 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s2 = vld1q_u16(s);
|
||||
s += p;
|
||||
*s3 = vld1q_u16(s);
|
||||
}
|
||||
|
||||
static INLINE void load_8x8(const int16_t *s, const ptrdiff_t p,
|
||||
int16x8_t *const s0, int16x8_t *const s1,
|
||||
int16x8_t *const s2, int16x8_t *const s3,
|
||||
int16x8_t *const s4, int16x8_t *const s5,
|
||||
int16x8_t *const s6, int16x8_t *const s7) {
|
||||
*s0 = vld1q_s16(s);
|
||||
s += p;
|
||||
*s1 = vld1q_s16(s);
|
||||
s += p;
|
||||
*s2 = vld1q_s16(s);
|
||||
s += p;
|
||||
*s3 = vld1q_s16(s);
|
||||
s += p;
|
||||
*s4 = vld1q_s16(s);
|
||||
s += p;
|
||||
*s5 = vld1q_s16(s);
|
||||
s += p;
|
||||
*s6 = vld1q_s16(s);
|
||||
s += p;
|
||||
*s7 = vld1q_s16(s);
|
||||
}
|
||||
|
||||
static INLINE void store_8x8(uint16_t *s, const ptrdiff_t p,
|
||||
const uint16x8_t s0, const uint16x8_t s1,
|
||||
const uint16x8_t s2, const uint16x8_t s3,
|
||||
const uint16x8_t s4, const uint16x8_t s5,
|
||||
const uint16x8_t s6, const uint16x8_t s7) {
|
||||
vst1q_u16(s, s0);
|
||||
s += p;
|
||||
vst1q_u16(s, s1);
|
||||
s += p;
|
||||
vst1q_u16(s, s2);
|
||||
s += p;
|
||||
vst1q_u16(s, s3);
|
||||
s += p;
|
||||
vst1q_u16(s, s4);
|
||||
s += p;
|
||||
vst1q_u16(s, s5);
|
||||
s += p;
|
||||
vst1q_u16(s, s6);
|
||||
s += p;
|
||||
vst1q_u16(s, s7);
|
||||
}
|
||||
|
||||
static INLINE int32x4_t highbd_convolve8_4(
|
||||
const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
|
||||
const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
|
||||
const int16x4_t s6, const int16x4_t s7, const int16x8_t filters) {
|
||||
const int16x4_t filters_lo = vget_low_s16(filters);
|
||||
const int16x4_t filters_hi = vget_high_s16(filters);
|
||||
int32x4_t sum;
|
||||
|
||||
sum = vmull_lane_s16(s0, filters_lo, 0);
|
||||
sum = vmlal_lane_s16(sum, s1, filters_lo, 1);
|
||||
sum = vmlal_lane_s16(sum, s2, filters_lo, 2);
|
||||
sum = vmlal_lane_s16(sum, s3, filters_lo, 3);
|
||||
sum = vmlal_lane_s16(sum, s4, filters_hi, 0);
|
||||
sum = vmlal_lane_s16(sum, s5, filters_hi, 1);
|
||||
sum = vmlal_lane_s16(sum, s6, filters_hi, 2);
|
||||
sum = vmlal_lane_s16(sum, s7, filters_hi, 3);
|
||||
return sum;
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t
|
||||
highbd_convolve8_8(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
|
||||
const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
|
||||
const int16x8_t s6, const int16x8_t s7,
|
||||
const int16x8_t filters, const uint16x8_t max) {
|
||||
const int16x4_t filters_lo = vget_low_s16(filters);
|
||||
const int16x4_t filters_hi = vget_high_s16(filters);
|
||||
int32x4_t sum0, sum1;
|
||||
uint16x8_t d;
|
||||
|
||||
sum0 = vmull_lane_s16(vget_low_s16(s0), filters_lo, 0);
|
||||
sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), filters_lo, 1);
|
||||
sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), filters_lo, 2);
|
||||
sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), filters_lo, 3);
|
||||
sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), filters_hi, 0);
|
||||
sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), filters_hi, 1);
|
||||
sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), filters_hi, 2);
|
||||
sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), filters_hi, 3);
|
||||
sum1 = vmull_lane_s16(vget_high_s16(s0), filters_lo, 0);
|
||||
sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), filters_lo, 1);
|
||||
sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), filters_lo, 2);
|
||||
sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), filters_lo, 3);
|
||||
sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), filters_hi, 0);
|
||||
sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), filters_hi, 1);
|
||||
sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), filters_hi, 2);
|
||||
sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), filters_hi, 3);
|
||||
d = vcombine_u16(vqrshrun_n_s32(sum0, 7), vqrshrun_n_s32(sum1, 7));
|
||||
d = vminq_u16(d, max);
|
||||
return d;
|
||||
}
|
||||
|
||||
void vpx_highbd_convolve8_horiz_neon(const uint16_t *src, ptrdiff_t src_stride,
|
||||
uint16_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h, int bd) {
|
||||
if (x_step_q4 != 16) {
|
||||
vpx_highbd_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter,
|
||||
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h, bd);
|
||||
} else {
|
||||
const int16x8_t filters = vld1q_s16(filter[x0_q4]);
|
||||
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
|
||||
uint16x8_t t0, t1, t2, t3;
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
|
||||
src -= 3;
|
||||
|
||||
if (h == 4) {
|
||||
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
int32x4_t d0, d1, d2, d3;
|
||||
uint16x8_t d01, d23;
|
||||
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
load_8x4(src, src_stride, &t0, &t1, &t2, &t3);
|
||||
transpose_u16_8x4(&t0, &t1, &t2, &t3);
|
||||
s0 = vreinterpret_s16_u16(vget_low_u16(t0));
|
||||
s1 = vreinterpret_s16_u16(vget_low_u16(t1));
|
||||
s2 = vreinterpret_s16_u16(vget_low_u16(t2));
|
||||
s3 = vreinterpret_s16_u16(vget_low_u16(t3));
|
||||
s4 = vreinterpret_s16_u16(vget_high_u16(t0));
|
||||
s5 = vreinterpret_s16_u16(vget_high_u16(t1));
|
||||
s6 = vreinterpret_s16_u16(vget_high_u16(t2));
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
src += 7;
|
||||
|
||||
do {
|
||||
load_4x4((const int16_t *)src, src_stride, &s7, &s8, &s9, &s10);
|
||||
transpose_s16_4x4d(&s7, &s8, &s9, &s10);
|
||||
|
||||
d0 = highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters);
|
||||
d1 = highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters);
|
||||
d2 = highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters);
|
||||
d3 = highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters);
|
||||
|
||||
d01 = vcombine_u16(vqrshrun_n_s32(d0, 7), vqrshrun_n_s32(d1, 7));
|
||||
d23 = vcombine_u16(vqrshrun_n_s32(d2, 7), vqrshrun_n_s32(d3, 7));
|
||||
d01 = vminq_u16(d01, max);
|
||||
d23 = vminq_u16(d23, max);
|
||||
transpose_u16_4x4q(&d01, &d23);
|
||||
|
||||
vst1_u16(dst + 0 * dst_stride, vget_low_u16(d01));
|
||||
vst1_u16(dst + 1 * dst_stride, vget_low_u16(d23));
|
||||
vst1_u16(dst + 2 * dst_stride, vget_high_u16(d01));
|
||||
vst1_u16(dst + 3 * dst_stride, vget_high_u16(d23));
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
src += 4;
|
||||
dst += 4;
|
||||
w -= 4;
|
||||
} while (w > 0);
|
||||
} else {
|
||||
int16x8_t t4, t5, t6, t7;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
uint16x8_t d0, d1, d2, d3;
|
||||
|
||||
if (w == 4) {
|
||||
do {
|
||||
load_8x8((const int16_t *)src, src_stride, &s0, &s1, &s2, &s3, &s4,
|
||||
&s5, &s6, &s7);
|
||||
transpose_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
|
||||
|
||||
load_8x8((const int16_t *)(src + 7), src_stride, &s7, &s8, &s9, &s10,
|
||||
&t4, &t5, &t6, &t7);
|
||||
src += 8 * src_stride;
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(dst + 4 * dst_stride);
|
||||
__builtin_prefetch(dst + 5 * dst_stride);
|
||||
__builtin_prefetch(dst + 6 * dst_stride);
|
||||
__builtin_prefetch(dst + 7 * dst_stride);
|
||||
transpose_s16_8x8(&s7, &s8, &s9, &s10, &t4, &t5, &t6, &t7);
|
||||
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
__builtin_prefetch(src + 7 * src_stride);
|
||||
d0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max);
|
||||
d1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max);
|
||||
d2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max);
|
||||
d3 =
|
||||
highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max);
|
||||
|
||||
transpose_u16_8x4(&d0, &d1, &d2, &d3);
|
||||
vst1_u16(dst, vget_low_u16(d0));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_low_u16(d1));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_low_u16(d2));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_low_u16(d3));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d0));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d1));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d2));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d3));
|
||||
dst += dst_stride;
|
||||
h -= 8;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
int width;
|
||||
const uint16_t *s;
|
||||
uint16_t *d;
|
||||
int16x8_t s11, s12, s13, s14;
|
||||
uint16x8_t d4, d5, d6, d7;
|
||||
|
||||
do {
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
__builtin_prefetch(src + 7 * src_stride);
|
||||
load_8x8((const int16_t *)src, src_stride, &s0, &s1, &s2, &s3, &s4,
|
||||
&s5, &s6, &s7);
|
||||
transpose_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
|
||||
|
||||
width = w;
|
||||
s = src + 7;
|
||||
d = dst;
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(dst + 4 * dst_stride);
|
||||
__builtin_prefetch(dst + 5 * dst_stride);
|
||||
__builtin_prefetch(dst + 6 * dst_stride);
|
||||
__builtin_prefetch(dst + 7 * dst_stride);
|
||||
|
||||
do {
|
||||
load_8x8((const int16_t *)s, src_stride, &s7, &s8, &s9, &s10, &s11,
|
||||
&s12, &s13, &s14);
|
||||
transpose_s16_8x8(&s7, &s8, &s9, &s10, &s11, &s12, &s13, &s14);
|
||||
|
||||
d0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters,
|
||||
max);
|
||||
d1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters,
|
||||
max);
|
||||
d2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters,
|
||||
max);
|
||||
d3 = highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters,
|
||||
max);
|
||||
d4 = highbd_convolve8_8(s4, s5, s6, s7, s8, s9, s10, s11, filters,
|
||||
max);
|
||||
d5 = highbd_convolve8_8(s5, s6, s7, s8, s9, s10, s11, s12, filters,
|
||||
max);
|
||||
d6 = highbd_convolve8_8(s6, s7, s8, s9, s10, s11, s12, s13, filters,
|
||||
max);
|
||||
d7 = highbd_convolve8_8(s7, s8, s9, s10, s11, s12, s13, s14,
|
||||
filters, max);
|
||||
|
||||
transpose_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
|
||||
store_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
|
||||
|
||||
s0 = s8;
|
||||
s1 = s9;
|
||||
s2 = s10;
|
||||
s3 = s11;
|
||||
s4 = s12;
|
||||
s5 = s13;
|
||||
s6 = s14;
|
||||
s += 8;
|
||||
d += 8;
|
||||
width -= 8;
|
||||
} while (width > 0);
|
||||
src += 8 * src_stride;
|
||||
dst += 8 * dst_stride;
|
||||
h -= 8;
|
||||
} while (h > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_highbd_convolve8_avg_horiz_neon(const uint16_t *src,
|
||||
ptrdiff_t src_stride, uint16_t *dst,
|
||||
ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4,
|
||||
int y_step_q4, int w, int h, int bd) {
|
||||
if (x_step_q4 != 16) {
|
||||
vpx_highbd_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter,
|
||||
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h,
|
||||
bd);
|
||||
} else {
|
||||
const int16x8_t filters = vld1q_s16(filter[x0_q4]);
|
||||
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
|
||||
uint16x8_t t0, t1, t2, t3;
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
|
||||
src -= 3;
|
||||
|
||||
if (h == 4) {
|
||||
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
int32x4_t d0, d1, d2, d3;
|
||||
uint16x8_t d01, d23, t01, t23;
|
||||
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
load_8x4(src, src_stride, &t0, &t1, &t2, &t3);
|
||||
transpose_u16_8x4(&t0, &t1, &t2, &t3);
|
||||
s0 = vreinterpret_s16_u16(vget_low_u16(t0));
|
||||
s1 = vreinterpret_s16_u16(vget_low_u16(t1));
|
||||
s2 = vreinterpret_s16_u16(vget_low_u16(t2));
|
||||
s3 = vreinterpret_s16_u16(vget_low_u16(t3));
|
||||
s4 = vreinterpret_s16_u16(vget_high_u16(t0));
|
||||
s5 = vreinterpret_s16_u16(vget_high_u16(t1));
|
||||
s6 = vreinterpret_s16_u16(vget_high_u16(t2));
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
src += 7;
|
||||
|
||||
do {
|
||||
load_4x4((const int16_t *)src, src_stride, &s7, &s8, &s9, &s10);
|
||||
transpose_s16_4x4d(&s7, &s8, &s9, &s10);
|
||||
|
||||
d0 = highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters);
|
||||
d1 = highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters);
|
||||
d2 = highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters);
|
||||
d3 = highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters);
|
||||
|
||||
t01 = vcombine_u16(vqrshrun_n_s32(d0, 7), vqrshrun_n_s32(d1, 7));
|
||||
t23 = vcombine_u16(vqrshrun_n_s32(d2, 7), vqrshrun_n_s32(d3, 7));
|
||||
t01 = vminq_u16(t01, max);
|
||||
t23 = vminq_u16(t23, max);
|
||||
transpose_u16_4x4q(&t01, &t23);
|
||||
|
||||
d01 = vcombine_u16(vld1_u16(dst + 0 * dst_stride),
|
||||
vld1_u16(dst + 2 * dst_stride));
|
||||
d23 = vcombine_u16(vld1_u16(dst + 1 * dst_stride),
|
||||
vld1_u16(dst + 3 * dst_stride));
|
||||
d01 = vrhaddq_u16(d01, t01);
|
||||
d23 = vrhaddq_u16(d23, t23);
|
||||
|
||||
vst1_u16(dst + 0 * dst_stride, vget_low_u16(d01));
|
||||
vst1_u16(dst + 1 * dst_stride, vget_low_u16(d23));
|
||||
vst1_u16(dst + 2 * dst_stride, vget_high_u16(d01));
|
||||
vst1_u16(dst + 3 * dst_stride, vget_high_u16(d23));
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
src += 4;
|
||||
dst += 4;
|
||||
w -= 4;
|
||||
} while (w > 0);
|
||||
} else {
|
||||
int16x8_t t4, t5, t6, t7;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
uint16x8_t d0, d1, d2, d3, t0, t1, t2, t3;
|
||||
|
||||
if (w == 4) {
|
||||
do {
|
||||
load_8x8((const int16_t *)src, src_stride, &s0, &s1, &s2, &s3, &s4,
|
||||
&s5, &s6, &s7);
|
||||
transpose_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
|
||||
|
||||
load_8x8((const int16_t *)(src + 7), src_stride, &s7, &s8, &s9, &s10,
|
||||
&t4, &t5, &t6, &t7);
|
||||
src += 8 * src_stride;
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(dst + 4 * dst_stride);
|
||||
__builtin_prefetch(dst + 5 * dst_stride);
|
||||
__builtin_prefetch(dst + 6 * dst_stride);
|
||||
__builtin_prefetch(dst + 7 * dst_stride);
|
||||
transpose_s16_8x8(&s7, &s8, &s9, &s10, &t4, &t5, &t6, &t7);
|
||||
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
__builtin_prefetch(src + 7 * src_stride);
|
||||
t0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max);
|
||||
t1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max);
|
||||
t2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max);
|
||||
t3 =
|
||||
highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max);
|
||||
transpose_u16_8x4(&t0, &t1, &t2, &t3);
|
||||
|
||||
d0 = vcombine_u16(vld1_u16(dst + 0 * dst_stride),
|
||||
vld1_u16(dst + 4 * dst_stride));
|
||||
d1 = vcombine_u16(vld1_u16(dst + 1 * dst_stride),
|
||||
vld1_u16(dst + 5 * dst_stride));
|
||||
d2 = vcombine_u16(vld1_u16(dst + 2 * dst_stride),
|
||||
vld1_u16(dst + 6 * dst_stride));
|
||||
d3 = vcombine_u16(vld1_u16(dst + 3 * dst_stride),
|
||||
vld1_u16(dst + 7 * dst_stride));
|
||||
d0 = vrhaddq_u16(d0, t0);
|
||||
d1 = vrhaddq_u16(d1, t1);
|
||||
d2 = vrhaddq_u16(d2, t2);
|
||||
d3 = vrhaddq_u16(d3, t3);
|
||||
|
||||
vst1_u16(dst, vget_low_u16(d0));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_low_u16(d1));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_low_u16(d2));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_low_u16(d3));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d0));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d1));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d2));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d3));
|
||||
dst += dst_stride;
|
||||
h -= 8;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
int width;
|
||||
const uint16_t *s;
|
||||
uint16_t *d;
|
||||
int16x8_t s11, s12, s13, s14;
|
||||
uint16x8_t d4, d5, d6, d7;
|
||||
|
||||
do {
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
__builtin_prefetch(src + 7 * src_stride);
|
||||
load_8x8((const int16_t *)src, src_stride, &s0, &s1, &s2, &s3, &s4,
|
||||
&s5, &s6, &s7);
|
||||
transpose_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
|
||||
|
||||
width = w;
|
||||
s = src + 7;
|
||||
d = dst;
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(dst + 4 * dst_stride);
|
||||
__builtin_prefetch(dst + 5 * dst_stride);
|
||||
__builtin_prefetch(dst + 6 * dst_stride);
|
||||
__builtin_prefetch(dst + 7 * dst_stride);
|
||||
|
||||
do {
|
||||
load_8x8((const int16_t *)s, src_stride, &s7, &s8, &s9, &s10, &s11,
|
||||
&s12, &s13, &s14);
|
||||
transpose_s16_8x8(&s7, &s8, &s9, &s10, &s11, &s12, &s13, &s14);
|
||||
|
||||
d0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters,
|
||||
max);
|
||||
d1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters,
|
||||
max);
|
||||
d2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters,
|
||||
max);
|
||||
d3 = highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters,
|
||||
max);
|
||||
d4 = highbd_convolve8_8(s4, s5, s6, s7, s8, s9, s10, s11, filters,
|
||||
max);
|
||||
d5 = highbd_convolve8_8(s5, s6, s7, s8, s9, s10, s11, s12, filters,
|
||||
max);
|
||||
d6 = highbd_convolve8_8(s6, s7, s8, s9, s10, s11, s12, s13, filters,
|
||||
max);
|
||||
d7 = highbd_convolve8_8(s7, s8, s9, s10, s11, s12, s13, s14,
|
||||
filters, max);
|
||||
|
||||
transpose_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
|
||||
|
||||
d0 = vrhaddq_u16(d0, vld1q_u16(d + 0 * dst_stride));
|
||||
d1 = vrhaddq_u16(d1, vld1q_u16(d + 1 * dst_stride));
|
||||
d2 = vrhaddq_u16(d2, vld1q_u16(d + 2 * dst_stride));
|
||||
d3 = vrhaddq_u16(d3, vld1q_u16(d + 3 * dst_stride));
|
||||
d4 = vrhaddq_u16(d4, vld1q_u16(d + 4 * dst_stride));
|
||||
d5 = vrhaddq_u16(d5, vld1q_u16(d + 5 * dst_stride));
|
||||
d6 = vrhaddq_u16(d6, vld1q_u16(d + 6 * dst_stride));
|
||||
d7 = vrhaddq_u16(d7, vld1q_u16(d + 7 * dst_stride));
|
||||
|
||||
store_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
|
||||
|
||||
s0 = s8;
|
||||
s1 = s9;
|
||||
s2 = s10;
|
||||
s3 = s11;
|
||||
s4 = s12;
|
||||
s5 = s13;
|
||||
s6 = s14;
|
||||
s += 8;
|
||||
d += 8;
|
||||
width -= 8;
|
||||
} while (width > 0);
|
||||
src += 8 * src_stride;
|
||||
dst += 8 * dst_stride;
|
||||
h -= 8;
|
||||
} while (h > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_highbd_convolve8_vert_neon(const uint16_t *src, ptrdiff_t src_stride,
|
||||
uint16_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h, int bd) {
|
||||
if (y_step_q4 != 16) {
|
||||
vpx_highbd_convolve8_vert_c(src, src_stride, dst, dst_stride, filter, x0_q4,
|
||||
x_step_q4, y0_q4, y_step_q4, w, h, bd);
|
||||
} else {
|
||||
const int16x8_t filters = vld1q_s16(filter[y0_q4]);
|
||||
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
|
||||
src -= 3 * src_stride;
|
||||
|
||||
if (w == 4) {
|
||||
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
int32x4_t d0, d1, d2, d3;
|
||||
uint16x8_t d01, d23;
|
||||
|
||||
s0 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s1 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s2 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s3 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s4 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s5 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s6 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
|
||||
do {
|
||||
s7 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s8 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s9 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s10 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
d0 = highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters);
|
||||
d1 = highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters);
|
||||
d2 = highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters);
|
||||
d3 = highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters);
|
||||
|
||||
d01 = vcombine_u16(vqrshrun_n_s32(d0, 7), vqrshrun_n_s32(d1, 7));
|
||||
d23 = vcombine_u16(vqrshrun_n_s32(d2, 7), vqrshrun_n_s32(d3, 7));
|
||||
d01 = vminq_u16(d01, max);
|
||||
d23 = vminq_u16(d23, max);
|
||||
vst1_u16(dst, vget_low_u16(d01));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d01));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_low_u16(d23));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d23));
|
||||
dst += dst_stride;
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
int height;
|
||||
const uint16_t *s;
|
||||
uint16_t *d;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
uint16x8_t d0, d1, d2, d3;
|
||||
|
||||
do {
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
s = src;
|
||||
s0 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s1 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s2 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s3 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s4 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s5 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s6 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
d = dst;
|
||||
height = h;
|
||||
|
||||
do {
|
||||
s7 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s8 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s9 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s10 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
|
||||
__builtin_prefetch(d + 0 * dst_stride);
|
||||
__builtin_prefetch(d + 1 * dst_stride);
|
||||
__builtin_prefetch(d + 2 * dst_stride);
|
||||
__builtin_prefetch(d + 3 * dst_stride);
|
||||
__builtin_prefetch(s + 0 * src_stride);
|
||||
__builtin_prefetch(s + 1 * src_stride);
|
||||
__builtin_prefetch(s + 2 * src_stride);
|
||||
__builtin_prefetch(s + 3 * src_stride);
|
||||
d0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max);
|
||||
d1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max);
|
||||
d2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max);
|
||||
d3 =
|
||||
highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max);
|
||||
|
||||
vst1q_u16(d, d0);
|
||||
d += dst_stride;
|
||||
vst1q_u16(d, d1);
|
||||
d += dst_stride;
|
||||
vst1q_u16(d, d2);
|
||||
d += dst_stride;
|
||||
vst1q_u16(d, d3);
|
||||
d += dst_stride;
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
height -= 4;
|
||||
} while (height > 0);
|
||||
src += 8;
|
||||
dst += 8;
|
||||
w -= 8;
|
||||
} while (w > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_highbd_convolve8_avg_vert_neon(const uint16_t *src,
|
||||
ptrdiff_t src_stride, uint16_t *dst,
|
||||
ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h, int bd) {
|
||||
if (y_step_q4 != 16) {
|
||||
vpx_highbd_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter,
|
||||
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h,
|
||||
bd);
|
||||
} else {
|
||||
const int16x8_t filters = vld1q_s16(filter[y0_q4]);
|
||||
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
|
||||
src -= 3 * src_stride;
|
||||
|
||||
if (w == 4) {
|
||||
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
int32x4_t d0, d1, d2, d3;
|
||||
uint16x8_t d01, d23, t01, t23;
|
||||
|
||||
s0 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s1 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s2 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s3 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s4 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s5 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s6 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
|
||||
do {
|
||||
s7 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s8 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s9 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
s10 = vreinterpret_s16_u16(vld1_u16(src));
|
||||
src += src_stride;
|
||||
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
d0 = highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters);
|
||||
d1 = highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters);
|
||||
d2 = highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters);
|
||||
d3 = highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters);
|
||||
|
||||
t01 = vcombine_u16(vqrshrun_n_s32(d0, 7), vqrshrun_n_s32(d1, 7));
|
||||
t23 = vcombine_u16(vqrshrun_n_s32(d2, 7), vqrshrun_n_s32(d3, 7));
|
||||
t01 = vminq_u16(t01, max);
|
||||
t23 = vminq_u16(t23, max);
|
||||
|
||||
d01 = vcombine_u16(vld1_u16(dst + 0 * dst_stride),
|
||||
vld1_u16(dst + 1 * dst_stride));
|
||||
d23 = vcombine_u16(vld1_u16(dst + 2 * dst_stride),
|
||||
vld1_u16(dst + 3 * dst_stride));
|
||||
d01 = vrhaddq_u16(d01, t01);
|
||||
d23 = vrhaddq_u16(d23, t23);
|
||||
|
||||
vst1_u16(dst, vget_low_u16(d01));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d01));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_low_u16(d23));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d23));
|
||||
dst += dst_stride;
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
int height;
|
||||
const uint16_t *s;
|
||||
uint16_t *d;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
uint16x8_t d0, d1, d2, d3, t0, t1, t2, t3;
|
||||
|
||||
do {
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
s = src;
|
||||
s0 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s1 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s2 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s3 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s4 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s5 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s6 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
d = dst;
|
||||
height = h;
|
||||
|
||||
do {
|
||||
s7 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s8 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s9 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
s10 = vreinterpretq_s16_u16(vld1q_u16(s));
|
||||
s += src_stride;
|
||||
|
||||
__builtin_prefetch(d + 0 * dst_stride);
|
||||
__builtin_prefetch(d + 1 * dst_stride);
|
||||
__builtin_prefetch(d + 2 * dst_stride);
|
||||
__builtin_prefetch(d + 3 * dst_stride);
|
||||
__builtin_prefetch(s + 0 * src_stride);
|
||||
__builtin_prefetch(s + 1 * src_stride);
|
||||
__builtin_prefetch(s + 2 * src_stride);
|
||||
__builtin_prefetch(s + 3 * src_stride);
|
||||
t0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, max);
|
||||
t1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, max);
|
||||
t2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, max);
|
||||
t3 =
|
||||
highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, max);
|
||||
|
||||
d0 = vld1q_u16(d + 0 * dst_stride);
|
||||
d1 = vld1q_u16(d + 1 * dst_stride);
|
||||
d2 = vld1q_u16(d + 2 * dst_stride);
|
||||
d3 = vld1q_u16(d + 3 * dst_stride);
|
||||
d0 = vrhaddq_u16(d0, t0);
|
||||
d1 = vrhaddq_u16(d1, t1);
|
||||
d2 = vrhaddq_u16(d2, t2);
|
||||
d3 = vrhaddq_u16(d3, t3);
|
||||
|
||||
vst1q_u16(d, d0);
|
||||
d += dst_stride;
|
||||
vst1q_u16(d, d1);
|
||||
d += dst_stride;
|
||||
vst1q_u16(d, d2);
|
||||
d += dst_stride;
|
||||
vst1q_u16(d, d3);
|
||||
d += dst_stride;
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
height -= 4;
|
||||
} while (height > 0);
|
||||
src += 8;
|
||||
dst += 8;
|
||||
w -= 8;
|
||||
} while (w > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
|
||||
void vpx_highbd_convolve_avg_neon(const uint16_t *src, ptrdiff_t src_stride,
|
||||
uint16_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h, int bd) {
|
||||
(void)filter;
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
(void)bd;
|
||||
|
||||
if (w < 8) { // avg4
|
||||
uint16x4_t s0, s1, d0, d1;
|
||||
uint16x8_t s01, d01;
|
||||
do {
|
||||
s0 = vld1_u16(src);
|
||||
d0 = vld1_u16(dst);
|
||||
src += src_stride;
|
||||
s1 = vld1_u16(src);
|
||||
d1 = vld1_u16(dst + dst_stride);
|
||||
src += src_stride;
|
||||
s01 = vcombine_u16(s0, s1);
|
||||
d01 = vcombine_u16(d0, d1);
|
||||
d01 = vrhaddq_u16(s01, d01);
|
||||
vst1_u16(dst, vget_low_u16(d01));
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vget_high_u16(d01));
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w == 8) { // avg8
|
||||
uint16x8_t s0, s1, d0, d1;
|
||||
do {
|
||||
s0 = vld1q_u16(src);
|
||||
d0 = vld1q_u16(dst);
|
||||
src += src_stride;
|
||||
s1 = vld1q_u16(src);
|
||||
d1 = vld1q_u16(dst + dst_stride);
|
||||
src += src_stride;
|
||||
|
||||
d0 = vrhaddq_u16(s0, d0);
|
||||
d1 = vrhaddq_u16(s1, d1);
|
||||
|
||||
vst1q_u16(dst, d0);
|
||||
dst += dst_stride;
|
||||
vst1q_u16(dst, d1);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w < 32) { // avg16
|
||||
uint16x8_t s0l, s0h, s1l, s1h, d0l, d0h, d1l, d1h;
|
||||
do {
|
||||
s0l = vld1q_u16(src);
|
||||
s0h = vld1q_u16(src + 8);
|
||||
d0l = vld1q_u16(dst);
|
||||
d0h = vld1q_u16(dst + 8);
|
||||
src += src_stride;
|
||||
s1l = vld1q_u16(src);
|
||||
s1h = vld1q_u16(src + 8);
|
||||
d1l = vld1q_u16(dst + dst_stride);
|
||||
d1h = vld1q_u16(dst + dst_stride + 8);
|
||||
src += src_stride;
|
||||
|
||||
d0l = vrhaddq_u16(s0l, d0l);
|
||||
d0h = vrhaddq_u16(s0h, d0h);
|
||||
d1l = vrhaddq_u16(s1l, d1l);
|
||||
d1h = vrhaddq_u16(s1h, d1h);
|
||||
|
||||
vst1q_u16(dst, d0l);
|
||||
vst1q_u16(dst + 8, d0h);
|
||||
dst += dst_stride;
|
||||
vst1q_u16(dst, d1l);
|
||||
vst1q_u16(dst + 8, d1h);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w == 32) { // avg32
|
||||
uint16x8_t s0, s1, s2, s3, d0, d1, d2, d3;
|
||||
do {
|
||||
s0 = vld1q_u16(src);
|
||||
s1 = vld1q_u16(src + 8);
|
||||
s2 = vld1q_u16(src + 16);
|
||||
s3 = vld1q_u16(src + 24);
|
||||
d0 = vld1q_u16(dst);
|
||||
d1 = vld1q_u16(dst + 8);
|
||||
d2 = vld1q_u16(dst + 16);
|
||||
d3 = vld1q_u16(dst + 24);
|
||||
src += src_stride;
|
||||
|
||||
d0 = vrhaddq_u16(s0, d0);
|
||||
d1 = vrhaddq_u16(s1, d1);
|
||||
d2 = vrhaddq_u16(s2, d2);
|
||||
d3 = vrhaddq_u16(s3, d3);
|
||||
|
||||
vst1q_u16(dst, d0);
|
||||
vst1q_u16(dst + 8, d1);
|
||||
vst1q_u16(dst + 16, d2);
|
||||
vst1q_u16(dst + 24, d3);
|
||||
dst += dst_stride;
|
||||
|
||||
s0 = vld1q_u16(src);
|
||||
s1 = vld1q_u16(src + 8);
|
||||
s2 = vld1q_u16(src + 16);
|
||||
s3 = vld1q_u16(src + 24);
|
||||
d0 = vld1q_u16(dst);
|
||||
d1 = vld1q_u16(dst + 8);
|
||||
d2 = vld1q_u16(dst + 16);
|
||||
d3 = vld1q_u16(dst + 24);
|
||||
src += src_stride;
|
||||
|
||||
d0 = vrhaddq_u16(s0, d0);
|
||||
d1 = vrhaddq_u16(s1, d1);
|
||||
d2 = vrhaddq_u16(s2, d2);
|
||||
d3 = vrhaddq_u16(s3, d3);
|
||||
|
||||
vst1q_u16(dst, d0);
|
||||
vst1q_u16(dst + 8, d1);
|
||||
vst1q_u16(dst + 16, d2);
|
||||
vst1q_u16(dst + 24, d3);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else { // avg64
|
||||
uint16x8_t s0, s1, s2, s3, d0, d1, d2, d3;
|
||||
do {
|
||||
s0 = vld1q_u16(src);
|
||||
s1 = vld1q_u16(src + 8);
|
||||
s2 = vld1q_u16(src + 16);
|
||||
s3 = vld1q_u16(src + 24);
|
||||
d0 = vld1q_u16(dst);
|
||||
d1 = vld1q_u16(dst + 8);
|
||||
d2 = vld1q_u16(dst + 16);
|
||||
d3 = vld1q_u16(dst + 24);
|
||||
|
||||
d0 = vrhaddq_u16(s0, d0);
|
||||
d1 = vrhaddq_u16(s1, d1);
|
||||
d2 = vrhaddq_u16(s2, d2);
|
||||
d3 = vrhaddq_u16(s3, d3);
|
||||
|
||||
vst1q_u16(dst, d0);
|
||||
vst1q_u16(dst + 8, d1);
|
||||
vst1q_u16(dst + 16, d2);
|
||||
vst1q_u16(dst + 24, d3);
|
||||
|
||||
s0 = vld1q_u16(src + 32);
|
||||
s1 = vld1q_u16(src + 40);
|
||||
s2 = vld1q_u16(src + 48);
|
||||
s3 = vld1q_u16(src + 56);
|
||||
d0 = vld1q_u16(dst + 32);
|
||||
d1 = vld1q_u16(dst + 40);
|
||||
d2 = vld1q_u16(dst + 48);
|
||||
d3 = vld1q_u16(dst + 56);
|
||||
|
||||
d0 = vrhaddq_u16(s0, d0);
|
||||
d1 = vrhaddq_u16(s1, d1);
|
||||
d2 = vrhaddq_u16(s2, d2);
|
||||
d3 = vrhaddq_u16(s3, d3);
|
||||
|
||||
vst1q_u16(dst + 32, d0);
|
||||
vst1q_u16(dst + 40, d1);
|
||||
vst1q_u16(dst + 48, d2);
|
||||
vst1q_u16(dst + 56, d3);
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
} while (--h);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
|
||||
void vpx_highbd_convolve_copy_neon(const uint16_t *src, ptrdiff_t src_stride,
|
||||
uint16_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h, int bd) {
|
||||
(void)filter;
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
(void)bd;
|
||||
|
||||
if (w < 8) { // copy4
|
||||
do {
|
||||
vst1_u16(dst, vld1_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst1_u16(dst, vld1_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w == 8) { // copy8
|
||||
do {
|
||||
vst1q_u16(dst, vld1q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst1q_u16(dst, vld1q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w < 32) { // copy16
|
||||
do {
|
||||
vst2q_u16(dst, vld2q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst2q_u16(dst, vld2q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst2q_u16(dst, vld2q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst2q_u16(dst, vld2q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else if (w == 32) { // copy32
|
||||
do {
|
||||
vst4q_u16(dst, vld4q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst4q_u16(dst, vld4q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst4q_u16(dst, vld4q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst4q_u16(dst, vld4q_u16(src));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else { // copy64
|
||||
do {
|
||||
vst4q_u16(dst, vld4q_u16(src));
|
||||
vst4q_u16(dst + 32, vld4q_u16(src + 32));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst4q_u16(dst, vld4q_u16(src));
|
||||
vst4q_u16(dst + 32, vld4q_u16(src + 32));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst4q_u16(dst, vld4q_u16(src));
|
||||
vst4q_u16(dst + 32, vld4q_u16(src + 32));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
vst4q_u16(dst, vld4q_u16(src));
|
||||
vst4q_u16(dst + 32, vld4q_u16(src + 32));
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
#include "vpx_dsp/vpx_filter.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
|
||||
void vpx_highbd_convolve8_neon(const uint16_t *src, ptrdiff_t src_stride,
|
||||
uint16_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h, int bd) {
|
||||
// + 1 to make it divisible by 4
|
||||
uint16_t temp[64 * 136];
|
||||
const int intermediate_height =
|
||||
(((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
|
||||
|
||||
/* Filter starting 3 lines back. The neon implementation will ignore the given
|
||||
* height and filter a multiple of 4 lines. Since this goes in to the temp
|
||||
* buffer which has lots of extra room and is subsequently discarded this is
|
||||
* safe if somewhat less than ideal. */
|
||||
vpx_highbd_convolve8_horiz_neon(src - src_stride * 3, src_stride, temp, w,
|
||||
filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w,
|
||||
intermediate_height, bd);
|
||||
|
||||
/* Step into the temp buffer 3 lines to get the actual frame data */
|
||||
vpx_highbd_convolve8_vert_neon(temp + w * 3, w, dst, dst_stride, filter,
|
||||
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h, bd);
|
||||
}
|
||||
|
||||
void vpx_highbd_convolve8_avg_neon(const uint16_t *src, ptrdiff_t src_stride,
|
||||
uint16_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h, int bd) {
|
||||
// + 1 to make it divisible by 4
|
||||
uint16_t temp[64 * 136];
|
||||
const int intermediate_height =
|
||||
(((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
|
||||
|
||||
/* This implementation has the same issues as above. In addition, we only want
|
||||
* to average the values after both passes.
|
||||
*/
|
||||
vpx_highbd_convolve8_horiz_neon(src - src_stride * 3, src_stride, temp, w,
|
||||
filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w,
|
||||
intermediate_height, bd);
|
||||
vpx_highbd_convolve8_avg_vert_neon(temp + w * 3, w, dst, dst_stride, filter,
|
||||
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h,
|
||||
bd);
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/inv_txfm.h"
|
||||
|
||||
static INLINE void idct16x16_1_add_pos_kernel(uint8_t **dest, const int stride,
|
||||
const uint8x16_t res) {
|
||||
const uint8x16_t a = vld1q_u8(*dest);
|
||||
const uint8x16_t b = vqaddq_u8(a, res);
|
||||
vst1q_u8(*dest, b);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void idct16x16_1_add_neg_kernel(uint8_t **dest, const int stride,
|
||||
const uint8x16_t res) {
|
||||
const uint8x16_t a = vld1q_u8(*dest);
|
||||
const uint8x16_t b = vqsubq_u8(a, res);
|
||||
vst1q_u8(*dest, b);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
void vpx_idct16x16_1_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
const int16_t out0 =
|
||||
WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64));
|
||||
const int16_t out1 = WRAPLOW(dct_const_round_shift(out0 * cospi_16_64));
|
||||
const int16_t a1 = ROUND_POWER_OF_TWO(out1, 6);
|
||||
|
||||
if (a1 >= 0) {
|
||||
const uint8x16_t dc = create_dcq(a1);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_pos_kernel(&dest, stride, dc);
|
||||
} else {
|
||||
const uint8x16_t dc = create_dcq(-a1);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct16x16_1_add_neg_kernel(&dest, stride, dc);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,764 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
|
||||
static INLINE void wrap_low_4x2(const int32x4_t *const t32, int16x4_t *const d0,
|
||||
int16x4_t *const d1) {
|
||||
*d0 = vrshrn_n_s32(t32[0], DCT_CONST_BITS);
|
||||
*d1 = vrshrn_n_s32(t32[1], DCT_CONST_BITS);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_8_24_d_kernel(const int16x4_t s0,
|
||||
const int16x4_t s1,
|
||||
const int16x4_t cospi_0_8_16_24,
|
||||
int32x4_t *const t32) {
|
||||
t32[0] = vmull_lane_s16(s0, cospi_0_8_16_24, 3);
|
||||
t32[1] = vmull_lane_s16(s1, cospi_0_8_16_24, 3);
|
||||
t32[0] = vmlsl_lane_s16(t32[0], s1, cospi_0_8_16_24, 1);
|
||||
t32[1] = vmlal_lane_s16(t32[1], s0, cospi_0_8_16_24, 1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_8_24_d(const int16x4_t s0, const int16x4_t s1,
|
||||
const int16x4_t cospi_0_8_16_24,
|
||||
int16x4_t *const d0, int16x4_t *const d1) {
|
||||
int32x4_t t32[2];
|
||||
|
||||
idct_cospi_8_24_d_kernel(s0, s1, cospi_0_8_16_24, t32);
|
||||
wrap_low_4x2(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_8_24_neg_d(const int16x4_t s0, const int16x4_t s1,
|
||||
const int16x4_t cospi_0_8_16_24,
|
||||
int16x4_t *const d0,
|
||||
int16x4_t *const d1) {
|
||||
int32x4_t t32[2];
|
||||
|
||||
idct_cospi_8_24_d_kernel(s0, s1, cospi_0_8_16_24, t32);
|
||||
t32[1] = vnegq_s32(t32[1]);
|
||||
wrap_low_4x2(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_16_16_d(const int16x4_t s0, const int16x4_t s1,
|
||||
const int16x4_t cospi_0_8_16_24,
|
||||
int16x4_t *const d0,
|
||||
int16x4_t *const d1) {
|
||||
int32x4_t t32[3];
|
||||
|
||||
t32[2] = vmull_lane_s16(s1, cospi_0_8_16_24, 2);
|
||||
t32[0] = vmlsl_lane_s16(t32[2], s0, cospi_0_8_16_24, 2);
|
||||
t32[1] = vmlal_lane_s16(t32[2], s0, cospi_0_8_16_24, 2);
|
||||
wrap_low_4x2(t32, d0, d1);
|
||||
}
|
||||
|
||||
void vpx_idct16x16_256_add_half1d(const void *const input, int16_t *output,
|
||||
void *const dest, const int stride,
|
||||
const int highbd_flag) {
|
||||
const int16x8_t cospis0 = vld1q_s16(kCospi);
|
||||
const int16x8_t cospis1 = vld1q_s16(kCospi + 8);
|
||||
const int16x4_t cospi_0_8_16_24 = vget_low_s16(cospis0);
|
||||
const int16x4_t cospi_4_12_20N_28 = vget_high_s16(cospis0);
|
||||
const int16x4_t cospi_2_30_10_22 = vget_low_s16(cospis1);
|
||||
const int16x4_t cospi_6_26N_14_18N = vget_high_s16(cospis1);
|
||||
int16x8_t in[16], step1[16], step2[16], out[16];
|
||||
|
||||
// Load input (16x8)
|
||||
if (output) {
|
||||
const tran_low_t *inputT = (const tran_low_t *)input;
|
||||
in[0] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[8] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[1] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[9] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[2] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[10] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[3] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[11] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[4] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[12] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[5] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[13] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[6] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[14] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[7] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 8;
|
||||
in[15] = load_tran_low_to_s16q(inputT);
|
||||
} else {
|
||||
const int16_t *inputT = (const int16_t *)input;
|
||||
in[0] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[8] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[1] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[9] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[2] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[10] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[3] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[11] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[4] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[12] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[5] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[13] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[6] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[14] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[7] = vld1q_s16(inputT);
|
||||
inputT += 8;
|
||||
in[15] = vld1q_s16(inputT);
|
||||
}
|
||||
|
||||
// Transpose
|
||||
transpose_s16_8x8(&in[0], &in[1], &in[2], &in[3], &in[4], &in[5], &in[6],
|
||||
&in[7]);
|
||||
transpose_s16_8x8(&in[8], &in[9], &in[10], &in[11], &in[12], &in[13], &in[14],
|
||||
&in[15]);
|
||||
|
||||
// stage 1
|
||||
step1[0] = in[0 / 2];
|
||||
step1[1] = in[16 / 2];
|
||||
step1[2] = in[8 / 2];
|
||||
step1[3] = in[24 / 2];
|
||||
step1[4] = in[4 / 2];
|
||||
step1[5] = in[20 / 2];
|
||||
step1[6] = in[12 / 2];
|
||||
step1[7] = in[28 / 2];
|
||||
step1[8] = in[2 / 2];
|
||||
step1[9] = in[18 / 2];
|
||||
step1[10] = in[10 / 2];
|
||||
step1[11] = in[26 / 2];
|
||||
step1[12] = in[6 / 2];
|
||||
step1[13] = in[22 / 2];
|
||||
step1[14] = in[14 / 2];
|
||||
step1[15] = in[30 / 2];
|
||||
|
||||
// stage 2
|
||||
step2[0] = step1[0];
|
||||
step2[1] = step1[1];
|
||||
step2[2] = step1[2];
|
||||
step2[3] = step1[3];
|
||||
step2[4] = step1[4];
|
||||
step2[5] = step1[5];
|
||||
step2[6] = step1[6];
|
||||
step2[7] = step1[7];
|
||||
idct_cospi_2_30(step1[8], step1[15], cospi_2_30_10_22, &step2[8], &step2[15]);
|
||||
idct_cospi_14_18(step1[9], step1[14], cospi_6_26N_14_18N, &step2[9],
|
||||
&step2[14]);
|
||||
idct_cospi_10_22(step1[10], step1[13], cospi_2_30_10_22, &step2[10],
|
||||
&step2[13]);
|
||||
idct_cospi_6_26(step1[11], step1[12], cospi_6_26N_14_18N, &step2[11],
|
||||
&step2[12]);
|
||||
|
||||
// stage 3
|
||||
step1[0] = step2[0];
|
||||
step1[1] = step2[1];
|
||||
step1[2] = step2[2];
|
||||
step1[3] = step2[3];
|
||||
idct_cospi_4_28(step2[4], step2[7], cospi_4_12_20N_28, &step1[4], &step1[7]);
|
||||
idct_cospi_12_20(step2[5], step2[6], cospi_4_12_20N_28, &step1[5], &step1[6]);
|
||||
step1[8] = vaddq_s16(step2[8], step2[9]);
|
||||
step1[9] = vsubq_s16(step2[8], step2[9]);
|
||||
step1[10] = vsubq_s16(step2[11], step2[10]);
|
||||
step1[11] = vaddq_s16(step2[11], step2[10]);
|
||||
step1[12] = vaddq_s16(step2[12], step2[13]);
|
||||
step1[13] = vsubq_s16(step2[12], step2[13]);
|
||||
step1[14] = vsubq_s16(step2[15], step2[14]);
|
||||
step1[15] = vaddq_s16(step2[15], step2[14]);
|
||||
|
||||
// stage 4
|
||||
idct_cospi_16_16_q(step1[1], step1[0], cospi_0_8_16_24, &step2[1], &step2[0]);
|
||||
idct_cospi_8_24_q(step1[2], step1[3], cospi_0_8_16_24, &step2[2], &step2[3]);
|
||||
step2[4] = vaddq_s16(step1[4], step1[5]);
|
||||
step2[5] = vsubq_s16(step1[4], step1[5]);
|
||||
step2[6] = vsubq_s16(step1[7], step1[6]);
|
||||
step2[7] = vaddq_s16(step1[7], step1[6]);
|
||||
step2[8] = step1[8];
|
||||
idct_cospi_8_24_q(step1[14], step1[9], cospi_0_8_16_24, &step2[9],
|
||||
&step2[14]);
|
||||
idct_cospi_8_24_neg_q(step1[13], step1[10], cospi_0_8_16_24, &step2[13],
|
||||
&step2[10]);
|
||||
step2[11] = step1[11];
|
||||
step2[12] = step1[12];
|
||||
step2[15] = step1[15];
|
||||
|
||||
// stage 5
|
||||
step1[0] = vaddq_s16(step2[0], step2[3]);
|
||||
step1[1] = vaddq_s16(step2[1], step2[2]);
|
||||
step1[2] = vsubq_s16(step2[1], step2[2]);
|
||||
step1[3] = vsubq_s16(step2[0], step2[3]);
|
||||
step1[4] = step2[4];
|
||||
idct_cospi_16_16_q(step2[5], step2[6], cospi_0_8_16_24, &step1[5], &step1[6]);
|
||||
step1[7] = step2[7];
|
||||
step1[8] = vaddq_s16(step2[8], step2[11]);
|
||||
step1[9] = vaddq_s16(step2[9], step2[10]);
|
||||
step1[10] = vsubq_s16(step2[9], step2[10]);
|
||||
step1[11] = vsubq_s16(step2[8], step2[11]);
|
||||
step1[12] = vsubq_s16(step2[15], step2[12]);
|
||||
step1[13] = vsubq_s16(step2[14], step2[13]);
|
||||
step1[14] = vaddq_s16(step2[14], step2[13]);
|
||||
step1[15] = vaddq_s16(step2[15], step2[12]);
|
||||
|
||||
// stage 6
|
||||
step2[0] = vaddq_s16(step1[0], step1[7]);
|
||||
step2[1] = vaddq_s16(step1[1], step1[6]);
|
||||
step2[2] = vaddq_s16(step1[2], step1[5]);
|
||||
step2[3] = vaddq_s16(step1[3], step1[4]);
|
||||
step2[4] = vsubq_s16(step1[3], step1[4]);
|
||||
step2[5] = vsubq_s16(step1[2], step1[5]);
|
||||
step2[6] = vsubq_s16(step1[1], step1[6]);
|
||||
step2[7] = vsubq_s16(step1[0], step1[7]);
|
||||
idct_cospi_16_16_q(step1[10], step1[13], cospi_0_8_16_24, &step2[10],
|
||||
&step2[13]);
|
||||
idct_cospi_16_16_q(step1[11], step1[12], cospi_0_8_16_24, &step2[11],
|
||||
&step2[12]);
|
||||
step2[8] = step1[8];
|
||||
step2[9] = step1[9];
|
||||
step2[14] = step1[14];
|
||||
step2[15] = step1[15];
|
||||
|
||||
// stage 7
|
||||
idct16x16_add_stage7(step2, out);
|
||||
|
||||
if (output) {
|
||||
idct16x16_store_pass1(out, output);
|
||||
} else {
|
||||
if (highbd_flag) {
|
||||
idct16x16_add_store_bd8(out, dest, stride);
|
||||
} else {
|
||||
idct16x16_add_store(out, dest, stride);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_idct16x16_38_add_half1d(const void *const input, int16_t *const output,
|
||||
void *const dest, const int stride,
|
||||
const int highbd_flag) {
|
||||
const int16x8_t cospis0 = vld1q_s16(kCospi);
|
||||
const int16x8_t cospis1 = vld1q_s16(kCospi + 8);
|
||||
const int16x8_t cospisd0 = vaddq_s16(cospis0, cospis0);
|
||||
const int16x8_t cospisd1 = vaddq_s16(cospis1, cospis1);
|
||||
const int16x4_t cospi_0_8_16_24 = vget_low_s16(cospis0);
|
||||
const int16x4_t cospid_0_8_16_24 = vget_low_s16(cospisd0);
|
||||
const int16x4_t cospid_4_12_20N_28 = vget_high_s16(cospisd0);
|
||||
const int16x4_t cospid_2_30_10_22 = vget_low_s16(cospisd1);
|
||||
const int16x4_t cospid_6_26_14_18N = vget_high_s16(cospisd1);
|
||||
int16x8_t in[8], step1[16], step2[16], out[16];
|
||||
|
||||
// Load input (8x8)
|
||||
if (output) {
|
||||
const tran_low_t *inputT = (const tran_low_t *)input;
|
||||
in[0] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 16;
|
||||
in[1] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 16;
|
||||
in[2] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 16;
|
||||
in[3] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 16;
|
||||
in[4] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 16;
|
||||
in[5] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 16;
|
||||
in[6] = load_tran_low_to_s16q(inputT);
|
||||
inputT += 16;
|
||||
in[7] = load_tran_low_to_s16q(inputT);
|
||||
} else {
|
||||
const int16_t *inputT = (const int16_t *)input;
|
||||
in[0] = vld1q_s16(inputT);
|
||||
inputT += 16;
|
||||
in[1] = vld1q_s16(inputT);
|
||||
inputT += 16;
|
||||
in[2] = vld1q_s16(inputT);
|
||||
inputT += 16;
|
||||
in[3] = vld1q_s16(inputT);
|
||||
inputT += 16;
|
||||
in[4] = vld1q_s16(inputT);
|
||||
inputT += 16;
|
||||
in[5] = vld1q_s16(inputT);
|
||||
inputT += 16;
|
||||
in[6] = vld1q_s16(inputT);
|
||||
inputT += 16;
|
||||
in[7] = vld1q_s16(inputT);
|
||||
}
|
||||
|
||||
// Transpose
|
||||
transpose_s16_8x8(&in[0], &in[1], &in[2], &in[3], &in[4], &in[5], &in[6],
|
||||
&in[7]);
|
||||
|
||||
// stage 1
|
||||
step1[0] = in[0 / 2];
|
||||
step1[2] = in[8 / 2];
|
||||
step1[4] = in[4 / 2];
|
||||
step1[6] = in[12 / 2];
|
||||
step1[8] = in[2 / 2];
|
||||
step1[10] = in[10 / 2];
|
||||
step1[12] = in[6 / 2];
|
||||
step1[14] = in[14 / 2]; // 0 in pass 1
|
||||
|
||||
// stage 2
|
||||
step2[0] = step1[0];
|
||||
step2[2] = step1[2];
|
||||
step2[4] = step1[4];
|
||||
step2[6] = step1[6];
|
||||
step2[8] = vqrdmulhq_lane_s16(step1[8], cospid_2_30_10_22, 1);
|
||||
step2[9] = vqrdmulhq_lane_s16(step1[14], cospid_6_26_14_18N, 3);
|
||||
step2[10] = vqrdmulhq_lane_s16(step1[10], cospid_2_30_10_22, 3);
|
||||
step2[11] = vqrdmulhq_lane_s16(step1[12], cospid_6_26_14_18N, 1);
|
||||
step2[12] = vqrdmulhq_lane_s16(step1[12], cospid_6_26_14_18N, 0);
|
||||
step2[13] = vqrdmulhq_lane_s16(step1[10], cospid_2_30_10_22, 2);
|
||||
step2[14] = vqrdmulhq_lane_s16(step1[14], cospid_6_26_14_18N, 2);
|
||||
step2[15] = vqrdmulhq_lane_s16(step1[8], cospid_2_30_10_22, 0);
|
||||
|
||||
// stage 3
|
||||
step1[0] = step2[0];
|
||||
step1[2] = step2[2];
|
||||
step1[4] = vqrdmulhq_lane_s16(step2[4], cospid_4_12_20N_28, 3);
|
||||
step1[5] = vqrdmulhq_lane_s16(step2[6], cospid_4_12_20N_28, 2);
|
||||
step1[6] = vqrdmulhq_lane_s16(step2[6], cospid_4_12_20N_28, 1);
|
||||
step1[7] = vqrdmulhq_lane_s16(step2[4], cospid_4_12_20N_28, 0);
|
||||
step1[8] = vaddq_s16(step2[8], step2[9]);
|
||||
step1[9] = vsubq_s16(step2[8], step2[9]);
|
||||
step1[10] = vsubq_s16(step2[11], step2[10]);
|
||||
step1[11] = vaddq_s16(step2[11], step2[10]);
|
||||
step1[12] = vaddq_s16(step2[12], step2[13]);
|
||||
step1[13] = vsubq_s16(step2[12], step2[13]);
|
||||
step1[14] = vsubq_s16(step2[15], step2[14]);
|
||||
step1[15] = vaddq_s16(step2[15], step2[14]);
|
||||
|
||||
// stage 4
|
||||
step2[0] = step2[1] = vqrdmulhq_lane_s16(step1[0], cospid_0_8_16_24, 2);
|
||||
step2[2] = vqrdmulhq_lane_s16(step1[2], cospid_0_8_16_24, 3);
|
||||
step2[3] = vqrdmulhq_lane_s16(step1[2], cospid_0_8_16_24, 1);
|
||||
step2[4] = vaddq_s16(step1[4], step1[5]);
|
||||
step2[5] = vsubq_s16(step1[4], step1[5]);
|
||||
step2[6] = vsubq_s16(step1[7], step1[6]);
|
||||
step2[7] = vaddq_s16(step1[7], step1[6]);
|
||||
step2[8] = step1[8];
|
||||
idct_cospi_8_24_q(step1[14], step1[9], cospi_0_8_16_24, &step2[9],
|
||||
&step2[14]);
|
||||
idct_cospi_8_24_neg_q(step1[13], step1[10], cospi_0_8_16_24, &step2[13],
|
||||
&step2[10]);
|
||||
step2[11] = step1[11];
|
||||
step2[12] = step1[12];
|
||||
step2[15] = step1[15];
|
||||
|
||||
// stage 5
|
||||
step1[0] = vaddq_s16(step2[0], step2[3]);
|
||||
step1[1] = vaddq_s16(step2[1], step2[2]);
|
||||
step1[2] = vsubq_s16(step2[1], step2[2]);
|
||||
step1[3] = vsubq_s16(step2[0], step2[3]);
|
||||
step1[4] = step2[4];
|
||||
idct_cospi_16_16_q(step2[5], step2[6], cospi_0_8_16_24, &step1[5], &step1[6]);
|
||||
step1[7] = step2[7];
|
||||
step1[8] = vaddq_s16(step2[8], step2[11]);
|
||||
step1[9] = vaddq_s16(step2[9], step2[10]);
|
||||
step1[10] = vsubq_s16(step2[9], step2[10]);
|
||||
step1[11] = vsubq_s16(step2[8], step2[11]);
|
||||
step1[12] = vsubq_s16(step2[15], step2[12]);
|
||||
step1[13] = vsubq_s16(step2[14], step2[13]);
|
||||
step1[14] = vaddq_s16(step2[14], step2[13]);
|
||||
step1[15] = vaddq_s16(step2[15], step2[12]);
|
||||
|
||||
// stage 6
|
||||
step2[0] = vaddq_s16(step1[0], step1[7]);
|
||||
step2[1] = vaddq_s16(step1[1], step1[6]);
|
||||
step2[2] = vaddq_s16(step1[2], step1[5]);
|
||||
step2[3] = vaddq_s16(step1[3], step1[4]);
|
||||
step2[4] = vsubq_s16(step1[3], step1[4]);
|
||||
step2[5] = vsubq_s16(step1[2], step1[5]);
|
||||
step2[6] = vsubq_s16(step1[1], step1[6]);
|
||||
step2[7] = vsubq_s16(step1[0], step1[7]);
|
||||
idct_cospi_16_16_q(step1[10], step1[13], cospi_0_8_16_24, &step2[10],
|
||||
&step2[13]);
|
||||
idct_cospi_16_16_q(step1[11], step1[12], cospi_0_8_16_24, &step2[11],
|
||||
&step2[12]);
|
||||
step2[8] = step1[8];
|
||||
step2[9] = step1[9];
|
||||
step2[14] = step1[14];
|
||||
step2[15] = step1[15];
|
||||
|
||||
// stage 7
|
||||
idct16x16_add_stage7(step2, out);
|
||||
|
||||
if (output) {
|
||||
idct16x16_store_pass1(out, output);
|
||||
} else {
|
||||
if (highbd_flag) {
|
||||
idct16x16_add_store_bd8(out, dest, stride);
|
||||
} else {
|
||||
idct16x16_add_store(out, dest, stride);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_idct16x16_10_add_half1d_pass1(const tran_low_t *input,
|
||||
int16_t *output) {
|
||||
const int16x8_t cospis0 = vld1q_s16(kCospi);
|
||||
const int16x8_t cospis1 = vld1q_s16(kCospi + 8);
|
||||
const int16x8_t cospisd0 = vaddq_s16(cospis0, cospis0);
|
||||
const int16x8_t cospisd1 = vaddq_s16(cospis1, cospis1);
|
||||
const int16x4_t cospi_0_8_16_24 = vget_low_s16(cospis0);
|
||||
const int16x4_t cospid_0_8_16_24 = vget_low_s16(cospisd0);
|
||||
const int16x4_t cospid_4_12_20N_28 = vget_high_s16(cospisd0);
|
||||
const int16x4_t cospid_2_30_10_22 = vget_low_s16(cospisd1);
|
||||
const int16x4_t cospid_6_26_14_18N = vget_high_s16(cospisd1);
|
||||
int16x4_t in[4], step1[16], step2[16], out[16];
|
||||
|
||||
// Load input (4x4)
|
||||
in[0] = load_tran_low_to_s16d(input);
|
||||
input += 16;
|
||||
in[1] = load_tran_low_to_s16d(input);
|
||||
input += 16;
|
||||
in[2] = load_tran_low_to_s16d(input);
|
||||
input += 16;
|
||||
in[3] = load_tran_low_to_s16d(input);
|
||||
|
||||
// Transpose
|
||||
transpose_s16_4x4d(&in[0], &in[1], &in[2], &in[3]);
|
||||
|
||||
// stage 1
|
||||
step1[0] = in[0 / 2];
|
||||
step1[4] = in[4 / 2];
|
||||
step1[8] = in[2 / 2];
|
||||
step1[12] = in[6 / 2];
|
||||
|
||||
// stage 2
|
||||
step2[0] = step1[0];
|
||||
step2[4] = step1[4];
|
||||
step2[8] = vqrdmulh_lane_s16(step1[8], cospid_2_30_10_22, 1);
|
||||
step2[11] = vqrdmulh_lane_s16(step1[12], cospid_6_26_14_18N, 1);
|
||||
step2[12] = vqrdmulh_lane_s16(step1[12], cospid_6_26_14_18N, 0);
|
||||
step2[15] = vqrdmulh_lane_s16(step1[8], cospid_2_30_10_22, 0);
|
||||
|
||||
// stage 3
|
||||
step1[0] = step2[0];
|
||||
step1[4] = vqrdmulh_lane_s16(step2[4], cospid_4_12_20N_28, 3);
|
||||
step1[7] = vqrdmulh_lane_s16(step2[4], cospid_4_12_20N_28, 0);
|
||||
step1[8] = step2[8];
|
||||
step1[9] = step2[8];
|
||||
step1[10] = step2[11];
|
||||
step1[11] = step2[11];
|
||||
step1[12] = step2[12];
|
||||
step1[13] = step2[12];
|
||||
step1[14] = step2[15];
|
||||
step1[15] = step2[15];
|
||||
|
||||
// stage 4
|
||||
step2[0] = step2[1] = vqrdmulh_lane_s16(step1[0], cospid_0_8_16_24, 2);
|
||||
step2[4] = step1[4];
|
||||
step2[5] = step1[4];
|
||||
step2[6] = step1[7];
|
||||
step2[7] = step1[7];
|
||||
step2[8] = step1[8];
|
||||
idct_cospi_8_24_d(step1[14], step1[9], cospi_0_8_16_24, &step2[9],
|
||||
&step2[14]);
|
||||
idct_cospi_8_24_neg_d(step1[13], step1[10], cospi_0_8_16_24, &step2[13],
|
||||
&step2[10]);
|
||||
step2[11] = step1[11];
|
||||
step2[12] = step1[12];
|
||||
step2[15] = step1[15];
|
||||
|
||||
// stage 5
|
||||
step1[0] = step2[0];
|
||||
step1[1] = step2[1];
|
||||
step1[2] = step2[1];
|
||||
step1[3] = step2[0];
|
||||
step1[4] = step2[4];
|
||||
idct_cospi_16_16_d(step2[5], step2[6], cospi_0_8_16_24, &step1[5], &step1[6]);
|
||||
step1[7] = step2[7];
|
||||
step1[8] = vadd_s16(step2[8], step2[11]);
|
||||
step1[9] = vadd_s16(step2[9], step2[10]);
|
||||
step1[10] = vsub_s16(step2[9], step2[10]);
|
||||
step1[11] = vsub_s16(step2[8], step2[11]);
|
||||
step1[12] = vsub_s16(step2[15], step2[12]);
|
||||
step1[13] = vsub_s16(step2[14], step2[13]);
|
||||
step1[14] = vadd_s16(step2[14], step2[13]);
|
||||
step1[15] = vadd_s16(step2[15], step2[12]);
|
||||
|
||||
// stage 6
|
||||
step2[0] = vadd_s16(step1[0], step1[7]);
|
||||
step2[1] = vadd_s16(step1[1], step1[6]);
|
||||
step2[2] = vadd_s16(step1[2], step1[5]);
|
||||
step2[3] = vadd_s16(step1[3], step1[4]);
|
||||
step2[4] = vsub_s16(step1[3], step1[4]);
|
||||
step2[5] = vsub_s16(step1[2], step1[5]);
|
||||
step2[6] = vsub_s16(step1[1], step1[6]);
|
||||
step2[7] = vsub_s16(step1[0], step1[7]);
|
||||
idct_cospi_16_16_d(step1[10], step1[13], cospi_0_8_16_24, &step2[10],
|
||||
&step2[13]);
|
||||
idct_cospi_16_16_d(step1[11], step1[12], cospi_0_8_16_24, &step2[11],
|
||||
&step2[12]);
|
||||
step2[8] = step1[8];
|
||||
step2[9] = step1[9];
|
||||
step2[14] = step1[14];
|
||||
step2[15] = step1[15];
|
||||
|
||||
// stage 7
|
||||
out[0] = vadd_s16(step2[0], step2[15]);
|
||||
out[1] = vadd_s16(step2[1], step2[14]);
|
||||
out[2] = vadd_s16(step2[2], step2[13]);
|
||||
out[3] = vadd_s16(step2[3], step2[12]);
|
||||
out[4] = vadd_s16(step2[4], step2[11]);
|
||||
out[5] = vadd_s16(step2[5], step2[10]);
|
||||
out[6] = vadd_s16(step2[6], step2[9]);
|
||||
out[7] = vadd_s16(step2[7], step2[8]);
|
||||
out[8] = vsub_s16(step2[7], step2[8]);
|
||||
out[9] = vsub_s16(step2[6], step2[9]);
|
||||
out[10] = vsub_s16(step2[5], step2[10]);
|
||||
out[11] = vsub_s16(step2[4], step2[11]);
|
||||
out[12] = vsub_s16(step2[3], step2[12]);
|
||||
out[13] = vsub_s16(step2[2], step2[13]);
|
||||
out[14] = vsub_s16(step2[1], step2[14]);
|
||||
out[15] = vsub_s16(step2[0], step2[15]);
|
||||
|
||||
// pass 1: save the result into output
|
||||
vst1_s16(output, out[0]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[1]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[2]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[3]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[4]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[5]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[6]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[7]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[8]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[9]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[10]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[11]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[12]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[13]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[14]);
|
||||
output += 4;
|
||||
vst1_s16(output, out[15]);
|
||||
}
|
||||
|
||||
void vpx_idct16x16_10_add_half1d_pass2(const int16_t *input,
|
||||
int16_t *const output, void *const dest,
|
||||
const int stride,
|
||||
const int highbd_flag) {
|
||||
const int16x8_t cospis0 = vld1q_s16(kCospi);
|
||||
const int16x8_t cospis1 = vld1q_s16(kCospi + 8);
|
||||
const int16x8_t cospisd0 = vaddq_s16(cospis0, cospis0);
|
||||
const int16x8_t cospisd1 = vaddq_s16(cospis1, cospis1);
|
||||
const int16x4_t cospi_0_8_16_24 = vget_low_s16(cospis0);
|
||||
const int16x4_t cospid_0_8_16_24 = vget_low_s16(cospisd0);
|
||||
const int16x4_t cospid_4_12_20N_28 = vget_high_s16(cospisd0);
|
||||
const int16x4_t cospid_2_30_10_22 = vget_low_s16(cospisd1);
|
||||
const int16x4_t cospid_6_26_14_18N = vget_high_s16(cospisd1);
|
||||
int16x4_t ind[8];
|
||||
int16x8_t in[4], step1[16], step2[16], out[16];
|
||||
|
||||
// Load input (4x8)
|
||||
ind[0] = vld1_s16(input);
|
||||
input += 4;
|
||||
ind[1] = vld1_s16(input);
|
||||
input += 4;
|
||||
ind[2] = vld1_s16(input);
|
||||
input += 4;
|
||||
ind[3] = vld1_s16(input);
|
||||
input += 4;
|
||||
ind[4] = vld1_s16(input);
|
||||
input += 4;
|
||||
ind[5] = vld1_s16(input);
|
||||
input += 4;
|
||||
ind[6] = vld1_s16(input);
|
||||
input += 4;
|
||||
ind[7] = vld1_s16(input);
|
||||
|
||||
// Transpose
|
||||
transpose_s16_4x8(ind[0], ind[1], ind[2], ind[3], ind[4], ind[5], ind[6],
|
||||
ind[7], &in[0], &in[1], &in[2], &in[3]);
|
||||
|
||||
// stage 1
|
||||
step1[0] = in[0 / 2];
|
||||
step1[4] = in[4 / 2];
|
||||
step1[8] = in[2 / 2];
|
||||
step1[12] = in[6 / 2];
|
||||
|
||||
// stage 2
|
||||
step2[0] = step1[0];
|
||||
step2[4] = step1[4];
|
||||
step2[8] = vqrdmulhq_lane_s16(step1[8], cospid_2_30_10_22, 1);
|
||||
step2[11] = vqrdmulhq_lane_s16(step1[12], cospid_6_26_14_18N, 1);
|
||||
step2[12] = vqrdmulhq_lane_s16(step1[12], cospid_6_26_14_18N, 0);
|
||||
step2[15] = vqrdmulhq_lane_s16(step1[8], cospid_2_30_10_22, 0);
|
||||
|
||||
// stage 3
|
||||
step1[0] = step2[0];
|
||||
step1[4] = vqrdmulhq_lane_s16(step2[4], cospid_4_12_20N_28, 3);
|
||||
step1[7] = vqrdmulhq_lane_s16(step2[4], cospid_4_12_20N_28, 0);
|
||||
step1[8] = step2[8];
|
||||
step1[9] = step2[8];
|
||||
step1[10] = step2[11];
|
||||
step1[11] = step2[11];
|
||||
step1[12] = step2[12];
|
||||
step1[13] = step2[12];
|
||||
step1[14] = step2[15];
|
||||
step1[15] = step2[15];
|
||||
|
||||
// stage 4
|
||||
step2[0] = step2[1] = vqrdmulhq_lane_s16(step1[0], cospid_0_8_16_24, 2);
|
||||
step2[4] = step1[4];
|
||||
step2[5] = step1[4];
|
||||
step2[6] = step1[7];
|
||||
step2[7] = step1[7];
|
||||
step2[8] = step1[8];
|
||||
idct_cospi_8_24_q(step1[14], step1[9], cospi_0_8_16_24, &step2[9],
|
||||
&step2[14]);
|
||||
idct_cospi_8_24_neg_q(step1[13], step1[10], cospi_0_8_16_24, &step2[13],
|
||||
&step2[10]);
|
||||
step2[11] = step1[11];
|
||||
step2[12] = step1[12];
|
||||
step2[15] = step1[15];
|
||||
|
||||
// stage 5
|
||||
step1[0] = step2[0];
|
||||
step1[1] = step2[1];
|
||||
step1[2] = step2[1];
|
||||
step1[3] = step2[0];
|
||||
step1[4] = step2[4];
|
||||
idct_cospi_16_16_q(step2[5], step2[6], cospi_0_8_16_24, &step1[5], &step1[6]);
|
||||
step1[7] = step2[7];
|
||||
step1[8] = vaddq_s16(step2[8], step2[11]);
|
||||
step1[9] = vaddq_s16(step2[9], step2[10]);
|
||||
step1[10] = vsubq_s16(step2[9], step2[10]);
|
||||
step1[11] = vsubq_s16(step2[8], step2[11]);
|
||||
step1[12] = vsubq_s16(step2[15], step2[12]);
|
||||
step1[13] = vsubq_s16(step2[14], step2[13]);
|
||||
step1[14] = vaddq_s16(step2[14], step2[13]);
|
||||
step1[15] = vaddq_s16(step2[15], step2[12]);
|
||||
|
||||
// stage 6
|
||||
step2[0] = vaddq_s16(step1[0], step1[7]);
|
||||
step2[1] = vaddq_s16(step1[1], step1[6]);
|
||||
step2[2] = vaddq_s16(step1[2], step1[5]);
|
||||
step2[3] = vaddq_s16(step1[3], step1[4]);
|
||||
step2[4] = vsubq_s16(step1[3], step1[4]);
|
||||
step2[5] = vsubq_s16(step1[2], step1[5]);
|
||||
step2[6] = vsubq_s16(step1[1], step1[6]);
|
||||
step2[7] = vsubq_s16(step1[0], step1[7]);
|
||||
idct_cospi_16_16_q(step1[10], step1[13], cospi_0_8_16_24, &step2[10],
|
||||
&step2[13]);
|
||||
idct_cospi_16_16_q(step1[11], step1[12], cospi_0_8_16_24, &step2[11],
|
||||
&step2[12]);
|
||||
step2[8] = step1[8];
|
||||
step2[9] = step1[9];
|
||||
step2[14] = step1[14];
|
||||
step2[15] = step1[15];
|
||||
|
||||
// stage 7
|
||||
idct16x16_add_stage7(step2, out);
|
||||
|
||||
if (output) {
|
||||
idct16x16_store_pass1(out, output);
|
||||
} else {
|
||||
if (highbd_flag) {
|
||||
idct16x16_add_store_bd8(out, dest, stride);
|
||||
} else {
|
||||
idct16x16_add_store(out, dest, stride);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_idct16x16_256_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
int16_t row_idct_output[16 * 16];
|
||||
|
||||
// pass 1
|
||||
// Parallel idct on the upper 8 rows
|
||||
vpx_idct16x16_256_add_half1d(input, row_idct_output, dest, stride, 0);
|
||||
|
||||
// Parallel idct on the lower 8 rows
|
||||
vpx_idct16x16_256_add_half1d(input + 8 * 16, row_idct_output + 8, dest,
|
||||
stride, 0);
|
||||
|
||||
// pass 2
|
||||
// Parallel idct to get the left 8 columns
|
||||
vpx_idct16x16_256_add_half1d(row_idct_output, NULL, dest, stride, 0);
|
||||
|
||||
// Parallel idct to get the right 8 columns
|
||||
vpx_idct16x16_256_add_half1d(row_idct_output + 16 * 8, NULL, dest + 8, stride,
|
||||
0);
|
||||
}
|
||||
|
||||
void vpx_idct16x16_38_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
int16_t row_idct_output[16 * 16];
|
||||
|
||||
// pass 1
|
||||
// Parallel idct on the upper 8 rows
|
||||
vpx_idct16x16_38_add_half1d(input, row_idct_output, dest, stride, 0);
|
||||
|
||||
// pass 2
|
||||
// Parallel idct to get the left 8 columns
|
||||
vpx_idct16x16_38_add_half1d(row_idct_output, NULL, dest, stride, 0);
|
||||
|
||||
// Parallel idct to get the right 8 columns
|
||||
vpx_idct16x16_38_add_half1d(row_idct_output + 16 * 8, NULL, dest + 8, stride,
|
||||
0);
|
||||
}
|
||||
|
||||
void vpx_idct16x16_10_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
int16_t row_idct_output[4 * 16];
|
||||
|
||||
// pass 1
|
||||
// Parallel idct on the upper 8 rows
|
||||
vpx_idct16x16_10_add_half1d_pass1(input, row_idct_output);
|
||||
|
||||
// pass 2
|
||||
// Parallel idct to get the left 8 columns
|
||||
vpx_idct16x16_10_add_half1d_pass2(row_idct_output, NULL, dest, stride, 0);
|
||||
|
||||
// Parallel idct to get the right 8 columns
|
||||
vpx_idct16x16_10_add_half1d_pass2(row_idct_output + 4 * 8, NULL, dest + 8,
|
||||
stride, 0);
|
||||
}
|
||||
@@ -0,0 +1,674 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
|
||||
static INLINE void load_8x8_s16(const tran_low_t *input, int16x8_t *const in0,
|
||||
int16x8_t *const in1, int16x8_t *const in2,
|
||||
int16x8_t *const in3, int16x8_t *const in4,
|
||||
int16x8_t *const in5, int16x8_t *const in6,
|
||||
int16x8_t *const in7) {
|
||||
*in0 = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
*in1 = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
*in2 = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
*in3 = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
*in4 = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
*in5 = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
*in6 = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
*in7 = load_tran_low_to_s16q(input);
|
||||
}
|
||||
|
||||
static INLINE void load_4x8_s16(const tran_low_t *input, int16x4_t *const in0,
|
||||
int16x4_t *const in1, int16x4_t *const in2,
|
||||
int16x4_t *const in3, int16x4_t *const in4,
|
||||
int16x4_t *const in5, int16x4_t *const in6,
|
||||
int16x4_t *const in7) {
|
||||
*in0 = load_tran_low_to_s16d(input);
|
||||
input += 32;
|
||||
*in1 = load_tran_low_to_s16d(input);
|
||||
input += 32;
|
||||
*in2 = load_tran_low_to_s16d(input);
|
||||
input += 32;
|
||||
*in3 = load_tran_low_to_s16d(input);
|
||||
input += 32;
|
||||
*in4 = load_tran_low_to_s16d(input);
|
||||
input += 32;
|
||||
*in5 = load_tran_low_to_s16d(input);
|
||||
input += 32;
|
||||
*in6 = load_tran_low_to_s16d(input);
|
||||
input += 32;
|
||||
*in7 = load_tran_low_to_s16d(input);
|
||||
}
|
||||
|
||||
// Only for the first pass of the _135_ variant. Since it only uses values from
|
||||
// the top left 16x16 it can safely assume all the remaining values are 0 and
|
||||
// skip an awful lot of calculations. In fact, only the first 12 columns make
|
||||
// the cut. None of the elements in the 13th, 14th, 15th or 16th columns are
|
||||
// used so it skips any calls to input[12|13|14|15] too.
|
||||
// In C this does a single row of 32 for each call. Here it transposes the top
|
||||
// left 12x8 to allow using SIMD.
|
||||
|
||||
// vp9/common/vp9_scan.c:vp9_default_iscan_32x32 arranges the first 135 non-zero
|
||||
// coefficients as follows:
|
||||
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
|
||||
// 0 0 2 5 10 17 25 38 47 62 83 101 121
|
||||
// 1 1 4 8 15 22 30 45 58 74 92 112 133
|
||||
// 2 3 7 12 18 28 36 52 64 82 102 118
|
||||
// 3 6 11 16 23 31 43 60 73 90 109 126
|
||||
// 4 9 14 19 29 37 50 65 78 98 116 134
|
||||
// 5 13 20 26 35 44 54 72 85 105 123
|
||||
// 6 21 27 33 42 53 63 80 94 113 132
|
||||
// 7 24 32 39 48 57 71 88 104 120
|
||||
// 8 34 40 46 56 68 81 96 111 130
|
||||
// 9 41 49 55 67 77 91 107 124
|
||||
// 10 51 59 66 76 89 99 119 131
|
||||
// 11 61 69 75 87 100 114 129
|
||||
// 12 70 79 86 97 108 122
|
||||
// 13 84 93 103 110 125
|
||||
// 14 98 106 115 127
|
||||
// 15 117 128
|
||||
void vpx_idct32_12_neon(const tran_low_t *const input, int16_t *output) {
|
||||
int16x4_t tmp[8];
|
||||
int16x8_t in[12], s1[32], s2[32], s3[32], s4[32], s5[32], s6[32], s7[32];
|
||||
|
||||
load_8x8_s16(input, &in[0], &in[1], &in[2], &in[3], &in[4], &in[5], &in[6],
|
||||
&in[7]);
|
||||
transpose_s16_8x8(&in[0], &in[1], &in[2], &in[3], &in[4], &in[5], &in[6],
|
||||
&in[7]);
|
||||
|
||||
load_4x8_s16(input + 8, &tmp[0], &tmp[1], &tmp[2], &tmp[3], &tmp[4], &tmp[5],
|
||||
&tmp[6], &tmp[7]);
|
||||
transpose_s16_4x8(tmp[0], tmp[1], tmp[2], tmp[3], tmp[4], tmp[5], tmp[6],
|
||||
tmp[7], &in[8], &in[9], &in[10], &in[11]);
|
||||
|
||||
// stage 1
|
||||
s1[16] = multiply_shift_and_narrow_s16(in[1], cospi_31_64);
|
||||
s1[31] = multiply_shift_and_narrow_s16(in[1], cospi_1_64);
|
||||
|
||||
s1[18] = multiply_shift_and_narrow_s16(in[9], cospi_23_64);
|
||||
s1[29] = multiply_shift_and_narrow_s16(in[9], cospi_9_64);
|
||||
|
||||
s1[19] = multiply_shift_and_narrow_s16(in[7], -cospi_25_64);
|
||||
s1[28] = multiply_shift_and_narrow_s16(in[7], cospi_7_64);
|
||||
|
||||
s1[20] = multiply_shift_and_narrow_s16(in[5], cospi_27_64);
|
||||
s1[27] = multiply_shift_and_narrow_s16(in[5], cospi_5_64);
|
||||
|
||||
s1[21] = multiply_shift_and_narrow_s16(in[11], -cospi_21_64);
|
||||
s1[26] = multiply_shift_and_narrow_s16(in[11], cospi_11_64);
|
||||
|
||||
s1[23] = multiply_shift_and_narrow_s16(in[3], -cospi_29_64);
|
||||
s1[24] = multiply_shift_and_narrow_s16(in[3], cospi_3_64);
|
||||
|
||||
// stage 2
|
||||
s2[8] = multiply_shift_and_narrow_s16(in[2], cospi_30_64);
|
||||
s2[15] = multiply_shift_and_narrow_s16(in[2], cospi_2_64);
|
||||
|
||||
s2[10] = multiply_shift_and_narrow_s16(in[10], cospi_22_64);
|
||||
s2[13] = multiply_shift_and_narrow_s16(in[10], cospi_10_64);
|
||||
|
||||
s2[11] = multiply_shift_and_narrow_s16(in[6], -cospi_26_64);
|
||||
s2[12] = multiply_shift_and_narrow_s16(in[6], cospi_6_64);
|
||||
|
||||
s2[18] = vsubq_s16(s1[19], s1[18]);
|
||||
s2[19] = vaddq_s16(s1[18], s1[19]);
|
||||
s2[20] = vaddq_s16(s1[20], s1[21]);
|
||||
s2[21] = vsubq_s16(s1[20], s1[21]);
|
||||
s2[26] = vsubq_s16(s1[27], s1[26]);
|
||||
s2[27] = vaddq_s16(s1[26], s1[27]);
|
||||
s2[28] = vaddq_s16(s1[28], s1[29]);
|
||||
s2[29] = vsubq_s16(s1[28], s1[29]);
|
||||
|
||||
// stage 3
|
||||
s3[4] = multiply_shift_and_narrow_s16(in[4], cospi_28_64);
|
||||
s3[7] = multiply_shift_and_narrow_s16(in[4], cospi_4_64);
|
||||
|
||||
s3[10] = vsubq_s16(s2[11], s2[10]);
|
||||
s3[11] = vaddq_s16(s2[10], s2[11]);
|
||||
s3[12] = vaddq_s16(s2[12], s2[13]);
|
||||
s3[13] = vsubq_s16(s2[12], s2[13]);
|
||||
|
||||
s3[17] = multiply_accumulate_shift_and_narrow_s16(s1[16], -cospi_4_64, s1[31],
|
||||
cospi_28_64);
|
||||
s3[30] = multiply_accumulate_shift_and_narrow_s16(s1[16], cospi_28_64, s1[31],
|
||||
cospi_4_64);
|
||||
|
||||
s3[18] = multiply_accumulate_shift_and_narrow_s16(s2[18], -cospi_28_64,
|
||||
s2[29], -cospi_4_64);
|
||||
s3[29] = multiply_accumulate_shift_and_narrow_s16(s2[18], -cospi_4_64, s2[29],
|
||||
cospi_28_64);
|
||||
|
||||
s3[21] = multiply_accumulate_shift_and_narrow_s16(s2[21], -cospi_20_64,
|
||||
s2[26], cospi_12_64);
|
||||
s3[26] = multiply_accumulate_shift_and_narrow_s16(s2[21], cospi_12_64, s2[26],
|
||||
cospi_20_64);
|
||||
|
||||
s3[22] = multiply_accumulate_shift_and_narrow_s16(s1[23], -cospi_12_64,
|
||||
s1[24], -cospi_20_64);
|
||||
s3[25] = multiply_accumulate_shift_and_narrow_s16(s1[23], -cospi_20_64,
|
||||
s1[24], cospi_12_64);
|
||||
|
||||
// stage 4
|
||||
s4[0] = multiply_shift_and_narrow_s16(in[0], cospi_16_64);
|
||||
s4[2] = multiply_shift_and_narrow_s16(in[8], cospi_24_64);
|
||||
s4[3] = multiply_shift_and_narrow_s16(in[8], cospi_8_64);
|
||||
|
||||
s4[9] = multiply_accumulate_shift_and_narrow_s16(s2[8], -cospi_8_64, s2[15],
|
||||
cospi_24_64);
|
||||
s4[14] = multiply_accumulate_shift_and_narrow_s16(s2[8], cospi_24_64, s2[15],
|
||||
cospi_8_64);
|
||||
|
||||
s4[10] = multiply_accumulate_shift_and_narrow_s16(s3[10], -cospi_24_64,
|
||||
s3[13], -cospi_8_64);
|
||||
s4[13] = multiply_accumulate_shift_and_narrow_s16(s3[10], -cospi_8_64, s3[13],
|
||||
cospi_24_64);
|
||||
|
||||
s4[16] = vaddq_s16(s1[16], s2[19]);
|
||||
s4[17] = vaddq_s16(s3[17], s3[18]);
|
||||
s4[18] = vsubq_s16(s3[17], s3[18]);
|
||||
s4[19] = vsubq_s16(s1[16], s2[19]);
|
||||
s4[20] = vsubq_s16(s1[23], s2[20]);
|
||||
s4[21] = vsubq_s16(s3[22], s3[21]);
|
||||
s4[22] = vaddq_s16(s3[21], s3[22]);
|
||||
s4[23] = vaddq_s16(s2[20], s1[23]);
|
||||
s4[24] = vaddq_s16(s1[24], s2[27]);
|
||||
s4[25] = vaddq_s16(s3[25], s3[26]);
|
||||
s4[26] = vsubq_s16(s3[25], s3[26]);
|
||||
s4[27] = vsubq_s16(s1[24], s2[27]);
|
||||
s4[28] = vsubq_s16(s1[31], s2[28]);
|
||||
s4[29] = vsubq_s16(s3[30], s3[29]);
|
||||
s4[30] = vaddq_s16(s3[29], s3[30]);
|
||||
s4[31] = vaddq_s16(s2[28], s1[31]);
|
||||
|
||||
// stage 5
|
||||
s5[0] = vaddq_s16(s4[0], s4[3]);
|
||||
s5[1] = vaddq_s16(s4[0], s4[2]);
|
||||
s5[2] = vsubq_s16(s4[0], s4[2]);
|
||||
s5[3] = vsubq_s16(s4[0], s4[3]);
|
||||
|
||||
s5[5] = sub_multiply_shift_and_narrow_s16(s3[7], s3[4], cospi_16_64);
|
||||
s5[6] = add_multiply_shift_and_narrow_s16(s3[4], s3[7], cospi_16_64);
|
||||
|
||||
s5[8] = vaddq_s16(s2[8], s3[11]);
|
||||
s5[9] = vaddq_s16(s4[9], s4[10]);
|
||||
s5[10] = vsubq_s16(s4[9], s4[10]);
|
||||
s5[11] = vsubq_s16(s2[8], s3[11]);
|
||||
s5[12] = vsubq_s16(s2[15], s3[12]);
|
||||
s5[13] = vsubq_s16(s4[14], s4[13]);
|
||||
s5[14] = vaddq_s16(s4[13], s4[14]);
|
||||
s5[15] = vaddq_s16(s2[15], s3[12]);
|
||||
|
||||
s5[18] = multiply_accumulate_shift_and_narrow_s16(s4[18], -cospi_8_64, s4[29],
|
||||
cospi_24_64);
|
||||
s5[29] = multiply_accumulate_shift_and_narrow_s16(s4[18], cospi_24_64, s4[29],
|
||||
cospi_8_64);
|
||||
|
||||
s5[19] = multiply_accumulate_shift_and_narrow_s16(s4[19], -cospi_8_64, s4[28],
|
||||
cospi_24_64);
|
||||
s5[28] = multiply_accumulate_shift_and_narrow_s16(s4[19], cospi_24_64, s4[28],
|
||||
cospi_8_64);
|
||||
|
||||
s5[20] = multiply_accumulate_shift_and_narrow_s16(s4[20], -cospi_24_64,
|
||||
s4[27], -cospi_8_64);
|
||||
s5[27] = multiply_accumulate_shift_and_narrow_s16(s4[20], -cospi_8_64, s4[27],
|
||||
cospi_24_64);
|
||||
|
||||
s5[21] = multiply_accumulate_shift_and_narrow_s16(s4[21], -cospi_24_64,
|
||||
s4[26], -cospi_8_64);
|
||||
s5[26] = multiply_accumulate_shift_and_narrow_s16(s4[21], -cospi_8_64, s4[26],
|
||||
cospi_24_64);
|
||||
|
||||
// stage 6
|
||||
s6[0] = vaddq_s16(s5[0], s3[7]);
|
||||
s6[1] = vaddq_s16(s5[1], s5[6]);
|
||||
s6[2] = vaddq_s16(s5[2], s5[5]);
|
||||
s6[3] = vaddq_s16(s5[3], s3[4]);
|
||||
s6[4] = vsubq_s16(s5[3], s3[4]);
|
||||
s6[5] = vsubq_s16(s5[2], s5[5]);
|
||||
s6[6] = vsubq_s16(s5[1], s5[6]);
|
||||
s6[7] = vsubq_s16(s5[0], s3[7]);
|
||||
|
||||
s6[10] = sub_multiply_shift_and_narrow_s16(s5[13], s5[10], cospi_16_64);
|
||||
s6[13] = add_multiply_shift_and_narrow_s16(s5[10], s5[13], cospi_16_64);
|
||||
|
||||
s6[11] = sub_multiply_shift_and_narrow_s16(s5[12], s5[11], cospi_16_64);
|
||||
s6[12] = add_multiply_shift_and_narrow_s16(s5[11], s5[12], cospi_16_64);
|
||||
|
||||
s6[16] = vaddq_s16(s4[16], s4[23]);
|
||||
s6[17] = vaddq_s16(s4[17], s4[22]);
|
||||
s6[18] = vaddq_s16(s5[18], s5[21]);
|
||||
s6[19] = vaddq_s16(s5[19], s5[20]);
|
||||
s6[20] = vsubq_s16(s5[19], s5[20]);
|
||||
s6[21] = vsubq_s16(s5[18], s5[21]);
|
||||
s6[22] = vsubq_s16(s4[17], s4[22]);
|
||||
s6[23] = vsubq_s16(s4[16], s4[23]);
|
||||
|
||||
s6[24] = vsubq_s16(s4[31], s4[24]);
|
||||
s6[25] = vsubq_s16(s4[30], s4[25]);
|
||||
s6[26] = vsubq_s16(s5[29], s5[26]);
|
||||
s6[27] = vsubq_s16(s5[28], s5[27]);
|
||||
s6[28] = vaddq_s16(s5[27], s5[28]);
|
||||
s6[29] = vaddq_s16(s5[26], s5[29]);
|
||||
s6[30] = vaddq_s16(s4[25], s4[30]);
|
||||
s6[31] = vaddq_s16(s4[24], s4[31]);
|
||||
|
||||
// stage 7
|
||||
s7[0] = vaddq_s16(s6[0], s5[15]);
|
||||
s7[1] = vaddq_s16(s6[1], s5[14]);
|
||||
s7[2] = vaddq_s16(s6[2], s6[13]);
|
||||
s7[3] = vaddq_s16(s6[3], s6[12]);
|
||||
s7[4] = vaddq_s16(s6[4], s6[11]);
|
||||
s7[5] = vaddq_s16(s6[5], s6[10]);
|
||||
s7[6] = vaddq_s16(s6[6], s5[9]);
|
||||
s7[7] = vaddq_s16(s6[7], s5[8]);
|
||||
s7[8] = vsubq_s16(s6[7], s5[8]);
|
||||
s7[9] = vsubq_s16(s6[6], s5[9]);
|
||||
s7[10] = vsubq_s16(s6[5], s6[10]);
|
||||
s7[11] = vsubq_s16(s6[4], s6[11]);
|
||||
s7[12] = vsubq_s16(s6[3], s6[12]);
|
||||
s7[13] = vsubq_s16(s6[2], s6[13]);
|
||||
s7[14] = vsubq_s16(s6[1], s5[14]);
|
||||
s7[15] = vsubq_s16(s6[0], s5[15]);
|
||||
|
||||
s7[20] = sub_multiply_shift_and_narrow_s16(s6[27], s6[20], cospi_16_64);
|
||||
s7[27] = add_multiply_shift_and_narrow_s16(s6[20], s6[27], cospi_16_64);
|
||||
|
||||
s7[21] = sub_multiply_shift_and_narrow_s16(s6[26], s6[21], cospi_16_64);
|
||||
s7[26] = add_multiply_shift_and_narrow_s16(s6[21], s6[26], cospi_16_64);
|
||||
|
||||
s7[22] = sub_multiply_shift_and_narrow_s16(s6[25], s6[22], cospi_16_64);
|
||||
s7[25] = add_multiply_shift_and_narrow_s16(s6[22], s6[25], cospi_16_64);
|
||||
|
||||
s7[23] = sub_multiply_shift_and_narrow_s16(s6[24], s6[23], cospi_16_64);
|
||||
s7[24] = add_multiply_shift_and_narrow_s16(s6[23], s6[24], cospi_16_64);
|
||||
|
||||
// final stage
|
||||
vst1q_s16(output, vaddq_s16(s7[0], s6[31]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[1], s6[30]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[2], s6[29]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[3], s6[28]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[4], s7[27]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[5], s7[26]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[6], s7[25]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[7], s7[24]));
|
||||
output += 16;
|
||||
|
||||
vst1q_s16(output, vaddq_s16(s7[8], s7[23]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[9], s7[22]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[10], s7[21]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[11], s7[20]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[12], s6[19]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[13], s6[18]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[14], s6[17]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vaddq_s16(s7[15], s6[16]));
|
||||
output += 16;
|
||||
|
||||
vst1q_s16(output, vsubq_s16(s7[15], s6[16]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[14], s6[17]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[13], s6[18]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[12], s6[19]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[11], s7[20]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[10], s7[21]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[9], s7[22]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[8], s7[23]));
|
||||
output += 16;
|
||||
|
||||
vst1q_s16(output, vsubq_s16(s7[7], s7[24]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[6], s7[25]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[5], s7[26]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[4], s7[27]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[3], s6[28]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[2], s6[29]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[1], s6[30]));
|
||||
output += 16;
|
||||
vst1q_s16(output, vsubq_s16(s7[0], s6[31]));
|
||||
}
|
||||
|
||||
void vpx_idct32_16_neon(const int16_t *const input, void *const output,
|
||||
const int stride, const int highbd_flag) {
|
||||
int16x8_t in[16], s1[32], s2[32], s3[32], s4[32], s5[32], s6[32], s7[32],
|
||||
out[32];
|
||||
|
||||
load_and_transpose_s16_8x8(input, 16, &in[0], &in[1], &in[2], &in[3], &in[4],
|
||||
&in[5], &in[6], &in[7]);
|
||||
|
||||
load_and_transpose_s16_8x8(input + 8, 16, &in[8], &in[9], &in[10], &in[11],
|
||||
&in[12], &in[13], &in[14], &in[15]);
|
||||
|
||||
// stage 1
|
||||
s1[16] = multiply_shift_and_narrow_s16(in[1], cospi_31_64);
|
||||
s1[31] = multiply_shift_and_narrow_s16(in[1], cospi_1_64);
|
||||
|
||||
s1[17] = multiply_shift_and_narrow_s16(in[15], -cospi_17_64);
|
||||
s1[30] = multiply_shift_and_narrow_s16(in[15], cospi_15_64);
|
||||
|
||||
s1[18] = multiply_shift_and_narrow_s16(in[9], cospi_23_64);
|
||||
s1[29] = multiply_shift_and_narrow_s16(in[9], cospi_9_64);
|
||||
|
||||
s1[19] = multiply_shift_and_narrow_s16(in[7], -cospi_25_64);
|
||||
s1[28] = multiply_shift_and_narrow_s16(in[7], cospi_7_64);
|
||||
|
||||
s1[20] = multiply_shift_and_narrow_s16(in[5], cospi_27_64);
|
||||
s1[27] = multiply_shift_and_narrow_s16(in[5], cospi_5_64);
|
||||
|
||||
s1[21] = multiply_shift_and_narrow_s16(in[11], -cospi_21_64);
|
||||
s1[26] = multiply_shift_and_narrow_s16(in[11], cospi_11_64);
|
||||
|
||||
s1[22] = multiply_shift_and_narrow_s16(in[13], cospi_19_64);
|
||||
s1[25] = multiply_shift_and_narrow_s16(in[13], cospi_13_64);
|
||||
|
||||
s1[23] = multiply_shift_and_narrow_s16(in[3], -cospi_29_64);
|
||||
s1[24] = multiply_shift_and_narrow_s16(in[3], cospi_3_64);
|
||||
|
||||
// stage 2
|
||||
s2[8] = multiply_shift_and_narrow_s16(in[2], cospi_30_64);
|
||||
s2[15] = multiply_shift_and_narrow_s16(in[2], cospi_2_64);
|
||||
|
||||
s2[9] = multiply_shift_and_narrow_s16(in[14], -cospi_18_64);
|
||||
s2[14] = multiply_shift_and_narrow_s16(in[14], cospi_14_64);
|
||||
|
||||
s2[10] = multiply_shift_and_narrow_s16(in[10], cospi_22_64);
|
||||
s2[13] = multiply_shift_and_narrow_s16(in[10], cospi_10_64);
|
||||
|
||||
s2[11] = multiply_shift_and_narrow_s16(in[6], -cospi_26_64);
|
||||
s2[12] = multiply_shift_and_narrow_s16(in[6], cospi_6_64);
|
||||
|
||||
s2[16] = vaddq_s16(s1[16], s1[17]);
|
||||
s2[17] = vsubq_s16(s1[16], s1[17]);
|
||||
s2[18] = vsubq_s16(s1[19], s1[18]);
|
||||
s2[19] = vaddq_s16(s1[18], s1[19]);
|
||||
s2[20] = vaddq_s16(s1[20], s1[21]);
|
||||
s2[21] = vsubq_s16(s1[20], s1[21]);
|
||||
s2[22] = vsubq_s16(s1[23], s1[22]);
|
||||
s2[23] = vaddq_s16(s1[22], s1[23]);
|
||||
s2[24] = vaddq_s16(s1[24], s1[25]);
|
||||
s2[25] = vsubq_s16(s1[24], s1[25]);
|
||||
s2[26] = vsubq_s16(s1[27], s1[26]);
|
||||
s2[27] = vaddq_s16(s1[26], s1[27]);
|
||||
s2[28] = vaddq_s16(s1[28], s1[29]);
|
||||
s2[29] = vsubq_s16(s1[28], s1[29]);
|
||||
s2[30] = vsubq_s16(s1[31], s1[30]);
|
||||
s2[31] = vaddq_s16(s1[30], s1[31]);
|
||||
|
||||
// stage 3
|
||||
s3[4] = multiply_shift_and_narrow_s16(in[4], cospi_28_64);
|
||||
s3[7] = multiply_shift_and_narrow_s16(in[4], cospi_4_64);
|
||||
|
||||
s3[5] = multiply_shift_and_narrow_s16(in[12], -cospi_20_64);
|
||||
s3[6] = multiply_shift_and_narrow_s16(in[12], cospi_12_64);
|
||||
|
||||
s3[8] = vaddq_s16(s2[8], s2[9]);
|
||||
s3[9] = vsubq_s16(s2[8], s2[9]);
|
||||
s3[10] = vsubq_s16(s2[11], s2[10]);
|
||||
s3[11] = vaddq_s16(s2[10], s2[11]);
|
||||
s3[12] = vaddq_s16(s2[12], s2[13]);
|
||||
s3[13] = vsubq_s16(s2[12], s2[13]);
|
||||
s3[14] = vsubq_s16(s2[15], s2[14]);
|
||||
s3[15] = vaddq_s16(s2[14], s2[15]);
|
||||
|
||||
s3[17] = multiply_accumulate_shift_and_narrow_s16(s2[17], -cospi_4_64, s2[30],
|
||||
cospi_28_64);
|
||||
s3[30] = multiply_accumulate_shift_and_narrow_s16(s2[17], cospi_28_64, s2[30],
|
||||
cospi_4_64);
|
||||
|
||||
s3[18] = multiply_accumulate_shift_and_narrow_s16(s2[18], -cospi_28_64,
|
||||
s2[29], -cospi_4_64);
|
||||
s3[29] = multiply_accumulate_shift_and_narrow_s16(s2[18], -cospi_4_64, s2[29],
|
||||
cospi_28_64);
|
||||
|
||||
s3[21] = multiply_accumulate_shift_and_narrow_s16(s2[21], -cospi_20_64,
|
||||
s2[26], cospi_12_64);
|
||||
s3[26] = multiply_accumulate_shift_and_narrow_s16(s2[21], cospi_12_64, s2[26],
|
||||
cospi_20_64);
|
||||
|
||||
s3[22] = multiply_accumulate_shift_and_narrow_s16(s2[22], -cospi_12_64,
|
||||
s2[25], -cospi_20_64);
|
||||
s3[25] = multiply_accumulate_shift_and_narrow_s16(s2[22], -cospi_20_64,
|
||||
s2[25], cospi_12_64);
|
||||
|
||||
// stage 4
|
||||
s4[0] = multiply_shift_and_narrow_s16(in[0], cospi_16_64);
|
||||
s4[2] = multiply_shift_and_narrow_s16(in[8], cospi_24_64);
|
||||
s4[3] = multiply_shift_and_narrow_s16(in[8], cospi_8_64);
|
||||
|
||||
s4[4] = vaddq_s16(s3[4], s3[5]);
|
||||
s4[5] = vsubq_s16(s3[4], s3[5]);
|
||||
s4[6] = vsubq_s16(s3[7], s3[6]);
|
||||
s4[7] = vaddq_s16(s3[6], s3[7]);
|
||||
|
||||
s4[9] = multiply_accumulate_shift_and_narrow_s16(s3[9], -cospi_8_64, s3[14],
|
||||
cospi_24_64);
|
||||
s4[14] = multiply_accumulate_shift_and_narrow_s16(s3[9], cospi_24_64, s3[14],
|
||||
cospi_8_64);
|
||||
|
||||
s4[10] = multiply_accumulate_shift_and_narrow_s16(s3[10], -cospi_24_64,
|
||||
s3[13], -cospi_8_64);
|
||||
s4[13] = multiply_accumulate_shift_and_narrow_s16(s3[10], -cospi_8_64, s3[13],
|
||||
cospi_24_64);
|
||||
|
||||
s4[16] = vaddq_s16(s2[16], s2[19]);
|
||||
s4[17] = vaddq_s16(s3[17], s3[18]);
|
||||
s4[18] = vsubq_s16(s3[17], s3[18]);
|
||||
s4[19] = vsubq_s16(s2[16], s2[19]);
|
||||
s4[20] = vsubq_s16(s2[23], s2[20]);
|
||||
s4[21] = vsubq_s16(s3[22], s3[21]);
|
||||
s4[22] = vaddq_s16(s3[21], s3[22]);
|
||||
s4[23] = vaddq_s16(s2[20], s2[23]);
|
||||
s4[24] = vaddq_s16(s2[24], s2[27]);
|
||||
s4[25] = vaddq_s16(s3[25], s3[26]);
|
||||
s4[26] = vsubq_s16(s3[25], s3[26]);
|
||||
s4[27] = vsubq_s16(s2[24], s2[27]);
|
||||
s4[28] = vsubq_s16(s2[31], s2[28]);
|
||||
s4[29] = vsubq_s16(s3[30], s3[29]);
|
||||
s4[30] = vaddq_s16(s3[29], s3[30]);
|
||||
s4[31] = vaddq_s16(s2[28], s2[31]);
|
||||
|
||||
// stage 5
|
||||
s5[0] = vaddq_s16(s4[0], s4[3]);
|
||||
s5[1] = vaddq_s16(s4[0], s4[2]);
|
||||
s5[2] = vsubq_s16(s4[0], s4[2]);
|
||||
s5[3] = vsubq_s16(s4[0], s4[3]);
|
||||
|
||||
s5[5] = sub_multiply_shift_and_narrow_s16(s4[6], s4[5], cospi_16_64);
|
||||
s5[6] = add_multiply_shift_and_narrow_s16(s4[5], s4[6], cospi_16_64);
|
||||
|
||||
s5[8] = vaddq_s16(s3[8], s3[11]);
|
||||
s5[9] = vaddq_s16(s4[9], s4[10]);
|
||||
s5[10] = vsubq_s16(s4[9], s4[10]);
|
||||
s5[11] = vsubq_s16(s3[8], s3[11]);
|
||||
s5[12] = vsubq_s16(s3[15], s3[12]);
|
||||
s5[13] = vsubq_s16(s4[14], s4[13]);
|
||||
s5[14] = vaddq_s16(s4[13], s4[14]);
|
||||
s5[15] = vaddq_s16(s3[15], s3[12]);
|
||||
|
||||
s5[18] = multiply_accumulate_shift_and_narrow_s16(s4[18], -cospi_8_64, s4[29],
|
||||
cospi_24_64);
|
||||
s5[29] = multiply_accumulate_shift_and_narrow_s16(s4[18], cospi_24_64, s4[29],
|
||||
cospi_8_64);
|
||||
|
||||
s5[19] = multiply_accumulate_shift_and_narrow_s16(s4[19], -cospi_8_64, s4[28],
|
||||
cospi_24_64);
|
||||
s5[28] = multiply_accumulate_shift_and_narrow_s16(s4[19], cospi_24_64, s4[28],
|
||||
cospi_8_64);
|
||||
|
||||
s5[20] = multiply_accumulate_shift_and_narrow_s16(s4[20], -cospi_24_64,
|
||||
s4[27], -cospi_8_64);
|
||||
s5[27] = multiply_accumulate_shift_and_narrow_s16(s4[20], -cospi_8_64, s4[27],
|
||||
cospi_24_64);
|
||||
|
||||
s5[21] = multiply_accumulate_shift_and_narrow_s16(s4[21], -cospi_24_64,
|
||||
s4[26], -cospi_8_64);
|
||||
s5[26] = multiply_accumulate_shift_and_narrow_s16(s4[21], -cospi_8_64, s4[26],
|
||||
cospi_24_64);
|
||||
|
||||
// stage 6
|
||||
s6[0] = vaddq_s16(s5[0], s4[7]);
|
||||
s6[1] = vaddq_s16(s5[1], s5[6]);
|
||||
s6[2] = vaddq_s16(s5[2], s5[5]);
|
||||
s6[3] = vaddq_s16(s5[3], s4[4]);
|
||||
s6[4] = vsubq_s16(s5[3], s4[4]);
|
||||
s6[5] = vsubq_s16(s5[2], s5[5]);
|
||||
s6[6] = vsubq_s16(s5[1], s5[6]);
|
||||
s6[7] = vsubq_s16(s5[0], s4[7]);
|
||||
|
||||
s6[10] = sub_multiply_shift_and_narrow_s16(s5[13], s5[10], cospi_16_64);
|
||||
s6[13] = add_multiply_shift_and_narrow_s16(s5[10], s5[13], cospi_16_64);
|
||||
|
||||
s6[11] = sub_multiply_shift_and_narrow_s16(s5[12], s5[11], cospi_16_64);
|
||||
s6[12] = add_multiply_shift_and_narrow_s16(s5[11], s5[12], cospi_16_64);
|
||||
|
||||
s6[16] = vaddq_s16(s4[16], s4[23]);
|
||||
s6[17] = vaddq_s16(s4[17], s4[22]);
|
||||
s6[18] = vaddq_s16(s5[18], s5[21]);
|
||||
s6[19] = vaddq_s16(s5[19], s5[20]);
|
||||
s6[20] = vsubq_s16(s5[19], s5[20]);
|
||||
s6[21] = vsubq_s16(s5[18], s5[21]);
|
||||
s6[22] = vsubq_s16(s4[17], s4[22]);
|
||||
s6[23] = vsubq_s16(s4[16], s4[23]);
|
||||
s6[24] = vsubq_s16(s4[31], s4[24]);
|
||||
s6[25] = vsubq_s16(s4[30], s4[25]);
|
||||
s6[26] = vsubq_s16(s5[29], s5[26]);
|
||||
s6[27] = vsubq_s16(s5[28], s5[27]);
|
||||
s6[28] = vaddq_s16(s5[27], s5[28]);
|
||||
s6[29] = vaddq_s16(s5[26], s5[29]);
|
||||
s6[30] = vaddq_s16(s4[25], s4[30]);
|
||||
s6[31] = vaddq_s16(s4[24], s4[31]);
|
||||
|
||||
// stage 7
|
||||
s7[0] = vaddq_s16(s6[0], s5[15]);
|
||||
s7[1] = vaddq_s16(s6[1], s5[14]);
|
||||
s7[2] = vaddq_s16(s6[2], s6[13]);
|
||||
s7[3] = vaddq_s16(s6[3], s6[12]);
|
||||
s7[4] = vaddq_s16(s6[4], s6[11]);
|
||||
s7[5] = vaddq_s16(s6[5], s6[10]);
|
||||
s7[6] = vaddq_s16(s6[6], s5[9]);
|
||||
s7[7] = vaddq_s16(s6[7], s5[8]);
|
||||
s7[8] = vsubq_s16(s6[7], s5[8]);
|
||||
s7[9] = vsubq_s16(s6[6], s5[9]);
|
||||
s7[10] = vsubq_s16(s6[5], s6[10]);
|
||||
s7[11] = vsubq_s16(s6[4], s6[11]);
|
||||
s7[12] = vsubq_s16(s6[3], s6[12]);
|
||||
s7[13] = vsubq_s16(s6[2], s6[13]);
|
||||
s7[14] = vsubq_s16(s6[1], s5[14]);
|
||||
s7[15] = vsubq_s16(s6[0], s5[15]);
|
||||
|
||||
s7[20] = sub_multiply_shift_and_narrow_s16(s6[27], s6[20], cospi_16_64);
|
||||
s7[27] = add_multiply_shift_and_narrow_s16(s6[20], s6[27], cospi_16_64);
|
||||
|
||||
s7[21] = sub_multiply_shift_and_narrow_s16(s6[26], s6[21], cospi_16_64);
|
||||
s7[26] = add_multiply_shift_and_narrow_s16(s6[21], s6[26], cospi_16_64);
|
||||
|
||||
s7[22] = sub_multiply_shift_and_narrow_s16(s6[25], s6[22], cospi_16_64);
|
||||
s7[25] = add_multiply_shift_and_narrow_s16(s6[22], s6[25], cospi_16_64);
|
||||
|
||||
s7[23] = sub_multiply_shift_and_narrow_s16(s6[24], s6[23], cospi_16_64);
|
||||
s7[24] = add_multiply_shift_and_narrow_s16(s6[23], s6[24], cospi_16_64);
|
||||
|
||||
// final stage
|
||||
out[0] = final_add(s7[0], s6[31]);
|
||||
out[1] = final_add(s7[1], s6[30]);
|
||||
out[2] = final_add(s7[2], s6[29]);
|
||||
out[3] = final_add(s7[3], s6[28]);
|
||||
out[4] = final_add(s7[4], s7[27]);
|
||||
out[5] = final_add(s7[5], s7[26]);
|
||||
out[6] = final_add(s7[6], s7[25]);
|
||||
out[7] = final_add(s7[7], s7[24]);
|
||||
out[8] = final_add(s7[8], s7[23]);
|
||||
out[9] = final_add(s7[9], s7[22]);
|
||||
out[10] = final_add(s7[10], s7[21]);
|
||||
out[11] = final_add(s7[11], s7[20]);
|
||||
out[12] = final_add(s7[12], s6[19]);
|
||||
out[13] = final_add(s7[13], s6[18]);
|
||||
out[14] = final_add(s7[14], s6[17]);
|
||||
out[15] = final_add(s7[15], s6[16]);
|
||||
out[16] = final_sub(s7[15], s6[16]);
|
||||
out[17] = final_sub(s7[14], s6[17]);
|
||||
out[18] = final_sub(s7[13], s6[18]);
|
||||
out[19] = final_sub(s7[12], s6[19]);
|
||||
out[20] = final_sub(s7[11], s7[20]);
|
||||
out[21] = final_sub(s7[10], s7[21]);
|
||||
out[22] = final_sub(s7[9], s7[22]);
|
||||
out[23] = final_sub(s7[8], s7[23]);
|
||||
out[24] = final_sub(s7[7], s7[24]);
|
||||
out[25] = final_sub(s7[6], s7[25]);
|
||||
out[26] = final_sub(s7[5], s7[26]);
|
||||
out[27] = final_sub(s7[4], s7[27]);
|
||||
out[28] = final_sub(s7[3], s6[28]);
|
||||
out[29] = final_sub(s7[2], s6[29]);
|
||||
out[30] = final_sub(s7[1], s6[30]);
|
||||
out[31] = final_sub(s7[0], s6[31]);
|
||||
|
||||
if (highbd_flag) {
|
||||
highbd_add_and_store_bd8(out, output, stride);
|
||||
} else {
|
||||
uint8_t *const outputT = (uint8_t *)output;
|
||||
add_and_store_u8_s16(out + 0, outputT, stride);
|
||||
add_and_store_u8_s16(out + 8, outputT + (8 * stride), stride);
|
||||
add_and_store_u8_s16(out + 16, outputT + (16 * stride), stride);
|
||||
add_and_store_u8_s16(out + 24, outputT + (24 * stride), stride);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_idct32x32_135_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
int i;
|
||||
int16_t temp[32 * 16];
|
||||
int16_t *t = temp;
|
||||
|
||||
vpx_idct32_12_neon(input, temp);
|
||||
vpx_idct32_12_neon(input + 32 * 8, temp + 8);
|
||||
|
||||
for (i = 0; i < 32; i += 8) {
|
||||
vpx_idct32_16_neon(t, dest, stride, 0);
|
||||
t += (16 * 8);
|
||||
dest += 8;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/inv_txfm.h"
|
||||
|
||||
static INLINE void idct32x32_1_add_pos_kernel(uint8_t **dest, const int stride,
|
||||
const uint8x16_t res) {
|
||||
const uint8x16_t a0 = vld1q_u8(*dest);
|
||||
const uint8x16_t a1 = vld1q_u8(*dest + 16);
|
||||
const uint8x16_t b0 = vqaddq_u8(a0, res);
|
||||
const uint8x16_t b1 = vqaddq_u8(a1, res);
|
||||
vst1q_u8(*dest, b0);
|
||||
vst1q_u8(*dest + 16, b1);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void idct32x32_1_add_neg_kernel(uint8_t **dest, const int stride,
|
||||
const uint8x16_t res) {
|
||||
const uint8x16_t a0 = vld1q_u8(*dest);
|
||||
const uint8x16_t a1 = vld1q_u8(*dest + 16);
|
||||
const uint8x16_t b0 = vqsubq_u8(a0, res);
|
||||
const uint8x16_t b1 = vqsubq_u8(a1, res);
|
||||
vst1q_u8(*dest, b0);
|
||||
vst1q_u8(*dest + 16, b1);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
void vpx_idct32x32_1_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
int i;
|
||||
const int16_t out0 =
|
||||
WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64));
|
||||
const int16_t out1 = WRAPLOW(dct_const_round_shift(out0 * cospi_16_64));
|
||||
const int16_t a1 = ROUND_POWER_OF_TWO(out1, 6);
|
||||
|
||||
if (a1 >= 0) {
|
||||
const uint8x16_t dc = create_dcq(a1);
|
||||
for (i = 0; i < 32; i++) {
|
||||
idct32x32_1_add_pos_kernel(&dest, stride, dc);
|
||||
}
|
||||
} else {
|
||||
const uint8x16_t dc = create_dcq(-a1);
|
||||
for (i = 0; i < 32; i++) {
|
||||
idct32x32_1_add_neg_kernel(&dest, stride, dc);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,513 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
|
||||
// Only for the first pass of the _34_ variant. Since it only uses values from
|
||||
// the top left 8x8 it can safely assume all the remaining values are 0 and skip
|
||||
// an awful lot of calculations. In fact, only the first 6 columns make the cut.
|
||||
// None of the elements in the 7th or 8th column are used so it skips any calls
|
||||
// to input[67] too.
|
||||
// In C this does a single row of 32 for each call. Here it transposes the top
|
||||
// left 8x8 to allow using SIMD.
|
||||
|
||||
// vp9/common/vp9_scan.c:vp9_default_iscan_32x32 arranges the first 34 non-zero
|
||||
// coefficients as follows:
|
||||
// 0 1 2 3 4 5 6 7
|
||||
// 0 0 2 5 10 17 25
|
||||
// 1 1 4 8 15 22 30
|
||||
// 2 3 7 12 18 28
|
||||
// 3 6 11 16 23 31
|
||||
// 4 9 14 19 29
|
||||
// 5 13 20 26
|
||||
// 6 21 27 33
|
||||
// 7 24 32
|
||||
void vpx_idct32_6_neon(const tran_low_t *input, int16_t *output) {
|
||||
int16x8_t in[8], s1[32], s2[32], s3[32];
|
||||
|
||||
in[0] = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
in[1] = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
in[2] = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
in[3] = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
in[4] = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
in[5] = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
in[6] = load_tran_low_to_s16q(input);
|
||||
input += 32;
|
||||
in[7] = load_tran_low_to_s16q(input);
|
||||
transpose_s16_8x8(&in[0], &in[1], &in[2], &in[3], &in[4], &in[5], &in[6],
|
||||
&in[7]);
|
||||
|
||||
// stage 1
|
||||
// input[1] * cospi_31_64 - input[31] * cospi_1_64 (but input[31] == 0)
|
||||
s1[16] = multiply_shift_and_narrow_s16(in[1], cospi_31_64);
|
||||
// input[1] * cospi_1_64 + input[31] * cospi_31_64 (but input[31] == 0)
|
||||
s1[31] = multiply_shift_and_narrow_s16(in[1], cospi_1_64);
|
||||
|
||||
s1[20] = multiply_shift_and_narrow_s16(in[5], cospi_27_64);
|
||||
s1[27] = multiply_shift_and_narrow_s16(in[5], cospi_5_64);
|
||||
|
||||
s1[23] = multiply_shift_and_narrow_s16(in[3], -cospi_29_64);
|
||||
s1[24] = multiply_shift_and_narrow_s16(in[3], cospi_3_64);
|
||||
|
||||
// stage 2
|
||||
s2[8] = multiply_shift_and_narrow_s16(in[2], cospi_30_64);
|
||||
s2[15] = multiply_shift_and_narrow_s16(in[2], cospi_2_64);
|
||||
|
||||
// stage 3
|
||||
s1[4] = multiply_shift_and_narrow_s16(in[4], cospi_28_64);
|
||||
s1[7] = multiply_shift_and_narrow_s16(in[4], cospi_4_64);
|
||||
|
||||
s1[17] = multiply_accumulate_shift_and_narrow_s16(s1[16], -cospi_4_64, s1[31],
|
||||
cospi_28_64);
|
||||
s1[30] = multiply_accumulate_shift_and_narrow_s16(s1[16], cospi_28_64, s1[31],
|
||||
cospi_4_64);
|
||||
|
||||
s1[21] = multiply_accumulate_shift_and_narrow_s16(s1[20], -cospi_20_64,
|
||||
s1[27], cospi_12_64);
|
||||
s1[26] = multiply_accumulate_shift_and_narrow_s16(s1[20], cospi_12_64, s1[27],
|
||||
cospi_20_64);
|
||||
|
||||
s1[22] = multiply_accumulate_shift_and_narrow_s16(s1[23], -cospi_12_64,
|
||||
s1[24], -cospi_20_64);
|
||||
s1[25] = multiply_accumulate_shift_and_narrow_s16(s1[23], -cospi_20_64,
|
||||
s1[24], cospi_12_64);
|
||||
|
||||
// stage 4
|
||||
s1[0] = multiply_shift_and_narrow_s16(in[0], cospi_16_64);
|
||||
|
||||
s2[9] = multiply_accumulate_shift_and_narrow_s16(s2[8], -cospi_8_64, s2[15],
|
||||
cospi_24_64);
|
||||
s2[14] = multiply_accumulate_shift_and_narrow_s16(s2[8], cospi_24_64, s2[15],
|
||||
cospi_8_64);
|
||||
|
||||
s2[20] = vsubq_s16(s1[23], s1[20]);
|
||||
s2[21] = vsubq_s16(s1[22], s1[21]);
|
||||
s2[22] = vaddq_s16(s1[21], s1[22]);
|
||||
s2[23] = vaddq_s16(s1[20], s1[23]);
|
||||
s2[24] = vaddq_s16(s1[24], s1[27]);
|
||||
s2[25] = vaddq_s16(s1[25], s1[26]);
|
||||
s2[26] = vsubq_s16(s1[25], s1[26]);
|
||||
s2[27] = vsubq_s16(s1[24], s1[27]);
|
||||
|
||||
// stage 5
|
||||
s1[5] = sub_multiply_shift_and_narrow_s16(s1[7], s1[4], cospi_16_64);
|
||||
s1[6] = add_multiply_shift_and_narrow_s16(s1[4], s1[7], cospi_16_64);
|
||||
|
||||
s1[18] = multiply_accumulate_shift_and_narrow_s16(s1[17], -cospi_8_64, s1[30],
|
||||
cospi_24_64);
|
||||
s1[29] = multiply_accumulate_shift_and_narrow_s16(s1[17], cospi_24_64, s1[30],
|
||||
cospi_8_64);
|
||||
|
||||
s1[19] = multiply_accumulate_shift_and_narrow_s16(s1[16], -cospi_8_64, s1[31],
|
||||
cospi_24_64);
|
||||
s1[28] = multiply_accumulate_shift_and_narrow_s16(s1[16], cospi_24_64, s1[31],
|
||||
cospi_8_64);
|
||||
|
||||
s1[20] = multiply_accumulate_shift_and_narrow_s16(s2[20], -cospi_24_64,
|
||||
s2[27], -cospi_8_64);
|
||||
s1[27] = multiply_accumulate_shift_and_narrow_s16(s2[20], -cospi_8_64, s2[27],
|
||||
cospi_24_64);
|
||||
|
||||
s1[21] = multiply_accumulate_shift_and_narrow_s16(s2[21], -cospi_24_64,
|
||||
s2[26], -cospi_8_64);
|
||||
s1[26] = multiply_accumulate_shift_and_narrow_s16(s2[21], -cospi_8_64, s2[26],
|
||||
cospi_24_64);
|
||||
|
||||
// stage 6
|
||||
s2[0] = vaddq_s16(s1[0], s1[7]);
|
||||
s2[1] = vaddq_s16(s1[0], s1[6]);
|
||||
s2[2] = vaddq_s16(s1[0], s1[5]);
|
||||
s2[3] = vaddq_s16(s1[0], s1[4]);
|
||||
s2[4] = vsubq_s16(s1[0], s1[4]);
|
||||
s2[5] = vsubq_s16(s1[0], s1[5]);
|
||||
s2[6] = vsubq_s16(s1[0], s1[6]);
|
||||
s2[7] = vsubq_s16(s1[0], s1[7]);
|
||||
|
||||
s2[10] = sub_multiply_shift_and_narrow_s16(s2[14], s2[9], cospi_16_64);
|
||||
s2[13] = add_multiply_shift_and_narrow_s16(s2[9], s2[14], cospi_16_64);
|
||||
|
||||
s2[11] = sub_multiply_shift_and_narrow_s16(s2[15], s2[8], cospi_16_64);
|
||||
s2[12] = add_multiply_shift_and_narrow_s16(s2[8], s2[15], cospi_16_64);
|
||||
|
||||
s2[16] = vaddq_s16(s1[16], s2[23]);
|
||||
s2[17] = vaddq_s16(s1[17], s2[22]);
|
||||
s2[18] = vaddq_s16(s1[18], s1[21]);
|
||||
s2[19] = vaddq_s16(s1[19], s1[20]);
|
||||
s2[20] = vsubq_s16(s1[19], s1[20]);
|
||||
s2[21] = vsubq_s16(s1[18], s1[21]);
|
||||
s2[22] = vsubq_s16(s1[17], s2[22]);
|
||||
s2[23] = vsubq_s16(s1[16], s2[23]);
|
||||
|
||||
s3[24] = vsubq_s16(s1[31], s2[24]);
|
||||
s3[25] = vsubq_s16(s1[30], s2[25]);
|
||||
s3[26] = vsubq_s16(s1[29], s1[26]);
|
||||
s3[27] = vsubq_s16(s1[28], s1[27]);
|
||||
s2[28] = vaddq_s16(s1[27], s1[28]);
|
||||
s2[29] = vaddq_s16(s1[26], s1[29]);
|
||||
s2[30] = vaddq_s16(s2[25], s1[30]);
|
||||
s2[31] = vaddq_s16(s2[24], s1[31]);
|
||||
|
||||
// stage 7
|
||||
s1[0] = vaddq_s16(s2[0], s2[15]);
|
||||
s1[1] = vaddq_s16(s2[1], s2[14]);
|
||||
s1[2] = vaddq_s16(s2[2], s2[13]);
|
||||
s1[3] = vaddq_s16(s2[3], s2[12]);
|
||||
s1[4] = vaddq_s16(s2[4], s2[11]);
|
||||
s1[5] = vaddq_s16(s2[5], s2[10]);
|
||||
s1[6] = vaddq_s16(s2[6], s2[9]);
|
||||
s1[7] = vaddq_s16(s2[7], s2[8]);
|
||||
s1[8] = vsubq_s16(s2[7], s2[8]);
|
||||
s1[9] = vsubq_s16(s2[6], s2[9]);
|
||||
s1[10] = vsubq_s16(s2[5], s2[10]);
|
||||
s1[11] = vsubq_s16(s2[4], s2[11]);
|
||||
s1[12] = vsubq_s16(s2[3], s2[12]);
|
||||
s1[13] = vsubq_s16(s2[2], s2[13]);
|
||||
s1[14] = vsubq_s16(s2[1], s2[14]);
|
||||
s1[15] = vsubq_s16(s2[0], s2[15]);
|
||||
|
||||
s1[20] = sub_multiply_shift_and_narrow_s16(s3[27], s2[20], cospi_16_64);
|
||||
s1[27] = add_multiply_shift_and_narrow_s16(s2[20], s3[27], cospi_16_64);
|
||||
|
||||
s1[21] = sub_multiply_shift_and_narrow_s16(s3[26], s2[21], cospi_16_64);
|
||||
s1[26] = add_multiply_shift_and_narrow_s16(s2[21], s3[26], cospi_16_64);
|
||||
|
||||
s1[22] = sub_multiply_shift_and_narrow_s16(s3[25], s2[22], cospi_16_64);
|
||||
s1[25] = add_multiply_shift_and_narrow_s16(s2[22], s3[25], cospi_16_64);
|
||||
|
||||
s1[23] = sub_multiply_shift_and_narrow_s16(s3[24], s2[23], cospi_16_64);
|
||||
s1[24] = add_multiply_shift_and_narrow_s16(s2[23], s3[24], cospi_16_64);
|
||||
|
||||
// final stage
|
||||
vst1q_s16(output, vaddq_s16(s1[0], s2[31]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[1], s2[30]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[2], s2[29]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[3], s2[28]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[4], s1[27]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[5], s1[26]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[6], s1[25]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[7], s1[24]));
|
||||
output += 8;
|
||||
|
||||
vst1q_s16(output, vaddq_s16(s1[8], s1[23]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[9], s1[22]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[10], s1[21]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[11], s1[20]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[12], s2[19]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[13], s2[18]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[14], s2[17]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vaddq_s16(s1[15], s2[16]));
|
||||
output += 8;
|
||||
|
||||
vst1q_s16(output, vsubq_s16(s1[15], s2[16]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[14], s2[17]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[13], s2[18]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[12], s2[19]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[11], s1[20]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[10], s1[21]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[9], s1[22]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[8], s1[23]));
|
||||
output += 8;
|
||||
|
||||
vst1q_s16(output, vsubq_s16(s1[7], s1[24]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[6], s1[25]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[5], s1[26]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[4], s1[27]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[3], s2[28]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[2], s2[29]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[1], s2[30]));
|
||||
output += 8;
|
||||
vst1q_s16(output, vsubq_s16(s1[0], s2[31]));
|
||||
}
|
||||
|
||||
void vpx_idct32_8_neon(const int16_t *input, void *const output, int stride,
|
||||
const int highbd_flag) {
|
||||
int16x8_t in[8], s1[32], s2[32], s3[32], out[32];
|
||||
|
||||
load_and_transpose_s16_8x8(input, 8, &in[0], &in[1], &in[2], &in[3], &in[4],
|
||||
&in[5], &in[6], &in[7]);
|
||||
|
||||
// stage 1
|
||||
s1[16] = multiply_shift_and_narrow_s16(in[1], cospi_31_64);
|
||||
s1[31] = multiply_shift_and_narrow_s16(in[1], cospi_1_64);
|
||||
|
||||
// Different for _8_
|
||||
s1[19] = multiply_shift_and_narrow_s16(in[7], -cospi_25_64);
|
||||
s1[28] = multiply_shift_and_narrow_s16(in[7], cospi_7_64);
|
||||
|
||||
s1[20] = multiply_shift_and_narrow_s16(in[5], cospi_27_64);
|
||||
s1[27] = multiply_shift_and_narrow_s16(in[5], cospi_5_64);
|
||||
|
||||
s1[23] = multiply_shift_and_narrow_s16(in[3], -cospi_29_64);
|
||||
s1[24] = multiply_shift_and_narrow_s16(in[3], cospi_3_64);
|
||||
|
||||
// stage 2
|
||||
s2[8] = multiply_shift_and_narrow_s16(in[2], cospi_30_64);
|
||||
s2[15] = multiply_shift_and_narrow_s16(in[2], cospi_2_64);
|
||||
|
||||
s2[11] = multiply_shift_and_narrow_s16(in[6], -cospi_26_64);
|
||||
s2[12] = multiply_shift_and_narrow_s16(in[6], cospi_6_64);
|
||||
|
||||
// stage 3
|
||||
s1[4] = multiply_shift_and_narrow_s16(in[4], cospi_28_64);
|
||||
s1[7] = multiply_shift_and_narrow_s16(in[4], cospi_4_64);
|
||||
|
||||
s1[17] = multiply_accumulate_shift_and_narrow_s16(s1[16], -cospi_4_64, s1[31],
|
||||
cospi_28_64);
|
||||
s1[30] = multiply_accumulate_shift_and_narrow_s16(s1[16], cospi_28_64, s1[31],
|
||||
cospi_4_64);
|
||||
|
||||
// Different for _8_
|
||||
s1[18] = multiply_accumulate_shift_and_narrow_s16(s1[19], -cospi_28_64,
|
||||
s1[28], -cospi_4_64);
|
||||
s1[29] = multiply_accumulate_shift_and_narrow_s16(s1[19], -cospi_4_64, s1[28],
|
||||
cospi_28_64);
|
||||
|
||||
s1[21] = multiply_accumulate_shift_and_narrow_s16(s1[20], -cospi_20_64,
|
||||
s1[27], cospi_12_64);
|
||||
s1[26] = multiply_accumulate_shift_and_narrow_s16(s1[20], cospi_12_64, s1[27],
|
||||
cospi_20_64);
|
||||
|
||||
s1[22] = multiply_accumulate_shift_and_narrow_s16(s1[23], -cospi_12_64,
|
||||
s1[24], -cospi_20_64);
|
||||
s1[25] = multiply_accumulate_shift_and_narrow_s16(s1[23], -cospi_20_64,
|
||||
s1[24], cospi_12_64);
|
||||
|
||||
// stage 4
|
||||
s1[0] = multiply_shift_and_narrow_s16(in[0], cospi_16_64);
|
||||
|
||||
s2[9] = multiply_accumulate_shift_and_narrow_s16(s2[8], -cospi_8_64, s2[15],
|
||||
cospi_24_64);
|
||||
s2[14] = multiply_accumulate_shift_and_narrow_s16(s2[8], cospi_24_64, s2[15],
|
||||
cospi_8_64);
|
||||
|
||||
s2[10] = multiply_accumulate_shift_and_narrow_s16(s2[11], -cospi_24_64,
|
||||
s2[12], -cospi_8_64);
|
||||
s2[13] = multiply_accumulate_shift_and_narrow_s16(s2[11], -cospi_8_64, s2[12],
|
||||
cospi_24_64);
|
||||
|
||||
s2[16] = vaddq_s16(s1[16], s1[19]);
|
||||
|
||||
s2[17] = vaddq_s16(s1[17], s1[18]);
|
||||
s2[18] = vsubq_s16(s1[17], s1[18]);
|
||||
|
||||
s2[19] = vsubq_s16(s1[16], s1[19]);
|
||||
|
||||
s2[20] = vsubq_s16(s1[23], s1[20]);
|
||||
s2[21] = vsubq_s16(s1[22], s1[21]);
|
||||
|
||||
s2[22] = vaddq_s16(s1[21], s1[22]);
|
||||
s2[23] = vaddq_s16(s1[20], s1[23]);
|
||||
|
||||
s2[24] = vaddq_s16(s1[24], s1[27]);
|
||||
s2[25] = vaddq_s16(s1[25], s1[26]);
|
||||
s2[26] = vsubq_s16(s1[25], s1[26]);
|
||||
s2[27] = vsubq_s16(s1[24], s1[27]);
|
||||
|
||||
s2[28] = vsubq_s16(s1[31], s1[28]);
|
||||
s2[29] = vsubq_s16(s1[30], s1[29]);
|
||||
s2[30] = vaddq_s16(s1[29], s1[30]);
|
||||
s2[31] = vaddq_s16(s1[28], s1[31]);
|
||||
|
||||
// stage 5
|
||||
s1[5] = sub_multiply_shift_and_narrow_s16(s1[7], s1[4], cospi_16_64);
|
||||
s1[6] = add_multiply_shift_and_narrow_s16(s1[4], s1[7], cospi_16_64);
|
||||
|
||||
s1[8] = vaddq_s16(s2[8], s2[11]);
|
||||
s1[9] = vaddq_s16(s2[9], s2[10]);
|
||||
s1[10] = vsubq_s16(s2[9], s2[10]);
|
||||
s1[11] = vsubq_s16(s2[8], s2[11]);
|
||||
s1[12] = vsubq_s16(s2[15], s2[12]);
|
||||
s1[13] = vsubq_s16(s2[14], s2[13]);
|
||||
s1[14] = vaddq_s16(s2[13], s2[14]);
|
||||
s1[15] = vaddq_s16(s2[12], s2[15]);
|
||||
|
||||
s1[18] = multiply_accumulate_shift_and_narrow_s16(s2[18], -cospi_8_64, s2[29],
|
||||
cospi_24_64);
|
||||
s1[29] = multiply_accumulate_shift_and_narrow_s16(s2[18], cospi_24_64, s2[29],
|
||||
cospi_8_64);
|
||||
|
||||
s1[19] = multiply_accumulate_shift_and_narrow_s16(s2[19], -cospi_8_64, s2[28],
|
||||
cospi_24_64);
|
||||
s1[28] = multiply_accumulate_shift_and_narrow_s16(s2[19], cospi_24_64, s2[28],
|
||||
cospi_8_64);
|
||||
|
||||
s1[20] = multiply_accumulate_shift_and_narrow_s16(s2[20], -cospi_24_64,
|
||||
s2[27], -cospi_8_64);
|
||||
s1[27] = multiply_accumulate_shift_and_narrow_s16(s2[20], -cospi_8_64, s2[27],
|
||||
cospi_24_64);
|
||||
|
||||
s1[21] = multiply_accumulate_shift_and_narrow_s16(s2[21], -cospi_24_64,
|
||||
s2[26], -cospi_8_64);
|
||||
s1[26] = multiply_accumulate_shift_and_narrow_s16(s2[21], -cospi_8_64, s2[26],
|
||||
cospi_24_64);
|
||||
|
||||
// stage 6
|
||||
s2[0] = vaddq_s16(s1[0], s1[7]);
|
||||
s2[1] = vaddq_s16(s1[0], s1[6]);
|
||||
s2[2] = vaddq_s16(s1[0], s1[5]);
|
||||
s2[3] = vaddq_s16(s1[0], s1[4]);
|
||||
s2[4] = vsubq_s16(s1[0], s1[4]);
|
||||
s2[5] = vsubq_s16(s1[0], s1[5]);
|
||||
s2[6] = vsubq_s16(s1[0], s1[6]);
|
||||
s2[7] = vsubq_s16(s1[0], s1[7]);
|
||||
|
||||
s2[10] = sub_multiply_shift_and_narrow_s16(s1[13], s1[10], cospi_16_64);
|
||||
s2[13] = add_multiply_shift_and_narrow_s16(s1[10], s1[13], cospi_16_64);
|
||||
|
||||
s2[11] = sub_multiply_shift_and_narrow_s16(s1[12], s1[11], cospi_16_64);
|
||||
s2[12] = add_multiply_shift_and_narrow_s16(s1[11], s1[12], cospi_16_64);
|
||||
|
||||
s1[16] = vaddq_s16(s2[16], s2[23]);
|
||||
s1[17] = vaddq_s16(s2[17], s2[22]);
|
||||
s2[18] = vaddq_s16(s1[18], s1[21]);
|
||||
s2[19] = vaddq_s16(s1[19], s1[20]);
|
||||
s2[20] = vsubq_s16(s1[19], s1[20]);
|
||||
s2[21] = vsubq_s16(s1[18], s1[21]);
|
||||
s1[22] = vsubq_s16(s2[17], s2[22]);
|
||||
s1[23] = vsubq_s16(s2[16], s2[23]);
|
||||
|
||||
s3[24] = vsubq_s16(s2[31], s2[24]);
|
||||
s3[25] = vsubq_s16(s2[30], s2[25]);
|
||||
s3[26] = vsubq_s16(s1[29], s1[26]);
|
||||
s3[27] = vsubq_s16(s1[28], s1[27]);
|
||||
s2[28] = vaddq_s16(s1[27], s1[28]);
|
||||
s2[29] = vaddq_s16(s1[26], s1[29]);
|
||||
s2[30] = vaddq_s16(s2[25], s2[30]);
|
||||
s2[31] = vaddq_s16(s2[24], s2[31]);
|
||||
|
||||
// stage 7
|
||||
s1[0] = vaddq_s16(s2[0], s1[15]);
|
||||
s1[1] = vaddq_s16(s2[1], s1[14]);
|
||||
s1[2] = vaddq_s16(s2[2], s2[13]);
|
||||
s1[3] = vaddq_s16(s2[3], s2[12]);
|
||||
s1[4] = vaddq_s16(s2[4], s2[11]);
|
||||
s1[5] = vaddq_s16(s2[5], s2[10]);
|
||||
s1[6] = vaddq_s16(s2[6], s1[9]);
|
||||
s1[7] = vaddq_s16(s2[7], s1[8]);
|
||||
s1[8] = vsubq_s16(s2[7], s1[8]);
|
||||
s1[9] = vsubq_s16(s2[6], s1[9]);
|
||||
s1[10] = vsubq_s16(s2[5], s2[10]);
|
||||
s1[11] = vsubq_s16(s2[4], s2[11]);
|
||||
s1[12] = vsubq_s16(s2[3], s2[12]);
|
||||
s1[13] = vsubq_s16(s2[2], s2[13]);
|
||||
s1[14] = vsubq_s16(s2[1], s1[14]);
|
||||
s1[15] = vsubq_s16(s2[0], s1[15]);
|
||||
|
||||
s1[20] = sub_multiply_shift_and_narrow_s16(s3[27], s2[20], cospi_16_64);
|
||||
s1[27] = add_multiply_shift_and_narrow_s16(s2[20], s3[27], cospi_16_64);
|
||||
|
||||
s1[21] = sub_multiply_shift_and_narrow_s16(s3[26], s2[21], cospi_16_64);
|
||||
s1[26] = add_multiply_shift_and_narrow_s16(s2[21], s3[26], cospi_16_64);
|
||||
|
||||
s2[22] = sub_multiply_shift_and_narrow_s16(s3[25], s1[22], cospi_16_64);
|
||||
s1[25] = add_multiply_shift_and_narrow_s16(s1[22], s3[25], cospi_16_64);
|
||||
|
||||
s2[23] = sub_multiply_shift_and_narrow_s16(s3[24], s1[23], cospi_16_64);
|
||||
s1[24] = add_multiply_shift_and_narrow_s16(s1[23], s3[24], cospi_16_64);
|
||||
|
||||
// final stage
|
||||
out[0] = final_add(s1[0], s2[31]);
|
||||
out[1] = final_add(s1[1], s2[30]);
|
||||
out[2] = final_add(s1[2], s2[29]);
|
||||
out[3] = final_add(s1[3], s2[28]);
|
||||
out[4] = final_add(s1[4], s1[27]);
|
||||
out[5] = final_add(s1[5], s1[26]);
|
||||
out[6] = final_add(s1[6], s1[25]);
|
||||
out[7] = final_add(s1[7], s1[24]);
|
||||
out[8] = final_add(s1[8], s2[23]);
|
||||
out[9] = final_add(s1[9], s2[22]);
|
||||
out[10] = final_add(s1[10], s1[21]);
|
||||
out[11] = final_add(s1[11], s1[20]);
|
||||
out[12] = final_add(s1[12], s2[19]);
|
||||
out[13] = final_add(s1[13], s2[18]);
|
||||
out[14] = final_add(s1[14], s1[17]);
|
||||
out[15] = final_add(s1[15], s1[16]);
|
||||
out[16] = final_sub(s1[15], s1[16]);
|
||||
out[17] = final_sub(s1[14], s1[17]);
|
||||
out[18] = final_sub(s1[13], s2[18]);
|
||||
out[19] = final_sub(s1[12], s2[19]);
|
||||
out[20] = final_sub(s1[11], s1[20]);
|
||||
out[21] = final_sub(s1[10], s1[21]);
|
||||
out[22] = final_sub(s1[9], s2[22]);
|
||||
out[23] = final_sub(s1[8], s2[23]);
|
||||
out[24] = final_sub(s1[7], s1[24]);
|
||||
out[25] = final_sub(s1[6], s1[25]);
|
||||
out[26] = final_sub(s1[5], s1[26]);
|
||||
out[27] = final_sub(s1[4], s1[27]);
|
||||
out[28] = final_sub(s1[3], s2[28]);
|
||||
out[29] = final_sub(s1[2], s2[29]);
|
||||
out[30] = final_sub(s1[1], s2[30]);
|
||||
out[31] = final_sub(s1[0], s2[31]);
|
||||
|
||||
if (highbd_flag) {
|
||||
highbd_add_and_store_bd8(out, output, stride);
|
||||
} else {
|
||||
uint8_t *const outputT = (uint8_t *)output;
|
||||
add_and_store_u8_s16(out + 0, outputT, stride);
|
||||
add_and_store_u8_s16(out + 8, outputT + (8 * stride), stride);
|
||||
add_and_store_u8_s16(out + 16, outputT + (16 * stride), stride);
|
||||
add_and_store_u8_s16(out + 24, outputT + (24 * stride), stride);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_idct32x32_34_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
int i;
|
||||
int16_t temp[32 * 8];
|
||||
int16_t *t = temp;
|
||||
|
||||
vpx_idct32_6_neon(input, t);
|
||||
|
||||
for (i = 0; i < 32; i += 8) {
|
||||
vpx_idct32_8_neon(t, dest, stride, 0);
|
||||
t += (8 * 8);
|
||||
dest += 8;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,776 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
|
||||
static INLINE void load_from_transformed(const int16_t *const trans_buf,
|
||||
const int first, const int second,
|
||||
int16x8_t *const q0,
|
||||
int16x8_t *const q1) {
|
||||
*q0 = vld1q_s16(trans_buf + first * 8);
|
||||
*q1 = vld1q_s16(trans_buf + second * 8);
|
||||
}
|
||||
|
||||
static INLINE void load_from_output(const int16_t *const out, const int first,
|
||||
const int second, int16x8_t *const q0,
|
||||
int16x8_t *const q1) {
|
||||
*q0 = vld1q_s16(out + first * 32);
|
||||
*q1 = vld1q_s16(out + second * 32);
|
||||
}
|
||||
|
||||
static INLINE void store_in_output(int16_t *const out, const int first,
|
||||
const int second, const int16x8_t q0,
|
||||
const int16x8_t q1) {
|
||||
vst1q_s16(out + first * 32, q0);
|
||||
vst1q_s16(out + second * 32, q1);
|
||||
}
|
||||
|
||||
static INLINE void store_combine_results(uint8_t *p1, uint8_t *p2,
|
||||
const int stride, int16x8_t q0,
|
||||
int16x8_t q1, int16x8_t q2,
|
||||
int16x8_t q3) {
|
||||
uint8x8_t d[4];
|
||||
|
||||
d[0] = vld1_u8(p1);
|
||||
p1 += stride;
|
||||
d[1] = vld1_u8(p1);
|
||||
d[3] = vld1_u8(p2);
|
||||
p2 -= stride;
|
||||
d[2] = vld1_u8(p2);
|
||||
|
||||
q0 = vrshrq_n_s16(q0, 6);
|
||||
q1 = vrshrq_n_s16(q1, 6);
|
||||
q2 = vrshrq_n_s16(q2, 6);
|
||||
q3 = vrshrq_n_s16(q3, 6);
|
||||
|
||||
q0 = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(q0), d[0]));
|
||||
q1 = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(q1), d[1]));
|
||||
q2 = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(q2), d[2]));
|
||||
q3 = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(q3), d[3]));
|
||||
|
||||
d[0] = vqmovun_s16(q0);
|
||||
d[1] = vqmovun_s16(q1);
|
||||
d[2] = vqmovun_s16(q2);
|
||||
d[3] = vqmovun_s16(q3);
|
||||
|
||||
vst1_u8(p1, d[1]);
|
||||
p1 -= stride;
|
||||
vst1_u8(p1, d[0]);
|
||||
vst1_u8(p2, d[2]);
|
||||
p2 += stride;
|
||||
vst1_u8(p2, d[3]);
|
||||
}
|
||||
|
||||
static INLINE void highbd_store_combine_results_bd8(uint16_t *p1, uint16_t *p2,
|
||||
const int stride,
|
||||
int16x8_t q0, int16x8_t q1,
|
||||
int16x8_t q2,
|
||||
int16x8_t q3) {
|
||||
uint16x8_t d[4];
|
||||
|
||||
d[0] = vld1q_u16(p1);
|
||||
p1 += stride;
|
||||
d[1] = vld1q_u16(p1);
|
||||
d[3] = vld1q_u16(p2);
|
||||
p2 -= stride;
|
||||
d[2] = vld1q_u16(p2);
|
||||
|
||||
q0 = vrshrq_n_s16(q0, 6);
|
||||
q1 = vrshrq_n_s16(q1, 6);
|
||||
q2 = vrshrq_n_s16(q2, 6);
|
||||
q3 = vrshrq_n_s16(q3, 6);
|
||||
|
||||
q0 = vaddq_s16(q0, vreinterpretq_s16_u16(d[0]));
|
||||
q1 = vaddq_s16(q1, vreinterpretq_s16_u16(d[1]));
|
||||
q2 = vaddq_s16(q2, vreinterpretq_s16_u16(d[2]));
|
||||
q3 = vaddq_s16(q3, vreinterpretq_s16_u16(d[3]));
|
||||
|
||||
d[0] = vmovl_u8(vqmovun_s16(q0));
|
||||
d[1] = vmovl_u8(vqmovun_s16(q1));
|
||||
d[2] = vmovl_u8(vqmovun_s16(q2));
|
||||
d[3] = vmovl_u8(vqmovun_s16(q3));
|
||||
|
||||
vst1q_u16(p1, d[1]);
|
||||
p1 -= stride;
|
||||
vst1q_u16(p1, d[0]);
|
||||
vst1q_u16(p2, d[2]);
|
||||
p2 += stride;
|
||||
vst1q_u16(p2, d[3]);
|
||||
}
|
||||
|
||||
static INLINE void do_butterfly(const int16x8_t qIn0, const int16x8_t qIn1,
|
||||
const int16_t first_const,
|
||||
const int16_t second_const,
|
||||
int16x8_t *const qOut0,
|
||||
int16x8_t *const qOut1) {
|
||||
int32x4_t q[4];
|
||||
int16x4_t d[6];
|
||||
|
||||
d[0] = vget_low_s16(qIn0);
|
||||
d[1] = vget_high_s16(qIn0);
|
||||
d[2] = vget_low_s16(qIn1);
|
||||
d[3] = vget_high_s16(qIn1);
|
||||
|
||||
// Note: using v{mul, mla, mls}l_n_s16 here slows down 35% with gcc 4.9.
|
||||
d[4] = vdup_n_s16(first_const);
|
||||
d[5] = vdup_n_s16(second_const);
|
||||
|
||||
q[0] = vmull_s16(d[0], d[4]);
|
||||
q[1] = vmull_s16(d[1], d[4]);
|
||||
q[0] = vmlsl_s16(q[0], d[2], d[5]);
|
||||
q[1] = vmlsl_s16(q[1], d[3], d[5]);
|
||||
|
||||
q[2] = vmull_s16(d[0], d[5]);
|
||||
q[3] = vmull_s16(d[1], d[5]);
|
||||
q[2] = vmlal_s16(q[2], d[2], d[4]);
|
||||
q[3] = vmlal_s16(q[3], d[3], d[4]);
|
||||
|
||||
*qOut0 = vcombine_s16(vrshrn_n_s32(q[0], DCT_CONST_BITS),
|
||||
vrshrn_n_s32(q[1], DCT_CONST_BITS));
|
||||
*qOut1 = vcombine_s16(vrshrn_n_s32(q[2], DCT_CONST_BITS),
|
||||
vrshrn_n_s32(q[3], DCT_CONST_BITS));
|
||||
}
|
||||
|
||||
static INLINE void load_s16x8q(const int16_t *in, int16x8_t *const s0,
|
||||
int16x8_t *const s1, int16x8_t *const s2,
|
||||
int16x8_t *const s3, int16x8_t *const s4,
|
||||
int16x8_t *const s5, int16x8_t *const s6,
|
||||
int16x8_t *const s7) {
|
||||
*s0 = vld1q_s16(in);
|
||||
in += 32;
|
||||
*s1 = vld1q_s16(in);
|
||||
in += 32;
|
||||
*s2 = vld1q_s16(in);
|
||||
in += 32;
|
||||
*s3 = vld1q_s16(in);
|
||||
in += 32;
|
||||
*s4 = vld1q_s16(in);
|
||||
in += 32;
|
||||
*s5 = vld1q_s16(in);
|
||||
in += 32;
|
||||
*s6 = vld1q_s16(in);
|
||||
in += 32;
|
||||
*s7 = vld1q_s16(in);
|
||||
}
|
||||
|
||||
static INLINE void transpose_and_store_s16_8x8(int16x8_t a0, int16x8_t a1,
|
||||
int16x8_t a2, int16x8_t a3,
|
||||
int16x8_t a4, int16x8_t a5,
|
||||
int16x8_t a6, int16x8_t a7,
|
||||
int16_t **out) {
|
||||
transpose_s16_8x8(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
|
||||
|
||||
vst1q_s16(*out, a0);
|
||||
*out += 8;
|
||||
vst1q_s16(*out, a1);
|
||||
*out += 8;
|
||||
vst1q_s16(*out, a2);
|
||||
*out += 8;
|
||||
vst1q_s16(*out, a3);
|
||||
*out += 8;
|
||||
vst1q_s16(*out, a4);
|
||||
*out += 8;
|
||||
vst1q_s16(*out, a5);
|
||||
*out += 8;
|
||||
vst1q_s16(*out, a6);
|
||||
*out += 8;
|
||||
vst1q_s16(*out, a7);
|
||||
*out += 8;
|
||||
}
|
||||
|
||||
static INLINE void idct32_transpose_pair(const int16_t *input, int16_t *t_buf) {
|
||||
int i;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
|
||||
|
||||
for (i = 0; i < 4; i++, input += 8) {
|
||||
load_s16x8q(input, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
|
||||
transpose_and_store_s16_8x8(s0, s1, s2, s3, s4, s5, s6, s7, &t_buf);
|
||||
}
|
||||
}
|
||||
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
static INLINE void load_s16x8q_tran_low(
|
||||
const tran_low_t *in, int16x8_t *const s0, int16x8_t *const s1,
|
||||
int16x8_t *const s2, int16x8_t *const s3, int16x8_t *const s4,
|
||||
int16x8_t *const s5, int16x8_t *const s6, int16x8_t *const s7) {
|
||||
*s0 = load_tran_low_to_s16q(in);
|
||||
in += 32;
|
||||
*s1 = load_tran_low_to_s16q(in);
|
||||
in += 32;
|
||||
*s2 = load_tran_low_to_s16q(in);
|
||||
in += 32;
|
||||
*s3 = load_tran_low_to_s16q(in);
|
||||
in += 32;
|
||||
*s4 = load_tran_low_to_s16q(in);
|
||||
in += 32;
|
||||
*s5 = load_tran_low_to_s16q(in);
|
||||
in += 32;
|
||||
*s6 = load_tran_low_to_s16q(in);
|
||||
in += 32;
|
||||
*s7 = load_tran_low_to_s16q(in);
|
||||
}
|
||||
|
||||
static INLINE void idct32_transpose_pair_tran_low(const tran_low_t *input,
|
||||
int16_t *t_buf) {
|
||||
int i;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
|
||||
|
||||
for (i = 0; i < 4; i++, input += 8) {
|
||||
load_s16x8q_tran_low(input, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
|
||||
transpose_and_store_s16_8x8(s0, s1, s2, s3, s4, s5, s6, s7, &t_buf);
|
||||
}
|
||||
}
|
||||
#else // !CONFIG_VP9_HIGHBITDEPTH
|
||||
#define idct32_transpose_pair_tran_low idct32_transpose_pair
|
||||
#endif // CONFIG_VP9_HIGHBITDEPTH
|
||||
|
||||
static INLINE void idct32_bands_end_1st_pass(int16_t *const out,
|
||||
int16x8_t *const q) {
|
||||
store_in_output(out, 16, 17, q[6], q[7]);
|
||||
store_in_output(out, 14, 15, q[8], q[9]);
|
||||
|
||||
load_from_output(out, 30, 31, &q[0], &q[1]);
|
||||
q[4] = vaddq_s16(q[2], q[1]);
|
||||
q[5] = vaddq_s16(q[3], q[0]);
|
||||
q[6] = vsubq_s16(q[3], q[0]);
|
||||
q[7] = vsubq_s16(q[2], q[1]);
|
||||
store_in_output(out, 30, 31, q[6], q[7]);
|
||||
store_in_output(out, 0, 1, q[4], q[5]);
|
||||
|
||||
load_from_output(out, 12, 13, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[10], q[1]);
|
||||
q[3] = vaddq_s16(q[11], q[0]);
|
||||
q[4] = vsubq_s16(q[11], q[0]);
|
||||
q[5] = vsubq_s16(q[10], q[1]);
|
||||
|
||||
load_from_output(out, 18, 19, &q[0], &q[1]);
|
||||
q[8] = vaddq_s16(q[4], q[1]);
|
||||
q[9] = vaddq_s16(q[5], q[0]);
|
||||
q[6] = vsubq_s16(q[5], q[0]);
|
||||
q[7] = vsubq_s16(q[4], q[1]);
|
||||
store_in_output(out, 18, 19, q[6], q[7]);
|
||||
store_in_output(out, 12, 13, q[8], q[9]);
|
||||
|
||||
load_from_output(out, 28, 29, &q[0], &q[1]);
|
||||
q[4] = vaddq_s16(q[2], q[1]);
|
||||
q[5] = vaddq_s16(q[3], q[0]);
|
||||
q[6] = vsubq_s16(q[3], q[0]);
|
||||
q[7] = vsubq_s16(q[2], q[1]);
|
||||
store_in_output(out, 28, 29, q[6], q[7]);
|
||||
store_in_output(out, 2, 3, q[4], q[5]);
|
||||
|
||||
load_from_output(out, 10, 11, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[12], q[1]);
|
||||
q[3] = vaddq_s16(q[13], q[0]);
|
||||
q[4] = vsubq_s16(q[13], q[0]);
|
||||
q[5] = vsubq_s16(q[12], q[1]);
|
||||
|
||||
load_from_output(out, 20, 21, &q[0], &q[1]);
|
||||
q[8] = vaddq_s16(q[4], q[1]);
|
||||
q[9] = vaddq_s16(q[5], q[0]);
|
||||
q[6] = vsubq_s16(q[5], q[0]);
|
||||
q[7] = vsubq_s16(q[4], q[1]);
|
||||
store_in_output(out, 20, 21, q[6], q[7]);
|
||||
store_in_output(out, 10, 11, q[8], q[9]);
|
||||
|
||||
load_from_output(out, 26, 27, &q[0], &q[1]);
|
||||
q[4] = vaddq_s16(q[2], q[1]);
|
||||
q[5] = vaddq_s16(q[3], q[0]);
|
||||
q[6] = vsubq_s16(q[3], q[0]);
|
||||
q[7] = vsubq_s16(q[2], q[1]);
|
||||
store_in_output(out, 26, 27, q[6], q[7]);
|
||||
store_in_output(out, 4, 5, q[4], q[5]);
|
||||
|
||||
load_from_output(out, 8, 9, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[14], q[1]);
|
||||
q[3] = vaddq_s16(q[15], q[0]);
|
||||
q[4] = vsubq_s16(q[15], q[0]);
|
||||
q[5] = vsubq_s16(q[14], q[1]);
|
||||
|
||||
load_from_output(out, 22, 23, &q[0], &q[1]);
|
||||
q[8] = vaddq_s16(q[4], q[1]);
|
||||
q[9] = vaddq_s16(q[5], q[0]);
|
||||
q[6] = vsubq_s16(q[5], q[0]);
|
||||
q[7] = vsubq_s16(q[4], q[1]);
|
||||
store_in_output(out, 22, 23, q[6], q[7]);
|
||||
store_in_output(out, 8, 9, q[8], q[9]);
|
||||
|
||||
load_from_output(out, 24, 25, &q[0], &q[1]);
|
||||
q[4] = vaddq_s16(q[2], q[1]);
|
||||
q[5] = vaddq_s16(q[3], q[0]);
|
||||
q[6] = vsubq_s16(q[3], q[0]);
|
||||
q[7] = vsubq_s16(q[2], q[1]);
|
||||
store_in_output(out, 24, 25, q[6], q[7]);
|
||||
store_in_output(out, 6, 7, q[4], q[5]);
|
||||
}
|
||||
|
||||
static INLINE void idct32_bands_end_2nd_pass(const int16_t *const out,
|
||||
uint8_t *const dest,
|
||||
const int stride,
|
||||
int16x8_t *const q) {
|
||||
uint8_t *dest0 = dest + 0 * stride;
|
||||
uint8_t *dest1 = dest + 31 * stride;
|
||||
uint8_t *dest2 = dest + 16 * stride;
|
||||
uint8_t *dest3 = dest + 15 * stride;
|
||||
const int str2 = stride << 1;
|
||||
|
||||
store_combine_results(dest2, dest3, stride, q[6], q[7], q[8], q[9]);
|
||||
dest2 += str2;
|
||||
dest3 -= str2;
|
||||
|
||||
load_from_output(out, 30, 31, &q[0], &q[1]);
|
||||
q[4] = final_add(q[2], q[1]);
|
||||
q[5] = final_add(q[3], q[0]);
|
||||
q[6] = final_sub(q[3], q[0]);
|
||||
q[7] = final_sub(q[2], q[1]);
|
||||
store_combine_results(dest0, dest1, stride, q[4], q[5], q[6], q[7]);
|
||||
dest0 += str2;
|
||||
dest1 -= str2;
|
||||
|
||||
load_from_output(out, 12, 13, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[10], q[1]);
|
||||
q[3] = vaddq_s16(q[11], q[0]);
|
||||
q[4] = vsubq_s16(q[11], q[0]);
|
||||
q[5] = vsubq_s16(q[10], q[1]);
|
||||
|
||||
load_from_output(out, 18, 19, &q[0], &q[1]);
|
||||
q[8] = final_add(q[4], q[1]);
|
||||
q[9] = final_add(q[5], q[0]);
|
||||
q[6] = final_sub(q[5], q[0]);
|
||||
q[7] = final_sub(q[4], q[1]);
|
||||
store_combine_results(dest2, dest3, stride, q[6], q[7], q[8], q[9]);
|
||||
dest2 += str2;
|
||||
dest3 -= str2;
|
||||
|
||||
load_from_output(out, 28, 29, &q[0], &q[1]);
|
||||
q[4] = final_add(q[2], q[1]);
|
||||
q[5] = final_add(q[3], q[0]);
|
||||
q[6] = final_sub(q[3], q[0]);
|
||||
q[7] = final_sub(q[2], q[1]);
|
||||
store_combine_results(dest0, dest1, stride, q[4], q[5], q[6], q[7]);
|
||||
dest0 += str2;
|
||||
dest1 -= str2;
|
||||
|
||||
load_from_output(out, 10, 11, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[12], q[1]);
|
||||
q[3] = vaddq_s16(q[13], q[0]);
|
||||
q[4] = vsubq_s16(q[13], q[0]);
|
||||
q[5] = vsubq_s16(q[12], q[1]);
|
||||
|
||||
load_from_output(out, 20, 21, &q[0], &q[1]);
|
||||
q[8] = final_add(q[4], q[1]);
|
||||
q[9] = final_add(q[5], q[0]);
|
||||
q[6] = final_sub(q[5], q[0]);
|
||||
q[7] = final_sub(q[4], q[1]);
|
||||
store_combine_results(dest2, dest3, stride, q[6], q[7], q[8], q[9]);
|
||||
dest2 += str2;
|
||||
dest3 -= str2;
|
||||
|
||||
load_from_output(out, 26, 27, &q[0], &q[1]);
|
||||
q[4] = final_add(q[2], q[1]);
|
||||
q[5] = final_add(q[3], q[0]);
|
||||
q[6] = final_sub(q[3], q[0]);
|
||||
q[7] = final_sub(q[2], q[1]);
|
||||
store_combine_results(dest0, dest1, stride, q[4], q[5], q[6], q[7]);
|
||||
dest0 += str2;
|
||||
dest1 -= str2;
|
||||
|
||||
load_from_output(out, 8, 9, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[14], q[1]);
|
||||
q[3] = vaddq_s16(q[15], q[0]);
|
||||
q[4] = vsubq_s16(q[15], q[0]);
|
||||
q[5] = vsubq_s16(q[14], q[1]);
|
||||
|
||||
load_from_output(out, 22, 23, &q[0], &q[1]);
|
||||
q[8] = final_add(q[4], q[1]);
|
||||
q[9] = final_add(q[5], q[0]);
|
||||
q[6] = final_sub(q[5], q[0]);
|
||||
q[7] = final_sub(q[4], q[1]);
|
||||
store_combine_results(dest2, dest3, stride, q[6], q[7], q[8], q[9]);
|
||||
|
||||
load_from_output(out, 24, 25, &q[0], &q[1]);
|
||||
q[4] = final_add(q[2], q[1]);
|
||||
q[5] = final_add(q[3], q[0]);
|
||||
q[6] = final_sub(q[3], q[0]);
|
||||
q[7] = final_sub(q[2], q[1]);
|
||||
store_combine_results(dest0, dest1, stride, q[4], q[5], q[6], q[7]);
|
||||
}
|
||||
|
||||
static INLINE void highbd_idct32_bands_end_2nd_pass_bd8(
|
||||
const int16_t *const out, uint16_t *const dest, const int stride,
|
||||
int16x8_t *const q) {
|
||||
uint16_t *dest0 = dest + 0 * stride;
|
||||
uint16_t *dest1 = dest + 31 * stride;
|
||||
uint16_t *dest2 = dest + 16 * stride;
|
||||
uint16_t *dest3 = dest + 15 * stride;
|
||||
const int str2 = stride << 1;
|
||||
|
||||
highbd_store_combine_results_bd8(dest2, dest3, stride, q[6], q[7], q[8],
|
||||
q[9]);
|
||||
dest2 += str2;
|
||||
dest3 -= str2;
|
||||
|
||||
load_from_output(out, 30, 31, &q[0], &q[1]);
|
||||
q[4] = final_add(q[2], q[1]);
|
||||
q[5] = final_add(q[3], q[0]);
|
||||
q[6] = final_sub(q[3], q[0]);
|
||||
q[7] = final_sub(q[2], q[1]);
|
||||
highbd_store_combine_results_bd8(dest0, dest1, stride, q[4], q[5], q[6],
|
||||
q[7]);
|
||||
dest0 += str2;
|
||||
dest1 -= str2;
|
||||
|
||||
load_from_output(out, 12, 13, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[10], q[1]);
|
||||
q[3] = vaddq_s16(q[11], q[0]);
|
||||
q[4] = vsubq_s16(q[11], q[0]);
|
||||
q[5] = vsubq_s16(q[10], q[1]);
|
||||
|
||||
load_from_output(out, 18, 19, &q[0], &q[1]);
|
||||
q[8] = final_add(q[4], q[1]);
|
||||
q[9] = final_add(q[5], q[0]);
|
||||
q[6] = final_sub(q[5], q[0]);
|
||||
q[7] = final_sub(q[4], q[1]);
|
||||
highbd_store_combine_results_bd8(dest2, dest3, stride, q[6], q[7], q[8],
|
||||
q[9]);
|
||||
dest2 += str2;
|
||||
dest3 -= str2;
|
||||
|
||||
load_from_output(out, 28, 29, &q[0], &q[1]);
|
||||
q[4] = final_add(q[2], q[1]);
|
||||
q[5] = final_add(q[3], q[0]);
|
||||
q[6] = final_sub(q[3], q[0]);
|
||||
q[7] = final_sub(q[2], q[1]);
|
||||
highbd_store_combine_results_bd8(dest0, dest1, stride, q[4], q[5], q[6],
|
||||
q[7]);
|
||||
dest0 += str2;
|
||||
dest1 -= str2;
|
||||
|
||||
load_from_output(out, 10, 11, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[12], q[1]);
|
||||
q[3] = vaddq_s16(q[13], q[0]);
|
||||
q[4] = vsubq_s16(q[13], q[0]);
|
||||
q[5] = vsubq_s16(q[12], q[1]);
|
||||
|
||||
load_from_output(out, 20, 21, &q[0], &q[1]);
|
||||
q[8] = final_add(q[4], q[1]);
|
||||
q[9] = final_add(q[5], q[0]);
|
||||
q[6] = final_sub(q[5], q[0]);
|
||||
q[7] = final_sub(q[4], q[1]);
|
||||
highbd_store_combine_results_bd8(dest2, dest3, stride, q[6], q[7], q[8],
|
||||
q[9]);
|
||||
dest2 += str2;
|
||||
dest3 -= str2;
|
||||
|
||||
load_from_output(out, 26, 27, &q[0], &q[1]);
|
||||
q[4] = final_add(q[2], q[1]);
|
||||
q[5] = final_add(q[3], q[0]);
|
||||
q[6] = final_sub(q[3], q[0]);
|
||||
q[7] = final_sub(q[2], q[1]);
|
||||
highbd_store_combine_results_bd8(dest0, dest1, stride, q[4], q[5], q[6],
|
||||
q[7]);
|
||||
dest0 += str2;
|
||||
dest1 -= str2;
|
||||
|
||||
load_from_output(out, 8, 9, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[14], q[1]);
|
||||
q[3] = vaddq_s16(q[15], q[0]);
|
||||
q[4] = vsubq_s16(q[15], q[0]);
|
||||
q[5] = vsubq_s16(q[14], q[1]);
|
||||
|
||||
load_from_output(out, 22, 23, &q[0], &q[1]);
|
||||
q[8] = final_add(q[4], q[1]);
|
||||
q[9] = final_add(q[5], q[0]);
|
||||
q[6] = final_sub(q[5], q[0]);
|
||||
q[7] = final_sub(q[4], q[1]);
|
||||
highbd_store_combine_results_bd8(dest2, dest3, stride, q[6], q[7], q[8],
|
||||
q[9]);
|
||||
|
||||
load_from_output(out, 24, 25, &q[0], &q[1]);
|
||||
q[4] = final_add(q[2], q[1]);
|
||||
q[5] = final_add(q[3], q[0]);
|
||||
q[6] = final_sub(q[3], q[0]);
|
||||
q[7] = final_sub(q[2], q[1]);
|
||||
highbd_store_combine_results_bd8(dest0, dest1, stride, q[4], q[5], q[6],
|
||||
q[7]);
|
||||
}
|
||||
|
||||
void vpx_idct32_32_neon(const tran_low_t *input, uint8_t *dest,
|
||||
const int stride, const int highbd_flag) {
|
||||
int i, idct32_pass_loop;
|
||||
int16_t trans_buf[32 * 8];
|
||||
int16_t pass1[32 * 32];
|
||||
int16_t pass2[32 * 32];
|
||||
const int16_t *input_pass2 = pass1; // input of pass2 is the result of pass1
|
||||
int16_t *out;
|
||||
int16x8_t q[16];
|
||||
uint16_t *dst = CAST_TO_SHORTPTR(dest);
|
||||
|
||||
for (idct32_pass_loop = 0, out = pass1; idct32_pass_loop < 2;
|
||||
idct32_pass_loop++, out = pass2) {
|
||||
for (i = 0; i < 4; i++, out += 8) { // idct32_bands_loop
|
||||
if (idct32_pass_loop == 0) {
|
||||
idct32_transpose_pair_tran_low(input, trans_buf);
|
||||
input += 32 * 8;
|
||||
} else {
|
||||
idct32_transpose_pair(input_pass2, trans_buf);
|
||||
input_pass2 += 32 * 8;
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// BLOCK A: 16-19,28-31
|
||||
// -----------------------------------------
|
||||
// generate 16,17,30,31
|
||||
// part of stage 1
|
||||
load_from_transformed(trans_buf, 1, 31, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_31_64, cospi_1_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 17, 15, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_15_64, cospi_17_64, &q[1], &q[3]);
|
||||
// part of stage 2
|
||||
q[4] = vaddq_s16(q[0], q[1]);
|
||||
q[13] = vsubq_s16(q[0], q[1]);
|
||||
q[6] = vaddq_s16(q[2], q[3]);
|
||||
q[14] = vsubq_s16(q[2], q[3]);
|
||||
// part of stage 3
|
||||
do_butterfly(q[14], q[13], cospi_28_64, cospi_4_64, &q[5], &q[7]);
|
||||
|
||||
// generate 18,19,28,29
|
||||
// part of stage 1
|
||||
load_from_transformed(trans_buf, 9, 23, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_23_64, cospi_9_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 25, 7, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_7_64, cospi_25_64, &q[1], &q[3]);
|
||||
// part of stage 2
|
||||
q[13] = vsubq_s16(q[3], q[2]);
|
||||
q[3] = vaddq_s16(q[3], q[2]);
|
||||
q[14] = vsubq_s16(q[1], q[0]);
|
||||
q[2] = vaddq_s16(q[1], q[0]);
|
||||
// part of stage 3
|
||||
do_butterfly(q[14], q[13], -cospi_4_64, -cospi_28_64, &q[1], &q[0]);
|
||||
// part of stage 4
|
||||
q[8] = vaddq_s16(q[4], q[2]);
|
||||
q[9] = vaddq_s16(q[5], q[0]);
|
||||
q[10] = vaddq_s16(q[7], q[1]);
|
||||
q[15] = vaddq_s16(q[6], q[3]);
|
||||
q[13] = vsubq_s16(q[5], q[0]);
|
||||
q[14] = vsubq_s16(q[7], q[1]);
|
||||
store_in_output(out, 16, 31, q[8], q[15]);
|
||||
store_in_output(out, 17, 30, q[9], q[10]);
|
||||
// part of stage 5
|
||||
do_butterfly(q[14], q[13], cospi_24_64, cospi_8_64, &q[0], &q[1]);
|
||||
store_in_output(out, 29, 18, q[1], q[0]);
|
||||
// part of stage 4
|
||||
q[13] = vsubq_s16(q[4], q[2]);
|
||||
q[14] = vsubq_s16(q[6], q[3]);
|
||||
// part of stage 5
|
||||
do_butterfly(q[14], q[13], cospi_24_64, cospi_8_64, &q[4], &q[6]);
|
||||
store_in_output(out, 19, 28, q[4], q[6]);
|
||||
|
||||
// -----------------------------------------
|
||||
// BLOCK B: 20-23,24-27
|
||||
// -----------------------------------------
|
||||
// generate 20,21,26,27
|
||||
// part of stage 1
|
||||
load_from_transformed(trans_buf, 5, 27, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_27_64, cospi_5_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 21, 11, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_11_64, cospi_21_64, &q[1], &q[3]);
|
||||
// part of stage 2
|
||||
q[13] = vsubq_s16(q[0], q[1]);
|
||||
q[0] = vaddq_s16(q[0], q[1]);
|
||||
q[14] = vsubq_s16(q[2], q[3]);
|
||||
q[2] = vaddq_s16(q[2], q[3]);
|
||||
// part of stage 3
|
||||
do_butterfly(q[14], q[13], cospi_12_64, cospi_20_64, &q[1], &q[3]);
|
||||
|
||||
// generate 22,23,24,25
|
||||
// part of stage 1
|
||||
load_from_transformed(trans_buf, 13, 19, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_19_64, cospi_13_64, &q[5], &q[7]);
|
||||
load_from_transformed(trans_buf, 29, 3, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_3_64, cospi_29_64, &q[4], &q[6]);
|
||||
// part of stage 2
|
||||
q[14] = vsubq_s16(q[4], q[5]);
|
||||
q[5] = vaddq_s16(q[4], q[5]);
|
||||
q[13] = vsubq_s16(q[6], q[7]);
|
||||
q[6] = vaddq_s16(q[6], q[7]);
|
||||
// part of stage 3
|
||||
do_butterfly(q[14], q[13], -cospi_20_64, -cospi_12_64, &q[4], &q[7]);
|
||||
// part of stage 4
|
||||
q[10] = vaddq_s16(q[7], q[1]);
|
||||
q[11] = vaddq_s16(q[5], q[0]);
|
||||
q[12] = vaddq_s16(q[6], q[2]);
|
||||
q[15] = vaddq_s16(q[4], q[3]);
|
||||
// part of stage 6
|
||||
load_from_output(out, 16, 17, &q[14], &q[13]);
|
||||
q[8] = vaddq_s16(q[14], q[11]);
|
||||
q[9] = vaddq_s16(q[13], q[10]);
|
||||
q[13] = vsubq_s16(q[13], q[10]);
|
||||
q[11] = vsubq_s16(q[14], q[11]);
|
||||
store_in_output(out, 17, 16, q[9], q[8]);
|
||||
load_from_output(out, 30, 31, &q[14], &q[9]);
|
||||
q[8] = vsubq_s16(q[9], q[12]);
|
||||
q[10] = vaddq_s16(q[14], q[15]);
|
||||
q[14] = vsubq_s16(q[14], q[15]);
|
||||
q[12] = vaddq_s16(q[9], q[12]);
|
||||
store_in_output(out, 30, 31, q[10], q[12]);
|
||||
// part of stage 7
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[13], &q[14]);
|
||||
store_in_output(out, 25, 22, q[14], q[13]);
|
||||
do_butterfly(q[8], q[11], cospi_16_64, cospi_16_64, &q[13], &q[14]);
|
||||
store_in_output(out, 24, 23, q[14], q[13]);
|
||||
// part of stage 4
|
||||
q[14] = vsubq_s16(q[5], q[0]);
|
||||
q[13] = vsubq_s16(q[6], q[2]);
|
||||
do_butterfly(q[14], q[13], -cospi_8_64, -cospi_24_64, &q[5], &q[6]);
|
||||
q[14] = vsubq_s16(q[7], q[1]);
|
||||
q[13] = vsubq_s16(q[4], q[3]);
|
||||
do_butterfly(q[14], q[13], -cospi_8_64, -cospi_24_64, &q[0], &q[1]);
|
||||
// part of stage 6
|
||||
load_from_output(out, 18, 19, &q[14], &q[13]);
|
||||
q[8] = vaddq_s16(q[14], q[1]);
|
||||
q[9] = vaddq_s16(q[13], q[6]);
|
||||
q[13] = vsubq_s16(q[13], q[6]);
|
||||
q[1] = vsubq_s16(q[14], q[1]);
|
||||
store_in_output(out, 18, 19, q[8], q[9]);
|
||||
load_from_output(out, 28, 29, &q[8], &q[9]);
|
||||
q[14] = vsubq_s16(q[8], q[5]);
|
||||
q[10] = vaddq_s16(q[8], q[5]);
|
||||
q[11] = vaddq_s16(q[9], q[0]);
|
||||
q[0] = vsubq_s16(q[9], q[0]);
|
||||
store_in_output(out, 28, 29, q[10], q[11]);
|
||||
// part of stage 7
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[13], &q[14]);
|
||||
store_in_output(out, 20, 27, q[13], q[14]);
|
||||
do_butterfly(q[0], q[1], cospi_16_64, cospi_16_64, &q[1], &q[0]);
|
||||
store_in_output(out, 21, 26, q[1], q[0]);
|
||||
|
||||
// -----------------------------------------
|
||||
// BLOCK C: 8-10,11-15
|
||||
// -----------------------------------------
|
||||
// generate 8,9,14,15
|
||||
// part of stage 2
|
||||
load_from_transformed(trans_buf, 2, 30, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_30_64, cospi_2_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 18, 14, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_14_64, cospi_18_64, &q[1], &q[3]);
|
||||
// part of stage 3
|
||||
q[13] = vsubq_s16(q[0], q[1]);
|
||||
q[0] = vaddq_s16(q[0], q[1]);
|
||||
q[14] = vsubq_s16(q[2], q[3]);
|
||||
q[2] = vaddq_s16(q[2], q[3]);
|
||||
// part of stage 4
|
||||
do_butterfly(q[14], q[13], cospi_24_64, cospi_8_64, &q[1], &q[3]);
|
||||
|
||||
// generate 10,11,12,13
|
||||
// part of stage 2
|
||||
load_from_transformed(trans_buf, 10, 22, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_22_64, cospi_10_64, &q[5], &q[7]);
|
||||
load_from_transformed(trans_buf, 26, 6, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_6_64, cospi_26_64, &q[4], &q[6]);
|
||||
// part of stage 3
|
||||
q[14] = vsubq_s16(q[4], q[5]);
|
||||
q[5] = vaddq_s16(q[4], q[5]);
|
||||
q[13] = vsubq_s16(q[6], q[7]);
|
||||
q[6] = vaddq_s16(q[6], q[7]);
|
||||
// part of stage 4
|
||||
do_butterfly(q[14], q[13], -cospi_8_64, -cospi_24_64, &q[4], &q[7]);
|
||||
// part of stage 5
|
||||
q[8] = vaddq_s16(q[0], q[5]);
|
||||
q[9] = vaddq_s16(q[1], q[7]);
|
||||
q[13] = vsubq_s16(q[1], q[7]);
|
||||
q[14] = vsubq_s16(q[3], q[4]);
|
||||
q[10] = vaddq_s16(q[3], q[4]);
|
||||
q[15] = vaddq_s16(q[2], q[6]);
|
||||
store_in_output(out, 8, 15, q[8], q[15]);
|
||||
store_in_output(out, 9, 14, q[9], q[10]);
|
||||
// part of stage 6
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[1], &q[3]);
|
||||
store_in_output(out, 13, 10, q[3], q[1]);
|
||||
q[13] = vsubq_s16(q[0], q[5]);
|
||||
q[14] = vsubq_s16(q[2], q[6]);
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[1], &q[3]);
|
||||
store_in_output(out, 11, 12, q[1], q[3]);
|
||||
|
||||
// -----------------------------------------
|
||||
// BLOCK D: 0-3,4-7
|
||||
// -----------------------------------------
|
||||
// generate 4,5,6,7
|
||||
// part of stage 3
|
||||
load_from_transformed(trans_buf, 4, 28, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_28_64, cospi_4_64, &q[0], &q[2]);
|
||||
load_from_transformed(trans_buf, 20, 12, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_12_64, cospi_20_64, &q[1], &q[3]);
|
||||
// part of stage 4
|
||||
q[13] = vsubq_s16(q[0], q[1]);
|
||||
q[0] = vaddq_s16(q[0], q[1]);
|
||||
q[14] = vsubq_s16(q[2], q[3]);
|
||||
q[2] = vaddq_s16(q[2], q[3]);
|
||||
// part of stage 5
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[1], &q[3]);
|
||||
|
||||
// generate 0,1,2,3
|
||||
// part of stage 4
|
||||
load_from_transformed(trans_buf, 0, 16, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_16_64, cospi_16_64, &q[5], &q[7]);
|
||||
load_from_transformed(trans_buf, 8, 24, &q[14], &q[13]);
|
||||
do_butterfly(q[14], q[13], cospi_24_64, cospi_8_64, &q[14], &q[6]);
|
||||
// part of stage 5
|
||||
q[4] = vaddq_s16(q[7], q[6]);
|
||||
q[7] = vsubq_s16(q[7], q[6]);
|
||||
q[6] = vsubq_s16(q[5], q[14]);
|
||||
q[5] = vaddq_s16(q[5], q[14]);
|
||||
// part of stage 6
|
||||
q[8] = vaddq_s16(q[4], q[2]);
|
||||
q[9] = vaddq_s16(q[5], q[3]);
|
||||
q[10] = vaddq_s16(q[6], q[1]);
|
||||
q[11] = vaddq_s16(q[7], q[0]);
|
||||
q[12] = vsubq_s16(q[7], q[0]);
|
||||
q[13] = vsubq_s16(q[6], q[1]);
|
||||
q[14] = vsubq_s16(q[5], q[3]);
|
||||
q[15] = vsubq_s16(q[4], q[2]);
|
||||
// part of stage 7
|
||||
load_from_output(out, 14, 15, &q[0], &q[1]);
|
||||
q[2] = vaddq_s16(q[8], q[1]);
|
||||
q[3] = vaddq_s16(q[9], q[0]);
|
||||
q[4] = vsubq_s16(q[9], q[0]);
|
||||
q[5] = vsubq_s16(q[8], q[1]);
|
||||
load_from_output(out, 16, 17, &q[0], &q[1]);
|
||||
q[8] = final_add(q[4], q[1]);
|
||||
q[9] = final_add(q[5], q[0]);
|
||||
q[6] = final_sub(q[5], q[0]);
|
||||
q[7] = final_sub(q[4], q[1]);
|
||||
|
||||
if (idct32_pass_loop == 0) {
|
||||
idct32_bands_end_1st_pass(out, q);
|
||||
} else {
|
||||
if (highbd_flag) {
|
||||
highbd_idct32_bands_end_2nd_pass_bd8(out, dst, stride, q);
|
||||
dst += 8;
|
||||
} else {
|
||||
idct32_bands_end_2nd_pass(out, dest, stride, q);
|
||||
dest += 8;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_idct32x32_1024_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
vpx_idct32_32_neon(input, dest, stride, 0);
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
;
|
||||
; Copyright (c) 2013 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license and patent
|
||||
; grant that can be found in the LICENSE file in the root of the source
|
||||
; tree. All contributing project authors may be found in the AUTHORS
|
||||
; file in the root of the source tree.
|
||||
;
|
||||
|
||||
|
||||
EXPORT |vpx_idct4x4_1_add_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
;void vpx_idct4x4_1_add_neon(int16_t *input, uint8_t *dest, int stride)
|
||||
;
|
||||
; r0 int16_t input
|
||||
; r1 uint8_t *dest
|
||||
; r2 int stride)
|
||||
|
||||
|vpx_idct4x4_1_add_neon| PROC
|
||||
ldrsh r0, [r0]
|
||||
|
||||
; cospi_16_64 = 11585
|
||||
movw r12, #0x2d41
|
||||
|
||||
; out = dct_const_round_shift(input[0] * cospi_16_64)
|
||||
mul r0, r0, r12 ; input[0] * cospi_16_64
|
||||
add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1))
|
||||
asr r0, r0, #14 ; >> DCT_CONST_BITS
|
||||
|
||||
; out = dct_const_round_shift(out * cospi_16_64)
|
||||
mul r0, r0, r12 ; out * cospi_16_64
|
||||
mov r12, r1 ; save dest
|
||||
add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1))
|
||||
asr r0, r0, #14 ; >> DCT_CONST_BITS
|
||||
|
||||
; a1 = ROUND_POWER_OF_TWO(out, 4)
|
||||
add r0, r0, #8 ; + (1 <<((4) - 1))
|
||||
asr r0, r0, #4 ; >> 4
|
||||
|
||||
vdup.s16 q0, r0 ; duplicate a1
|
||||
|
||||
vld1.32 {d2[0]}, [r1], r2
|
||||
vld1.32 {d2[1]}, [r1], r2
|
||||
vld1.32 {d4[0]}, [r1], r2
|
||||
vld1.32 {d4[1]}, [r1]
|
||||
|
||||
vaddw.u8 q8, q0, d2 ; dest[x] + a1
|
||||
vaddw.u8 q9, q0, d4
|
||||
|
||||
vqmovun.s16 d6, q8 ; clip_pixel
|
||||
vqmovun.s16 d7, q9
|
||||
|
||||
vst1.32 {d6[0]}, [r12], r2
|
||||
vst1.32 {d6[1]}, [r12], r2
|
||||
vst1.32 {d7[0]}, [r12], r2
|
||||
vst1.32 {d7[1]}, [r12]
|
||||
|
||||
bx lr
|
||||
ENDP ; |vpx_idct4x4_1_add_neon|
|
||||
|
||||
END
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/inv_txfm.h"
|
||||
|
||||
static INLINE void idct4x4_1_add_kernel(uint8_t **dest, const int stride,
|
||||
const int16x8_t res,
|
||||
uint32x2_t *const d) {
|
||||
uint16x8_t a;
|
||||
uint8x8_t b;
|
||||
*d = vld1_lane_u32((const uint32_t *)*dest, *d, 0);
|
||||
*d = vld1_lane_u32((const uint32_t *)(*dest + stride), *d, 1);
|
||||
a = vaddw_u8(vreinterpretq_u16_s16(res), vreinterpret_u8_u32(*d));
|
||||
b = vqmovun_s16(vreinterpretq_s16_u16(a));
|
||||
vst1_lane_u32((uint32_t *)*dest, vreinterpret_u32_u8(b), 0);
|
||||
*dest += stride;
|
||||
vst1_lane_u32((uint32_t *)*dest, vreinterpret_u32_u8(b), 1);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
void vpx_idct4x4_1_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
const int16_t out0 =
|
||||
WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64));
|
||||
const int16_t out1 = WRAPLOW(dct_const_round_shift(out0 * cospi_16_64));
|
||||
const int16_t a1 = ROUND_POWER_OF_TWO(out1, 4);
|
||||
const int16x8_t dc = vdupq_n_s16(a1);
|
||||
uint32x2_t d = vdup_n_u32(0);
|
||||
|
||||
assert(!((intptr_t)dest % sizeof(uint32_t)));
|
||||
assert(!(stride % sizeof(uint32_t)));
|
||||
|
||||
idct4x4_1_add_kernel(&dest, stride, dc, &d);
|
||||
idct4x4_1_add_kernel(&dest, stride, dc, &d);
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
;
|
||||
; Copyright (c) 2013 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
|
||||
EXPORT |vpx_idct4x4_16_add_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
INCLUDE vpx_dsp/arm/idct_neon.asm.S
|
||||
|
||||
AREA Block, CODE, READONLY ; name this block of code
|
||||
;void vpx_idct4x4_16_add_neon(int16_t *input, uint8_t *dest, int stride)
|
||||
;
|
||||
; r0 int16_t input
|
||||
; r1 uint8_t *dest
|
||||
; r2 int stride)
|
||||
|
||||
|vpx_idct4x4_16_add_neon| PROC
|
||||
|
||||
; The 2D transform is done with two passes which are actually pretty
|
||||
; similar. We first transform the rows. This is done by transposing
|
||||
; the inputs, doing an SIMD column transform (the columns are the
|
||||
; transposed rows) and then transpose the results (so that it goes back
|
||||
; in normal/row positions). Then, we transform the columns by doing
|
||||
; another SIMD column transform.
|
||||
; So, two passes of a transpose followed by a column transform.
|
||||
|
||||
; load the inputs into q8-q9, d16-d19
|
||||
LOAD_TRAN_LOW_TO_S16 d16, d17, d18, d19, r0
|
||||
|
||||
; generate scalar constants
|
||||
; cospi_8_64 = 15137
|
||||
movw r0, #0x3b21
|
||||
; cospi_16_64 = 11585
|
||||
movw r3, #0x2d41
|
||||
; cospi_24_64 = 6270
|
||||
movw r12, #0x187e
|
||||
|
||||
; transpose the input data
|
||||
; 00 01 02 03 d16
|
||||
; 10 11 12 13 d17
|
||||
; 20 21 22 23 d18
|
||||
; 30 31 32 33 d19
|
||||
vtrn.16 d16, d17
|
||||
vtrn.16 d18, d19
|
||||
|
||||
; generate constant vectors
|
||||
vdup.16 d20, r0 ; replicate cospi_8_64
|
||||
vdup.16 d21, r3 ; replicate cospi_16_64
|
||||
|
||||
; 00 10 02 12 d16
|
||||
; 01 11 03 13 d17
|
||||
; 20 30 22 32 d18
|
||||
; 21 31 23 33 d19
|
||||
vtrn.32 q8, q9
|
||||
; 00 10 20 30 d16
|
||||
; 01 11 21 31 d17
|
||||
; 02 12 22 32 d18
|
||||
; 03 13 23 33 d19
|
||||
|
||||
vdup.16 d22, r12 ; replicate cospi_24_64
|
||||
|
||||
; do the transform on transposed rows
|
||||
|
||||
; stage 1
|
||||
vmull.s16 q15, d17, d22 ; input[1] * cospi_24_64
|
||||
vmull.s16 q1, d17, d20 ; input[1] * cospi_8_64
|
||||
|
||||
; (input[0] + input[2]) * cospi_16_64;
|
||||
; (input[0] - input[2]) * cospi_16_64;
|
||||
vmull.s16 q8, d16, d21
|
||||
vmull.s16 q14, d18, d21
|
||||
vadd.s32 q13, q8, q14
|
||||
vsub.s32 q14, q8, q14
|
||||
|
||||
; input[1] * cospi_24_64 - input[3] * cospi_8_64;
|
||||
; input[1] * cospi_8_64 + input[3] * cospi_24_64;
|
||||
vmlsl.s16 q15, d19, d20
|
||||
vmlal.s16 q1, d19, d22
|
||||
|
||||
; dct_const_round_shift
|
||||
vrshrn.s32 d26, q13, #14
|
||||
vrshrn.s32 d27, q14, #14
|
||||
vrshrn.s32 d29, q15, #14
|
||||
vrshrn.s32 d28, q1, #14
|
||||
|
||||
; stage 2
|
||||
; output[0] = step[0] + step[3];
|
||||
; output[1] = step[1] + step[2];
|
||||
; output[3] = step[0] - step[3];
|
||||
; output[2] = step[1] - step[2];
|
||||
vadd.s16 q8, q13, q14
|
||||
vsub.s16 q9, q13, q14
|
||||
vswp d18, d19
|
||||
|
||||
; transpose the results
|
||||
; 00 01 02 03 d16
|
||||
; 10 11 12 13 d17
|
||||
; 20 21 22 23 d18
|
||||
; 30 31 32 33 d19
|
||||
vtrn.16 d16, d17
|
||||
vtrn.16 d18, d19
|
||||
; 00 10 02 12 d16
|
||||
; 01 11 03 13 d17
|
||||
; 20 30 22 32 d18
|
||||
; 21 31 23 33 d19
|
||||
vtrn.32 q8, q9
|
||||
; 00 10 20 30 d16
|
||||
; 01 11 21 31 d17
|
||||
; 02 12 22 32 d18
|
||||
; 03 13 23 33 d19
|
||||
|
||||
; do the transform on columns
|
||||
|
||||
; stage 1
|
||||
vadd.s16 d23, d16, d18 ; (input[0] + input[2])
|
||||
vsub.s16 d24, d16, d18 ; (input[0] - input[2])
|
||||
|
||||
vmull.s16 q15, d17, d22 ; input[1] * cospi_24_64
|
||||
vmull.s16 q1, d17, d20 ; input[1] * cospi_8_64
|
||||
|
||||
; (input[0] + input[2]) * cospi_16_64;
|
||||
; (input[0] - input[2]) * cospi_16_64;
|
||||
vmull.s16 q13, d23, d21
|
||||
vmull.s16 q14, d24, d21
|
||||
|
||||
; input[1] * cospi_24_64 - input[3] * cospi_8_64;
|
||||
; input[1] * cospi_8_64 + input[3] * cospi_24_64;
|
||||
vmlsl.s16 q15, d19, d20
|
||||
vmlal.s16 q1, d19, d22
|
||||
|
||||
; dct_const_round_shift
|
||||
vrshrn.s32 d26, q13, #14
|
||||
vrshrn.s32 d27, q14, #14
|
||||
vrshrn.s32 d29, q15, #14
|
||||
vrshrn.s32 d28, q1, #14
|
||||
|
||||
; stage 2
|
||||
; output[0] = step[0] + step[3];
|
||||
; output[1] = step[1] + step[2];
|
||||
; output[3] = step[0] - step[3];
|
||||
; output[2] = step[1] - step[2];
|
||||
vadd.s16 q8, q13, q14
|
||||
vsub.s16 q9, q13, q14
|
||||
|
||||
; The results are in two registers, one of them being swapped. This will
|
||||
; be taken care of by loading the 'dest' value in a swapped fashion and
|
||||
; also storing them in the same swapped fashion.
|
||||
; temp_out[0, 1] = d16, d17 = q8
|
||||
; temp_out[2, 3] = d19, d18 = q9 swapped
|
||||
|
||||
; ROUND_POWER_OF_TWO(temp_out[j], 4)
|
||||
vrshr.s16 q8, q8, #4
|
||||
vrshr.s16 q9, q9, #4
|
||||
|
||||
vld1.32 {d26[0]}, [r1], r2
|
||||
vld1.32 {d26[1]}, [r1], r2
|
||||
vld1.32 {d27[1]}, [r1], r2
|
||||
vld1.32 {d27[0]}, [r1] ; no post-increment
|
||||
|
||||
; ROUND_POWER_OF_TWO(temp_out[j], 4) + dest[j * stride + i]
|
||||
vaddw.u8 q8, q8, d26
|
||||
vaddw.u8 q9, q9, d27
|
||||
|
||||
; clip_pixel
|
||||
vqmovun.s16 d26, q8
|
||||
vqmovun.s16 d27, q9
|
||||
|
||||
; do the stores in reverse order with negative post-increment, by changing
|
||||
; the sign of the stride
|
||||
rsb r2, r2, #0
|
||||
vst1.32 {d27[0]}, [r1], r2
|
||||
vst1.32 {d27[1]}, [r1], r2
|
||||
vst1.32 {d26[1]}, [r1], r2
|
||||
vst1.32 {d26[0]}, [r1] ; no post-increment
|
||||
bx lr
|
||||
ENDP ; |vpx_idct4x4_16_add_neon|
|
||||
|
||||
END
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
|
||||
void vpx_idct4x4_16_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
const uint8_t *dst = dest;
|
||||
uint32x2_t s32 = vdup_n_u32(0);
|
||||
int16x8_t a[2];
|
||||
uint8x8_t s, d[2];
|
||||
uint16x8_t sum[2];
|
||||
|
||||
assert(!((intptr_t)dest % sizeof(uint32_t)));
|
||||
assert(!(stride % sizeof(uint32_t)));
|
||||
|
||||
// Rows
|
||||
a[0] = load_tran_low_to_s16q(input);
|
||||
a[1] = load_tran_low_to_s16q(input + 8);
|
||||
transpose_idct4x4_16_bd8(a);
|
||||
|
||||
// Columns
|
||||
a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1]));
|
||||
transpose_idct4x4_16_bd8(a);
|
||||
a[0] = vrshrq_n_s16(a[0], 4);
|
||||
a[1] = vrshrq_n_s16(a[1], 4);
|
||||
|
||||
s = load_u8(dst, stride);
|
||||
dst += 2 * stride;
|
||||
// The elements are loaded in reverse order.
|
||||
s32 = vld1_lane_u32((const uint32_t *)dst, s32, 1);
|
||||
dst += stride;
|
||||
s32 = vld1_lane_u32((const uint32_t *)dst, s32, 0);
|
||||
|
||||
sum[0] = vaddw_u8(vreinterpretq_u16_s16(a[0]), s);
|
||||
sum[1] = vaddw_u8(vreinterpretq_u16_s16(a[1]), vreinterpret_u8_u32(s32));
|
||||
d[0] = vqmovun_s16(vreinterpretq_s16_u16(sum[0]));
|
||||
d[1] = vqmovun_s16(vreinterpretq_s16_u16(sum[1]));
|
||||
|
||||
store_u8(dest, stride, d[0]);
|
||||
dest += 2 * stride;
|
||||
// The elements are stored in reverse order.
|
||||
vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d[1]), 1);
|
||||
dest += stride;
|
||||
vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d[1]), 0);
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/inv_txfm.h"
|
||||
|
||||
static INLINE uint8x8_t create_dcd(const int16_t dc) {
|
||||
int16x8_t t = vdupq_n_s16(dc);
|
||||
return vqmovun_s16(t);
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_1_add_pos_kernel(uint8_t **dest, const int stride,
|
||||
const uint8x8_t res) {
|
||||
const uint8x8_t a = vld1_u8(*dest);
|
||||
const uint8x8_t b = vqadd_u8(a, res);
|
||||
vst1_u8(*dest, b);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_1_add_neg_kernel(uint8_t **dest, const int stride,
|
||||
const uint8x8_t res) {
|
||||
const uint8x8_t a = vld1_u8(*dest);
|
||||
const uint8x8_t b = vqsub_u8(a, res);
|
||||
vst1_u8(*dest, b);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
void vpx_idct8x8_1_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
const int16_t out0 =
|
||||
WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64));
|
||||
const int16_t out1 = WRAPLOW(dct_const_round_shift(out0 * cospi_16_64));
|
||||
const int16_t a1 = ROUND_POWER_OF_TWO(out1, 5);
|
||||
|
||||
if (a1 >= 0) {
|
||||
const uint8x8_t dc = create_dcd(a1);
|
||||
idct8x8_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_pos_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_pos_kernel(&dest, stride, dc);
|
||||
} else {
|
||||
const uint8x8_t dc = create_dcd(-a1);
|
||||
idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
idct8x8_1_add_neg_kernel(&dest, stride, dc);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
|
||||
void vpx_idct8x8_64_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
const int16x8_t cospis = vld1q_s16(kCospi);
|
||||
const int16x4_t cospis0 = vget_low_s16(cospis); // cospi 0, 8, 16, 24
|
||||
const int16x4_t cospis1 = vget_high_s16(cospis); // cospi 4, 12, 20, 28
|
||||
int16x8_t a[8];
|
||||
|
||||
a[0] = load_tran_low_to_s16q(input);
|
||||
a[1] = load_tran_low_to_s16q(input + 8);
|
||||
a[2] = load_tran_low_to_s16q(input + 16);
|
||||
a[3] = load_tran_low_to_s16q(input + 24);
|
||||
a[4] = load_tran_low_to_s16q(input + 32);
|
||||
a[5] = load_tran_low_to_s16q(input + 40);
|
||||
a[6] = load_tran_low_to_s16q(input + 48);
|
||||
a[7] = load_tran_low_to_s16q(input + 56);
|
||||
|
||||
idct8x8_64_1d_bd8(cospis0, cospis1, a);
|
||||
idct8x8_64_1d_bd8(cospis0, cospis1, a);
|
||||
idct8x8_add8x8_neon(a, dest, stride);
|
||||
}
|
||||
|
||||
void vpx_idct8x8_12_add_neon(const tran_low_t *input, uint8_t *dest,
|
||||
int stride) {
|
||||
const int16x8_t cospis = vld1q_s16(kCospi);
|
||||
const int16x8_t cospisd = vaddq_s16(cospis, cospis);
|
||||
const int16x4_t cospis0 = vget_low_s16(cospis); // cospi 0, 8, 16, 24
|
||||
const int16x4_t cospisd0 = vget_low_s16(cospisd); // doubled 0, 8, 16, 24
|
||||
const int16x4_t cospisd1 = vget_high_s16(cospisd); // doubled 4, 12, 20, 28
|
||||
int16x4_t a[8];
|
||||
int16x8_t b[8];
|
||||
|
||||
a[0] = load_tran_low_to_s16d(input);
|
||||
a[1] = load_tran_low_to_s16d(input + 8);
|
||||
a[2] = load_tran_low_to_s16d(input + 16);
|
||||
a[3] = load_tran_low_to_s16d(input + 24);
|
||||
|
||||
idct8x8_12_pass1_bd8(cospis0, cospisd0, cospisd1, a);
|
||||
idct8x8_12_pass2_bd8(cospis0, cospisd0, cospisd1, a, b);
|
||||
idct8x8_add8x8_neon(b, dest, stride);
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
;
|
||||
; Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
|
||||
INCLUDE ./vpx_config.asm
|
||||
|
||||
; Helper functions used to load tran_low_t into int16, narrowing if
|
||||
; necessary.
|
||||
|
||||
; $dst0..3 are d registers with the pairs assumed to be contiguous in
|
||||
; non-high-bitdepth builds. q0-q3 are used as temporaries in high-bitdepth.
|
||||
MACRO
|
||||
LOAD_TRAN_LOW_TO_S16 $dst0, $dst1, $dst2, $dst3, $src
|
||||
IF CONFIG_VP9_HIGHBITDEPTH
|
||||
vld1.s32 {q0,q1}, [$src]!
|
||||
vld1.s32 {q2,q3}, [$src]!
|
||||
vmovn.i32 $dst0, q0
|
||||
vmovn.i32 $dst1, q1
|
||||
vmovn.i32 $dst2, q2
|
||||
vmovn.i32 $dst3, q3
|
||||
ELSE
|
||||
vld1.s16 {$dst0-$dst1,$dst2-$dst3}, [$src]!
|
||||
ENDIF
|
||||
MEND
|
||||
|
||||
; $dst0..3 are d registers. q0-q3 are used as temporaries in high-bitdepth.
|
||||
MACRO
|
||||
LOAD_TRAN_LOW_TO_S16X2 $dst0, $dst1, $dst2, $dst3, $src
|
||||
IF CONFIG_VP9_HIGHBITDEPTH
|
||||
vld2.s32 {q0,q1}, [$src]!
|
||||
vld2.s32 {q2,q3}, [$src]!
|
||||
vmovn.i32 $dst0, q0
|
||||
vmovn.i32 $dst1, q2
|
||||
vmovn.i32 $dst2, q1
|
||||
vmovn.i32 $dst3, q3
|
||||
ELSE
|
||||
vld2.s16 {$dst0,$dst1,$dst2,$dst3}, [$src]!
|
||||
ENDIF
|
||||
MEND
|
||||
END
|
||||
@@ -0,0 +1,919 @@
|
||||
/*
|
||||
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_ARM_IDCT_NEON_H_
|
||||
#define VPX_VPX_DSP_ARM_IDCT_NEON_H_
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
|
||||
static const int16_t kCospi[16] = {
|
||||
16384 /* cospi_0_64 */, 15137 /* cospi_8_64 */,
|
||||
11585 /* cospi_16_64 */, 6270 /* cospi_24_64 */,
|
||||
16069 /* cospi_4_64 */, 13623 /* cospi_12_64 */,
|
||||
-9102 /* -cospi_20_64 */, 3196 /* cospi_28_64 */,
|
||||
16305 /* cospi_2_64 */, 1606 /* cospi_30_64 */,
|
||||
14449 /* cospi_10_64 */, 7723 /* cospi_22_64 */,
|
||||
15679 /* cospi_6_64 */, -4756 /* -cospi_26_64 */,
|
||||
12665 /* cospi_14_64 */, -10394 /* -cospi_18_64 */
|
||||
};
|
||||
|
||||
static const int32_t kCospi32[16] = {
|
||||
16384 /* cospi_0_64 */, 15137 /* cospi_8_64 */,
|
||||
11585 /* cospi_16_64 */, 6270 /* cospi_24_64 */,
|
||||
16069 /* cospi_4_64 */, 13623 /* cospi_12_64 */,
|
||||
-9102 /* -cospi_20_64 */, 3196 /* cospi_28_64 */,
|
||||
16305 /* cospi_2_64 */, 1606 /* cospi_30_64 */,
|
||||
14449 /* cospi_10_64 */, 7723 /* cospi_22_64 */,
|
||||
15679 /* cospi_6_64 */, -4756 /* -cospi_26_64 */,
|
||||
12665 /* cospi_14_64 */, -10394 /* -cospi_18_64 */
|
||||
};
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Use saturating add/sub to avoid overflow in 2nd pass in high bit-depth
|
||||
static INLINE int16x8_t final_add(const int16x8_t a, const int16x8_t b) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
return vqaddq_s16(a, b);
|
||||
#else
|
||||
return vaddq_s16(a, b);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE int16x8_t final_sub(const int16x8_t a, const int16x8_t b) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
return vqsubq_s16(a, b);
|
||||
#else
|
||||
return vsubq_s16(a, b);
|
||||
#endif
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
static INLINE int32x4x2_t highbd_idct_add_dual(const int32x4x2_t s0,
|
||||
const int32x4x2_t s1) {
|
||||
int32x4x2_t t;
|
||||
t.val[0] = vaddq_s32(s0.val[0], s1.val[0]);
|
||||
t.val[1] = vaddq_s32(s0.val[1], s1.val[1]);
|
||||
return t;
|
||||
}
|
||||
|
||||
static INLINE int32x4x2_t highbd_idct_sub_dual(const int32x4x2_t s0,
|
||||
const int32x4x2_t s1) {
|
||||
int32x4x2_t t;
|
||||
t.val[0] = vsubq_s32(s0.val[0], s1.val[0]);
|
||||
t.val[1] = vsubq_s32(s0.val[1], s1.val[1]);
|
||||
return t;
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
static INLINE int16x8_t dct_const_round_shift_low_8(const int32x4_t *const in) {
|
||||
return vcombine_s16(vrshrn_n_s32(in[0], DCT_CONST_BITS),
|
||||
vrshrn_n_s32(in[1], DCT_CONST_BITS));
|
||||
}
|
||||
|
||||
static INLINE void dct_const_round_shift_low_8_dual(const int32x4_t *const t32,
|
||||
int16x8_t *const d0,
|
||||
int16x8_t *const d1) {
|
||||
*d0 = dct_const_round_shift_low_8(t32 + 0);
|
||||
*d1 = dct_const_round_shift_low_8(t32 + 2);
|
||||
}
|
||||
|
||||
static INLINE int32x4x2_t
|
||||
dct_const_round_shift_high_4x2(const int64x2_t *const in) {
|
||||
int32x4x2_t out;
|
||||
out.val[0] = vcombine_s32(vrshrn_n_s64(in[0], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(in[1], DCT_CONST_BITS));
|
||||
out.val[1] = vcombine_s32(vrshrn_n_s64(in[2], DCT_CONST_BITS),
|
||||
vrshrn_n_s64(in[3], DCT_CONST_BITS));
|
||||
return out;
|
||||
}
|
||||
|
||||
// Multiply a by a_const. Saturate, shift and narrow by DCT_CONST_BITS.
|
||||
static INLINE int16x8_t multiply_shift_and_narrow_s16(const int16x8_t a,
|
||||
const int16_t a_const) {
|
||||
// Shift by DCT_CONST_BITS + rounding will be within 16 bits for well formed
|
||||
// streams. See WRAPLOW and dct_const_round_shift for details.
|
||||
// This instruction doubles the result and returns the high half, essentially
|
||||
// resulting in a right shift by 15. By multiplying the constant first that
|
||||
// becomes a right shift by DCT_CONST_BITS.
|
||||
// The largest possible value used here is
|
||||
// vpx_dsp/txfm_common.h:cospi_1_64 = 16364 (* 2 = 32728) a which falls *just*
|
||||
// within the range of int16_t (+32767 / -32768) even when negated.
|
||||
return vqrdmulhq_n_s16(a, a_const * 2);
|
||||
}
|
||||
|
||||
// Add a and b, then multiply by ab_const. Shift and narrow by DCT_CONST_BITS.
|
||||
static INLINE int16x8_t add_multiply_shift_and_narrow_s16(
|
||||
const int16x8_t a, const int16x8_t b, const int16_t ab_const) {
|
||||
// In both add_ and it's pair, sub_, the input for well-formed streams will be
|
||||
// well within 16 bits (input to the idct is the difference between two frames
|
||||
// and will be within -255 to 255, or 9 bits)
|
||||
// However, for inputs over about 25,000 (valid for int16_t, but not for idct
|
||||
// input) this function can not use vaddq_s16.
|
||||
// In order to match existing behavior and intentionally out of range tests,
|
||||
// expand the addition up to 32 bits to prevent truncation.
|
||||
int32x4_t t[2];
|
||||
t[0] = vaddl_s16(vget_low_s16(a), vget_low_s16(b));
|
||||
t[1] = vaddl_s16(vget_high_s16(a), vget_high_s16(b));
|
||||
t[0] = vmulq_n_s32(t[0], ab_const);
|
||||
t[1] = vmulq_n_s32(t[1], ab_const);
|
||||
return dct_const_round_shift_low_8(t);
|
||||
}
|
||||
|
||||
// Subtract b from a, then multiply by ab_const. Shift and narrow by
|
||||
// DCT_CONST_BITS.
|
||||
static INLINE int16x8_t sub_multiply_shift_and_narrow_s16(
|
||||
const int16x8_t a, const int16x8_t b, const int16_t ab_const) {
|
||||
int32x4_t t[2];
|
||||
t[0] = vsubl_s16(vget_low_s16(a), vget_low_s16(b));
|
||||
t[1] = vsubl_s16(vget_high_s16(a), vget_high_s16(b));
|
||||
t[0] = vmulq_n_s32(t[0], ab_const);
|
||||
t[1] = vmulq_n_s32(t[1], ab_const);
|
||||
return dct_const_round_shift_low_8(t);
|
||||
}
|
||||
|
||||
// Multiply a by a_const and b by b_const, then accumulate. Shift and narrow by
|
||||
// DCT_CONST_BITS.
|
||||
static INLINE int16x8_t multiply_accumulate_shift_and_narrow_s16(
|
||||
const int16x8_t a, const int16_t a_const, const int16x8_t b,
|
||||
const int16_t b_const) {
|
||||
int32x4_t t[2];
|
||||
t[0] = vmull_n_s16(vget_low_s16(a), a_const);
|
||||
t[1] = vmull_n_s16(vget_high_s16(a), a_const);
|
||||
t[0] = vmlal_n_s16(t[0], vget_low_s16(b), b_const);
|
||||
t[1] = vmlal_n_s16(t[1], vget_high_s16(b), b_const);
|
||||
return dct_const_round_shift_low_8(t);
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
// Note: The following 4 functions could use 32-bit operations for bit-depth 10.
|
||||
// However, although it's 20% faster with gcc, it's 20% slower with clang.
|
||||
// Use 64-bit operations for now.
|
||||
|
||||
// Multiply a by a_const. Saturate, shift and narrow by DCT_CONST_BITS.
|
||||
static INLINE int32x4x2_t
|
||||
multiply_shift_and_narrow_s32_dual(const int32x4x2_t a, const int32_t a_const) {
|
||||
int64x2_t b[4];
|
||||
|
||||
b[0] = vmull_n_s32(vget_low_s32(a.val[0]), a_const);
|
||||
b[1] = vmull_n_s32(vget_high_s32(a.val[0]), a_const);
|
||||
b[2] = vmull_n_s32(vget_low_s32(a.val[1]), a_const);
|
||||
b[3] = vmull_n_s32(vget_high_s32(a.val[1]), a_const);
|
||||
return dct_const_round_shift_high_4x2(b);
|
||||
}
|
||||
|
||||
// Add a and b, then multiply by ab_const. Shift and narrow by DCT_CONST_BITS.
|
||||
static INLINE int32x4x2_t add_multiply_shift_and_narrow_s32_dual(
|
||||
const int32x4x2_t a, const int32x4x2_t b, const int32_t ab_const) {
|
||||
int32x4_t t[2];
|
||||
int64x2_t c[4];
|
||||
|
||||
t[0] = vaddq_s32(a.val[0], b.val[0]);
|
||||
t[1] = vaddq_s32(a.val[1], b.val[1]);
|
||||
c[0] = vmull_n_s32(vget_low_s32(t[0]), ab_const);
|
||||
c[1] = vmull_n_s32(vget_high_s32(t[0]), ab_const);
|
||||
c[2] = vmull_n_s32(vget_low_s32(t[1]), ab_const);
|
||||
c[3] = vmull_n_s32(vget_high_s32(t[1]), ab_const);
|
||||
return dct_const_round_shift_high_4x2(c);
|
||||
}
|
||||
|
||||
// Subtract b from a, then multiply by ab_const. Shift and narrow by
|
||||
// DCT_CONST_BITS.
|
||||
static INLINE int32x4x2_t sub_multiply_shift_and_narrow_s32_dual(
|
||||
const int32x4x2_t a, const int32x4x2_t b, const int32_t ab_const) {
|
||||
int32x4_t t[2];
|
||||
int64x2_t c[4];
|
||||
|
||||
t[0] = vsubq_s32(a.val[0], b.val[0]);
|
||||
t[1] = vsubq_s32(a.val[1], b.val[1]);
|
||||
c[0] = vmull_n_s32(vget_low_s32(t[0]), ab_const);
|
||||
c[1] = vmull_n_s32(vget_high_s32(t[0]), ab_const);
|
||||
c[2] = vmull_n_s32(vget_low_s32(t[1]), ab_const);
|
||||
c[3] = vmull_n_s32(vget_high_s32(t[1]), ab_const);
|
||||
return dct_const_round_shift_high_4x2(c);
|
||||
}
|
||||
|
||||
// Multiply a by a_const and b by b_const, then accumulate. Shift and narrow by
|
||||
// DCT_CONST_BITS.
|
||||
static INLINE int32x4x2_t multiply_accumulate_shift_and_narrow_s32_dual(
|
||||
const int32x4x2_t a, const int32_t a_const, const int32x4x2_t b,
|
||||
const int32_t b_const) {
|
||||
int64x2_t c[4];
|
||||
c[0] = vmull_n_s32(vget_low_s32(a.val[0]), a_const);
|
||||
c[1] = vmull_n_s32(vget_high_s32(a.val[0]), a_const);
|
||||
c[2] = vmull_n_s32(vget_low_s32(a.val[1]), a_const);
|
||||
c[3] = vmull_n_s32(vget_high_s32(a.val[1]), a_const);
|
||||
c[0] = vmlal_n_s32(c[0], vget_low_s32(b.val[0]), b_const);
|
||||
c[1] = vmlal_n_s32(c[1], vget_high_s32(b.val[0]), b_const);
|
||||
c[2] = vmlal_n_s32(c[2], vget_low_s32(b.val[1]), b_const);
|
||||
c[3] = vmlal_n_s32(c[3], vget_high_s32(b.val[1]), b_const);
|
||||
return dct_const_round_shift_high_4x2(c);
|
||||
}
|
||||
|
||||
// Shift the output down by 6 and add it to the destination buffer.
|
||||
static INLINE void add_and_store_u8_s16(const int16x8_t *const a, uint8_t *d,
|
||||
const int stride) {
|
||||
uint8x8_t b[8];
|
||||
int16x8_t c[8];
|
||||
|
||||
b[0] = vld1_u8(d);
|
||||
d += stride;
|
||||
b[1] = vld1_u8(d);
|
||||
d += stride;
|
||||
b[2] = vld1_u8(d);
|
||||
d += stride;
|
||||
b[3] = vld1_u8(d);
|
||||
d += stride;
|
||||
b[4] = vld1_u8(d);
|
||||
d += stride;
|
||||
b[5] = vld1_u8(d);
|
||||
d += stride;
|
||||
b[6] = vld1_u8(d);
|
||||
d += stride;
|
||||
b[7] = vld1_u8(d);
|
||||
d -= (7 * stride);
|
||||
|
||||
// c = b + (a >> 6)
|
||||
c[0] = vrsraq_n_s16(vreinterpretq_s16_u16(vmovl_u8(b[0])), a[0], 6);
|
||||
c[1] = vrsraq_n_s16(vreinterpretq_s16_u16(vmovl_u8(b[1])), a[1], 6);
|
||||
c[2] = vrsraq_n_s16(vreinterpretq_s16_u16(vmovl_u8(b[2])), a[2], 6);
|
||||
c[3] = vrsraq_n_s16(vreinterpretq_s16_u16(vmovl_u8(b[3])), a[3], 6);
|
||||
c[4] = vrsraq_n_s16(vreinterpretq_s16_u16(vmovl_u8(b[4])), a[4], 6);
|
||||
c[5] = vrsraq_n_s16(vreinterpretq_s16_u16(vmovl_u8(b[5])), a[5], 6);
|
||||
c[6] = vrsraq_n_s16(vreinterpretq_s16_u16(vmovl_u8(b[6])), a[6], 6);
|
||||
c[7] = vrsraq_n_s16(vreinterpretq_s16_u16(vmovl_u8(b[7])), a[7], 6);
|
||||
|
||||
b[0] = vqmovun_s16(c[0]);
|
||||
b[1] = vqmovun_s16(c[1]);
|
||||
b[2] = vqmovun_s16(c[2]);
|
||||
b[3] = vqmovun_s16(c[3]);
|
||||
b[4] = vqmovun_s16(c[4]);
|
||||
b[5] = vqmovun_s16(c[5]);
|
||||
b[6] = vqmovun_s16(c[6]);
|
||||
b[7] = vqmovun_s16(c[7]);
|
||||
|
||||
vst1_u8(d, b[0]);
|
||||
d += stride;
|
||||
vst1_u8(d, b[1]);
|
||||
d += stride;
|
||||
vst1_u8(d, b[2]);
|
||||
d += stride;
|
||||
vst1_u8(d, b[3]);
|
||||
d += stride;
|
||||
vst1_u8(d, b[4]);
|
||||
d += stride;
|
||||
vst1_u8(d, b[5]);
|
||||
d += stride;
|
||||
vst1_u8(d, b[6]);
|
||||
d += stride;
|
||||
vst1_u8(d, b[7]);
|
||||
}
|
||||
|
||||
static INLINE uint8x16_t create_dcq(const int16_t dc) {
|
||||
// Clip both sides and gcc may compile to assembly 'usat'.
|
||||
const int16_t t = (dc < 0) ? 0 : ((dc > 255) ? 255 : dc);
|
||||
return vdupq_n_u8((uint8_t)t);
|
||||
}
|
||||
|
||||
static INLINE void idct4x4_16_kernel_bd8(int16x8_t *const a) {
|
||||
const int16x4_t cospis = vld1_s16(kCospi);
|
||||
int16x4_t b[4];
|
||||
int32x4_t c[4];
|
||||
int16x8_t d[2];
|
||||
|
||||
b[0] = vget_low_s16(a[0]);
|
||||
b[1] = vget_high_s16(a[0]);
|
||||
b[2] = vget_low_s16(a[1]);
|
||||
b[3] = vget_high_s16(a[1]);
|
||||
c[0] = vmull_lane_s16(b[0], cospis, 2);
|
||||
c[2] = vmull_lane_s16(b[1], cospis, 2);
|
||||
c[1] = vsubq_s32(c[0], c[2]);
|
||||
c[0] = vaddq_s32(c[0], c[2]);
|
||||
c[3] = vmull_lane_s16(b[2], cospis, 3);
|
||||
c[2] = vmull_lane_s16(b[2], cospis, 1);
|
||||
c[3] = vmlsl_lane_s16(c[3], b[3], cospis, 1);
|
||||
c[2] = vmlal_lane_s16(c[2], b[3], cospis, 3);
|
||||
dct_const_round_shift_low_8_dual(c, &d[0], &d[1]);
|
||||
a[0] = vaddq_s16(d[0], d[1]);
|
||||
a[1] = vsubq_s16(d[0], d[1]);
|
||||
}
|
||||
|
||||
static INLINE void transpose_idct4x4_16_bd8(int16x8_t *const a) {
|
||||
transpose_s16_4x4q(&a[0], &a[1]);
|
||||
idct4x4_16_kernel_bd8(a);
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_12_pass1_bd8(const int16x4_t cospis0,
|
||||
const int16x4_t cospisd0,
|
||||
const int16x4_t cospisd1,
|
||||
int16x4_t *const io) {
|
||||
int16x4_t step1[8], step2[8];
|
||||
int32x4_t t32[2];
|
||||
|
||||
transpose_s16_4x4d(&io[0], &io[1], &io[2], &io[3]);
|
||||
|
||||
// stage 1
|
||||
step1[4] = vqrdmulh_lane_s16(io[1], cospisd1, 3);
|
||||
step1[5] = vqrdmulh_lane_s16(io[3], cospisd1, 2);
|
||||
step1[6] = vqrdmulh_lane_s16(io[3], cospisd1, 1);
|
||||
step1[7] = vqrdmulh_lane_s16(io[1], cospisd1, 0);
|
||||
|
||||
// stage 2
|
||||
step2[1] = vqrdmulh_lane_s16(io[0], cospisd0, 2);
|
||||
step2[2] = vqrdmulh_lane_s16(io[2], cospisd0, 3);
|
||||
step2[3] = vqrdmulh_lane_s16(io[2], cospisd0, 1);
|
||||
|
||||
step2[4] = vadd_s16(step1[4], step1[5]);
|
||||
step2[5] = vsub_s16(step1[4], step1[5]);
|
||||
step2[6] = vsub_s16(step1[7], step1[6]);
|
||||
step2[7] = vadd_s16(step1[7], step1[6]);
|
||||
|
||||
// stage 3
|
||||
step1[0] = vadd_s16(step2[1], step2[3]);
|
||||
step1[1] = vadd_s16(step2[1], step2[2]);
|
||||
step1[2] = vsub_s16(step2[1], step2[2]);
|
||||
step1[3] = vsub_s16(step2[1], step2[3]);
|
||||
|
||||
t32[1] = vmull_lane_s16(step2[6], cospis0, 2);
|
||||
t32[0] = vmlsl_lane_s16(t32[1], step2[5], cospis0, 2);
|
||||
t32[1] = vmlal_lane_s16(t32[1], step2[5], cospis0, 2);
|
||||
step1[5] = vrshrn_n_s32(t32[0], DCT_CONST_BITS);
|
||||
step1[6] = vrshrn_n_s32(t32[1], DCT_CONST_BITS);
|
||||
|
||||
// stage 4
|
||||
io[0] = vadd_s16(step1[0], step2[7]);
|
||||
io[1] = vadd_s16(step1[1], step1[6]);
|
||||
io[2] = vadd_s16(step1[2], step1[5]);
|
||||
io[3] = vadd_s16(step1[3], step2[4]);
|
||||
io[4] = vsub_s16(step1[3], step2[4]);
|
||||
io[5] = vsub_s16(step1[2], step1[5]);
|
||||
io[6] = vsub_s16(step1[1], step1[6]);
|
||||
io[7] = vsub_s16(step1[0], step2[7]);
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_12_pass2_bd8(const int16x4_t cospis0,
|
||||
const int16x4_t cospisd0,
|
||||
const int16x4_t cospisd1,
|
||||
const int16x4_t *const input,
|
||||
int16x8_t *const output) {
|
||||
int16x8_t in[4];
|
||||
int16x8_t step1[8], step2[8];
|
||||
int32x4_t t32[8];
|
||||
|
||||
transpose_s16_4x8(input[0], input[1], input[2], input[3], input[4], input[5],
|
||||
input[6], input[7], &in[0], &in[1], &in[2], &in[3]);
|
||||
|
||||
// stage 1
|
||||
step1[4] = vqrdmulhq_lane_s16(in[1], cospisd1, 3);
|
||||
step1[5] = vqrdmulhq_lane_s16(in[3], cospisd1, 2);
|
||||
step1[6] = vqrdmulhq_lane_s16(in[3], cospisd1, 1);
|
||||
step1[7] = vqrdmulhq_lane_s16(in[1], cospisd1, 0);
|
||||
|
||||
// stage 2
|
||||
step2[1] = vqrdmulhq_lane_s16(in[0], cospisd0, 2);
|
||||
step2[2] = vqrdmulhq_lane_s16(in[2], cospisd0, 3);
|
||||
step2[3] = vqrdmulhq_lane_s16(in[2], cospisd0, 1);
|
||||
|
||||
step2[4] = vaddq_s16(step1[4], step1[5]);
|
||||
step2[5] = vsubq_s16(step1[4], step1[5]);
|
||||
step2[6] = vsubq_s16(step1[7], step1[6]);
|
||||
step2[7] = vaddq_s16(step1[7], step1[6]);
|
||||
|
||||
// stage 3
|
||||
step1[0] = vaddq_s16(step2[1], step2[3]);
|
||||
step1[1] = vaddq_s16(step2[1], step2[2]);
|
||||
step1[2] = vsubq_s16(step2[1], step2[2]);
|
||||
step1[3] = vsubq_s16(step2[1], step2[3]);
|
||||
|
||||
t32[2] = vmull_lane_s16(vget_low_s16(step2[6]), cospis0, 2);
|
||||
t32[3] = vmull_lane_s16(vget_high_s16(step2[6]), cospis0, 2);
|
||||
t32[0] = vmlsl_lane_s16(t32[2], vget_low_s16(step2[5]), cospis0, 2);
|
||||
t32[1] = vmlsl_lane_s16(t32[3], vget_high_s16(step2[5]), cospis0, 2);
|
||||
t32[2] = vmlal_lane_s16(t32[2], vget_low_s16(step2[5]), cospis0, 2);
|
||||
t32[3] = vmlal_lane_s16(t32[3], vget_high_s16(step2[5]), cospis0, 2);
|
||||
dct_const_round_shift_low_8_dual(t32, &step1[5], &step1[6]);
|
||||
|
||||
// stage 4
|
||||
output[0] = vaddq_s16(step1[0], step2[7]);
|
||||
output[1] = vaddq_s16(step1[1], step1[6]);
|
||||
output[2] = vaddq_s16(step1[2], step1[5]);
|
||||
output[3] = vaddq_s16(step1[3], step2[4]);
|
||||
output[4] = vsubq_s16(step1[3], step2[4]);
|
||||
output[5] = vsubq_s16(step1[2], step1[5]);
|
||||
output[6] = vsubq_s16(step1[1], step1[6]);
|
||||
output[7] = vsubq_s16(step1[0], step2[7]);
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_64_1d_bd8_kernel(const int16x4_t cospis0,
|
||||
const int16x4_t cospis1,
|
||||
int16x8_t *const io) {
|
||||
int16x4_t input1l, input1h, input3l, input3h, input5l, input5h, input7l,
|
||||
input7h;
|
||||
int16x4_t step1l[4], step1h[4];
|
||||
int16x8_t step1[8], step2[8];
|
||||
int32x4_t t32[8];
|
||||
|
||||
// stage 1
|
||||
input1l = vget_low_s16(io[1]);
|
||||
input1h = vget_high_s16(io[1]);
|
||||
input3l = vget_low_s16(io[3]);
|
||||
input3h = vget_high_s16(io[3]);
|
||||
input5l = vget_low_s16(io[5]);
|
||||
input5h = vget_high_s16(io[5]);
|
||||
input7l = vget_low_s16(io[7]);
|
||||
input7h = vget_high_s16(io[7]);
|
||||
step1l[0] = vget_low_s16(io[0]);
|
||||
step1h[0] = vget_high_s16(io[0]);
|
||||
step1l[1] = vget_low_s16(io[2]);
|
||||
step1h[1] = vget_high_s16(io[2]);
|
||||
step1l[2] = vget_low_s16(io[4]);
|
||||
step1h[2] = vget_high_s16(io[4]);
|
||||
step1l[3] = vget_low_s16(io[6]);
|
||||
step1h[3] = vget_high_s16(io[6]);
|
||||
|
||||
t32[0] = vmull_lane_s16(input1l, cospis1, 3);
|
||||
t32[1] = vmull_lane_s16(input1h, cospis1, 3);
|
||||
t32[2] = vmull_lane_s16(input3l, cospis1, 2);
|
||||
t32[3] = vmull_lane_s16(input3h, cospis1, 2);
|
||||
t32[4] = vmull_lane_s16(input3l, cospis1, 1);
|
||||
t32[5] = vmull_lane_s16(input3h, cospis1, 1);
|
||||
t32[6] = vmull_lane_s16(input1l, cospis1, 0);
|
||||
t32[7] = vmull_lane_s16(input1h, cospis1, 0);
|
||||
t32[0] = vmlsl_lane_s16(t32[0], input7l, cospis1, 0);
|
||||
t32[1] = vmlsl_lane_s16(t32[1], input7h, cospis1, 0);
|
||||
t32[2] = vmlal_lane_s16(t32[2], input5l, cospis1, 1);
|
||||
t32[3] = vmlal_lane_s16(t32[3], input5h, cospis1, 1);
|
||||
t32[4] = vmlsl_lane_s16(t32[4], input5l, cospis1, 2);
|
||||
t32[5] = vmlsl_lane_s16(t32[5], input5h, cospis1, 2);
|
||||
t32[6] = vmlal_lane_s16(t32[6], input7l, cospis1, 3);
|
||||
t32[7] = vmlal_lane_s16(t32[7], input7h, cospis1, 3);
|
||||
dct_const_round_shift_low_8_dual(&t32[0], &step1[4], &step1[5]);
|
||||
dct_const_round_shift_low_8_dual(&t32[4], &step1[6], &step1[7]);
|
||||
|
||||
// stage 2
|
||||
t32[2] = vmull_lane_s16(step1l[0], cospis0, 2);
|
||||
t32[3] = vmull_lane_s16(step1h[0], cospis0, 2);
|
||||
t32[4] = vmull_lane_s16(step1l[1], cospis0, 3);
|
||||
t32[5] = vmull_lane_s16(step1h[1], cospis0, 3);
|
||||
t32[6] = vmull_lane_s16(step1l[1], cospis0, 1);
|
||||
t32[7] = vmull_lane_s16(step1h[1], cospis0, 1);
|
||||
t32[0] = vmlal_lane_s16(t32[2], step1l[2], cospis0, 2);
|
||||
t32[1] = vmlal_lane_s16(t32[3], step1h[2], cospis0, 2);
|
||||
t32[2] = vmlsl_lane_s16(t32[2], step1l[2], cospis0, 2);
|
||||
t32[3] = vmlsl_lane_s16(t32[3], step1h[2], cospis0, 2);
|
||||
t32[4] = vmlsl_lane_s16(t32[4], step1l[3], cospis0, 1);
|
||||
t32[5] = vmlsl_lane_s16(t32[5], step1h[3], cospis0, 1);
|
||||
t32[6] = vmlal_lane_s16(t32[6], step1l[3], cospis0, 3);
|
||||
t32[7] = vmlal_lane_s16(t32[7], step1h[3], cospis0, 3);
|
||||
dct_const_round_shift_low_8_dual(&t32[0], &step2[0], &step2[1]);
|
||||
dct_const_round_shift_low_8_dual(&t32[4], &step2[2], &step2[3]);
|
||||
|
||||
step2[4] = vaddq_s16(step1[4], step1[5]);
|
||||
step2[5] = vsubq_s16(step1[4], step1[5]);
|
||||
step2[6] = vsubq_s16(step1[7], step1[6]);
|
||||
step2[7] = vaddq_s16(step1[7], step1[6]);
|
||||
|
||||
// stage 3
|
||||
step1[0] = vaddq_s16(step2[0], step2[3]);
|
||||
step1[1] = vaddq_s16(step2[1], step2[2]);
|
||||
step1[2] = vsubq_s16(step2[1], step2[2]);
|
||||
step1[3] = vsubq_s16(step2[0], step2[3]);
|
||||
|
||||
t32[2] = vmull_lane_s16(vget_low_s16(step2[6]), cospis0, 2);
|
||||
t32[3] = vmull_lane_s16(vget_high_s16(step2[6]), cospis0, 2);
|
||||
t32[0] = vmlsl_lane_s16(t32[2], vget_low_s16(step2[5]), cospis0, 2);
|
||||
t32[1] = vmlsl_lane_s16(t32[3], vget_high_s16(step2[5]), cospis0, 2);
|
||||
t32[2] = vmlal_lane_s16(t32[2], vget_low_s16(step2[5]), cospis0, 2);
|
||||
t32[3] = vmlal_lane_s16(t32[3], vget_high_s16(step2[5]), cospis0, 2);
|
||||
dct_const_round_shift_low_8_dual(t32, &step1[5], &step1[6]);
|
||||
|
||||
// stage 4
|
||||
io[0] = vaddq_s16(step1[0], step2[7]);
|
||||
io[1] = vaddq_s16(step1[1], step1[6]);
|
||||
io[2] = vaddq_s16(step1[2], step1[5]);
|
||||
io[3] = vaddq_s16(step1[3], step2[4]);
|
||||
io[4] = vsubq_s16(step1[3], step2[4]);
|
||||
io[5] = vsubq_s16(step1[2], step1[5]);
|
||||
io[6] = vsubq_s16(step1[1], step1[6]);
|
||||
io[7] = vsubq_s16(step1[0], step2[7]);
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_64_1d_bd8(const int16x4_t cospis0,
|
||||
const int16x4_t cospis1,
|
||||
int16x8_t *const io) {
|
||||
transpose_s16_8x8(&io[0], &io[1], &io[2], &io[3], &io[4], &io[5], &io[6],
|
||||
&io[7]);
|
||||
idct8x8_64_1d_bd8_kernel(cospis0, cospis1, io);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_8_24_q_kernel(const int16x8_t s0,
|
||||
const int16x8_t s1,
|
||||
const int16x4_t cospi_0_8_16_24,
|
||||
int32x4_t *const t32) {
|
||||
t32[0] = vmull_lane_s16(vget_low_s16(s0), cospi_0_8_16_24, 3);
|
||||
t32[1] = vmull_lane_s16(vget_high_s16(s0), cospi_0_8_16_24, 3);
|
||||
t32[2] = vmull_lane_s16(vget_low_s16(s1), cospi_0_8_16_24, 3);
|
||||
t32[3] = vmull_lane_s16(vget_high_s16(s1), cospi_0_8_16_24, 3);
|
||||
t32[0] = vmlsl_lane_s16(t32[0], vget_low_s16(s1), cospi_0_8_16_24, 1);
|
||||
t32[1] = vmlsl_lane_s16(t32[1], vget_high_s16(s1), cospi_0_8_16_24, 1);
|
||||
t32[2] = vmlal_lane_s16(t32[2], vget_low_s16(s0), cospi_0_8_16_24, 1);
|
||||
t32[3] = vmlal_lane_s16(t32[3], vget_high_s16(s0), cospi_0_8_16_24, 1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_8_24_q(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x4_t cospi_0_8_16_24,
|
||||
int16x8_t *const d0, int16x8_t *const d1) {
|
||||
int32x4_t t32[4];
|
||||
|
||||
idct_cospi_8_24_q_kernel(s0, s1, cospi_0_8_16_24, t32);
|
||||
dct_const_round_shift_low_8_dual(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_8_24_neg_q(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x4_t cospi_0_8_16_24,
|
||||
int16x8_t *const d0,
|
||||
int16x8_t *const d1) {
|
||||
int32x4_t t32[4];
|
||||
|
||||
idct_cospi_8_24_q_kernel(s0, s1, cospi_0_8_16_24, t32);
|
||||
t32[2] = vnegq_s32(t32[2]);
|
||||
t32[3] = vnegq_s32(t32[3]);
|
||||
dct_const_round_shift_low_8_dual(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_16_16_q(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x4_t cospi_0_8_16_24,
|
||||
int16x8_t *const d0,
|
||||
int16x8_t *const d1) {
|
||||
int32x4_t t32[6];
|
||||
|
||||
t32[4] = vmull_lane_s16(vget_low_s16(s1), cospi_0_8_16_24, 2);
|
||||
t32[5] = vmull_lane_s16(vget_high_s16(s1), cospi_0_8_16_24, 2);
|
||||
t32[0] = vmlsl_lane_s16(t32[4], vget_low_s16(s0), cospi_0_8_16_24, 2);
|
||||
t32[1] = vmlsl_lane_s16(t32[5], vget_high_s16(s0), cospi_0_8_16_24, 2);
|
||||
t32[2] = vmlal_lane_s16(t32[4], vget_low_s16(s0), cospi_0_8_16_24, 2);
|
||||
t32[3] = vmlal_lane_s16(t32[5], vget_high_s16(s0), cospi_0_8_16_24, 2);
|
||||
dct_const_round_shift_low_8_dual(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_2_30(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x4_t cospi_2_30_10_22,
|
||||
int16x8_t *const d0, int16x8_t *const d1) {
|
||||
int32x4_t t32[4];
|
||||
|
||||
t32[0] = vmull_lane_s16(vget_low_s16(s0), cospi_2_30_10_22, 1);
|
||||
t32[1] = vmull_lane_s16(vget_high_s16(s0), cospi_2_30_10_22, 1);
|
||||
t32[2] = vmull_lane_s16(vget_low_s16(s1), cospi_2_30_10_22, 1);
|
||||
t32[3] = vmull_lane_s16(vget_high_s16(s1), cospi_2_30_10_22, 1);
|
||||
t32[0] = vmlsl_lane_s16(t32[0], vget_low_s16(s1), cospi_2_30_10_22, 0);
|
||||
t32[1] = vmlsl_lane_s16(t32[1], vget_high_s16(s1), cospi_2_30_10_22, 0);
|
||||
t32[2] = vmlal_lane_s16(t32[2], vget_low_s16(s0), cospi_2_30_10_22, 0);
|
||||
t32[3] = vmlal_lane_s16(t32[3], vget_high_s16(s0), cospi_2_30_10_22, 0);
|
||||
dct_const_round_shift_low_8_dual(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_4_28(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x4_t cospi_4_12_20N_28,
|
||||
int16x8_t *const d0, int16x8_t *const d1) {
|
||||
int32x4_t t32[4];
|
||||
|
||||
t32[0] = vmull_lane_s16(vget_low_s16(s0), cospi_4_12_20N_28, 3);
|
||||
t32[1] = vmull_lane_s16(vget_high_s16(s0), cospi_4_12_20N_28, 3);
|
||||
t32[2] = vmull_lane_s16(vget_low_s16(s1), cospi_4_12_20N_28, 3);
|
||||
t32[3] = vmull_lane_s16(vget_high_s16(s1), cospi_4_12_20N_28, 3);
|
||||
t32[0] = vmlsl_lane_s16(t32[0], vget_low_s16(s1), cospi_4_12_20N_28, 0);
|
||||
t32[1] = vmlsl_lane_s16(t32[1], vget_high_s16(s1), cospi_4_12_20N_28, 0);
|
||||
t32[2] = vmlal_lane_s16(t32[2], vget_low_s16(s0), cospi_4_12_20N_28, 0);
|
||||
t32[3] = vmlal_lane_s16(t32[3], vget_high_s16(s0), cospi_4_12_20N_28, 0);
|
||||
dct_const_round_shift_low_8_dual(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_6_26(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x4_t cospi_6_26N_14_18N,
|
||||
int16x8_t *const d0, int16x8_t *const d1) {
|
||||
int32x4_t t32[4];
|
||||
|
||||
t32[0] = vmull_lane_s16(vget_low_s16(s0), cospi_6_26N_14_18N, 0);
|
||||
t32[1] = vmull_lane_s16(vget_high_s16(s0), cospi_6_26N_14_18N, 0);
|
||||
t32[2] = vmull_lane_s16(vget_low_s16(s1), cospi_6_26N_14_18N, 0);
|
||||
t32[3] = vmull_lane_s16(vget_high_s16(s1), cospi_6_26N_14_18N, 0);
|
||||
t32[0] = vmlal_lane_s16(t32[0], vget_low_s16(s1), cospi_6_26N_14_18N, 1);
|
||||
t32[1] = vmlal_lane_s16(t32[1], vget_high_s16(s1), cospi_6_26N_14_18N, 1);
|
||||
t32[2] = vmlsl_lane_s16(t32[2], vget_low_s16(s0), cospi_6_26N_14_18N, 1);
|
||||
t32[3] = vmlsl_lane_s16(t32[3], vget_high_s16(s0), cospi_6_26N_14_18N, 1);
|
||||
dct_const_round_shift_low_8_dual(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_10_22(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x4_t cospi_2_30_10_22,
|
||||
int16x8_t *const d0, int16x8_t *const d1) {
|
||||
int32x4_t t32[4];
|
||||
|
||||
t32[0] = vmull_lane_s16(vget_low_s16(s0), cospi_2_30_10_22, 3);
|
||||
t32[1] = vmull_lane_s16(vget_high_s16(s0), cospi_2_30_10_22, 3);
|
||||
t32[2] = vmull_lane_s16(vget_low_s16(s1), cospi_2_30_10_22, 3);
|
||||
t32[3] = vmull_lane_s16(vget_high_s16(s1), cospi_2_30_10_22, 3);
|
||||
t32[0] = vmlsl_lane_s16(t32[0], vget_low_s16(s1), cospi_2_30_10_22, 2);
|
||||
t32[1] = vmlsl_lane_s16(t32[1], vget_high_s16(s1), cospi_2_30_10_22, 2);
|
||||
t32[2] = vmlal_lane_s16(t32[2], vget_low_s16(s0), cospi_2_30_10_22, 2);
|
||||
t32[3] = vmlal_lane_s16(t32[3], vget_high_s16(s0), cospi_2_30_10_22, 2);
|
||||
dct_const_round_shift_low_8_dual(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_12_20(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x4_t cospi_4_12_20N_28,
|
||||
int16x8_t *const d0, int16x8_t *const d1) {
|
||||
int32x4_t t32[4];
|
||||
|
||||
t32[0] = vmull_lane_s16(vget_low_s16(s0), cospi_4_12_20N_28, 1);
|
||||
t32[1] = vmull_lane_s16(vget_high_s16(s0), cospi_4_12_20N_28, 1);
|
||||
t32[2] = vmull_lane_s16(vget_low_s16(s1), cospi_4_12_20N_28, 1);
|
||||
t32[3] = vmull_lane_s16(vget_high_s16(s1), cospi_4_12_20N_28, 1);
|
||||
t32[0] = vmlal_lane_s16(t32[0], vget_low_s16(s1), cospi_4_12_20N_28, 2);
|
||||
t32[1] = vmlal_lane_s16(t32[1], vget_high_s16(s1), cospi_4_12_20N_28, 2);
|
||||
t32[2] = vmlsl_lane_s16(t32[2], vget_low_s16(s0), cospi_4_12_20N_28, 2);
|
||||
t32[3] = vmlsl_lane_s16(t32[3], vget_high_s16(s0), cospi_4_12_20N_28, 2);
|
||||
dct_const_round_shift_low_8_dual(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct_cospi_14_18(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x4_t cospi_6_26N_14_18N,
|
||||
int16x8_t *const d0, int16x8_t *const d1) {
|
||||
int32x4_t t32[4];
|
||||
|
||||
t32[0] = vmull_lane_s16(vget_low_s16(s0), cospi_6_26N_14_18N, 2);
|
||||
t32[1] = vmull_lane_s16(vget_high_s16(s0), cospi_6_26N_14_18N, 2);
|
||||
t32[2] = vmull_lane_s16(vget_low_s16(s1), cospi_6_26N_14_18N, 2);
|
||||
t32[3] = vmull_lane_s16(vget_high_s16(s1), cospi_6_26N_14_18N, 2);
|
||||
t32[0] = vmlal_lane_s16(t32[0], vget_low_s16(s1), cospi_6_26N_14_18N, 3);
|
||||
t32[1] = vmlal_lane_s16(t32[1], vget_high_s16(s1), cospi_6_26N_14_18N, 3);
|
||||
t32[2] = vmlsl_lane_s16(t32[2], vget_low_s16(s0), cospi_6_26N_14_18N, 3);
|
||||
t32[3] = vmlsl_lane_s16(t32[3], vget_high_s16(s0), cospi_6_26N_14_18N, 3);
|
||||
dct_const_round_shift_low_8_dual(t32, d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void idct16x16_add_stage7(const int16x8_t *const step2,
|
||||
int16x8_t *const out) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
// Use saturating add/sub to avoid overflow in 2nd pass
|
||||
out[0] = vqaddq_s16(step2[0], step2[15]);
|
||||
out[1] = vqaddq_s16(step2[1], step2[14]);
|
||||
out[2] = vqaddq_s16(step2[2], step2[13]);
|
||||
out[3] = vqaddq_s16(step2[3], step2[12]);
|
||||
out[4] = vqaddq_s16(step2[4], step2[11]);
|
||||
out[5] = vqaddq_s16(step2[5], step2[10]);
|
||||
out[6] = vqaddq_s16(step2[6], step2[9]);
|
||||
out[7] = vqaddq_s16(step2[7], step2[8]);
|
||||
out[8] = vqsubq_s16(step2[7], step2[8]);
|
||||
out[9] = vqsubq_s16(step2[6], step2[9]);
|
||||
out[10] = vqsubq_s16(step2[5], step2[10]);
|
||||
out[11] = vqsubq_s16(step2[4], step2[11]);
|
||||
out[12] = vqsubq_s16(step2[3], step2[12]);
|
||||
out[13] = vqsubq_s16(step2[2], step2[13]);
|
||||
out[14] = vqsubq_s16(step2[1], step2[14]);
|
||||
out[15] = vqsubq_s16(step2[0], step2[15]);
|
||||
#else
|
||||
out[0] = vaddq_s16(step2[0], step2[15]);
|
||||
out[1] = vaddq_s16(step2[1], step2[14]);
|
||||
out[2] = vaddq_s16(step2[2], step2[13]);
|
||||
out[3] = vaddq_s16(step2[3], step2[12]);
|
||||
out[4] = vaddq_s16(step2[4], step2[11]);
|
||||
out[5] = vaddq_s16(step2[5], step2[10]);
|
||||
out[6] = vaddq_s16(step2[6], step2[9]);
|
||||
out[7] = vaddq_s16(step2[7], step2[8]);
|
||||
out[8] = vsubq_s16(step2[7], step2[8]);
|
||||
out[9] = vsubq_s16(step2[6], step2[9]);
|
||||
out[10] = vsubq_s16(step2[5], step2[10]);
|
||||
out[11] = vsubq_s16(step2[4], step2[11]);
|
||||
out[12] = vsubq_s16(step2[3], step2[12]);
|
||||
out[13] = vsubq_s16(step2[2], step2[13]);
|
||||
out[14] = vsubq_s16(step2[1], step2[14]);
|
||||
out[15] = vsubq_s16(step2[0], step2[15]);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE void idct16x16_store_pass1(const int16x8_t *const out,
|
||||
int16_t *output) {
|
||||
// Save the result into output
|
||||
vst1q_s16(output, out[0]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[1]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[2]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[3]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[4]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[5]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[6]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[7]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[8]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[9]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[10]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[11]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[12]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[13]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[14]);
|
||||
output += 16;
|
||||
vst1q_s16(output, out[15]);
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_add8x1(const int16x8_t a, uint8_t **const dest,
|
||||
const int stride) {
|
||||
const uint8x8_t s = vld1_u8(*dest);
|
||||
const int16x8_t res = vrshrq_n_s16(a, 5);
|
||||
const uint16x8_t q = vaddw_u8(vreinterpretq_u16_s16(res), s);
|
||||
const uint8x8_t d = vqmovun_s16(vreinterpretq_s16_u16(q));
|
||||
vst1_u8(*dest, d);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void idct8x8_add8x8_neon(int16x8_t *const out, uint8_t *dest,
|
||||
const int stride) {
|
||||
idct8x8_add8x1(out[0], &dest, stride);
|
||||
idct8x8_add8x1(out[1], &dest, stride);
|
||||
idct8x8_add8x1(out[2], &dest, stride);
|
||||
idct8x8_add8x1(out[3], &dest, stride);
|
||||
idct8x8_add8x1(out[4], &dest, stride);
|
||||
idct8x8_add8x1(out[5], &dest, stride);
|
||||
idct8x8_add8x1(out[6], &dest, stride);
|
||||
idct8x8_add8x1(out[7], &dest, stride);
|
||||
}
|
||||
|
||||
static INLINE void idct16x16_add8x1(const int16x8_t a, uint8_t **const dest,
|
||||
const int stride) {
|
||||
const uint8x8_t s = vld1_u8(*dest);
|
||||
const int16x8_t res = vrshrq_n_s16(a, 6);
|
||||
const uint16x8_t q = vaddw_u8(vreinterpretq_u16_s16(res), s);
|
||||
const uint8x8_t d = vqmovun_s16(vreinterpretq_s16_u16(q));
|
||||
vst1_u8(*dest, d);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void idct16x16_add_store(const int16x8_t *const out,
|
||||
uint8_t *dest, const int stride) {
|
||||
// Add the result to dest
|
||||
idct16x16_add8x1(out[0], &dest, stride);
|
||||
idct16x16_add8x1(out[1], &dest, stride);
|
||||
idct16x16_add8x1(out[2], &dest, stride);
|
||||
idct16x16_add8x1(out[3], &dest, stride);
|
||||
idct16x16_add8x1(out[4], &dest, stride);
|
||||
idct16x16_add8x1(out[5], &dest, stride);
|
||||
idct16x16_add8x1(out[6], &dest, stride);
|
||||
idct16x16_add8x1(out[7], &dest, stride);
|
||||
idct16x16_add8x1(out[8], &dest, stride);
|
||||
idct16x16_add8x1(out[9], &dest, stride);
|
||||
idct16x16_add8x1(out[10], &dest, stride);
|
||||
idct16x16_add8x1(out[11], &dest, stride);
|
||||
idct16x16_add8x1(out[12], &dest, stride);
|
||||
idct16x16_add8x1(out[13], &dest, stride);
|
||||
idct16x16_add8x1(out[14], &dest, stride);
|
||||
idct16x16_add8x1(out[15], &dest, stride);
|
||||
}
|
||||
|
||||
static INLINE void highbd_idct16x16_add8x1(const int16x8_t a,
|
||||
const int16x8_t max,
|
||||
uint16_t **const dest,
|
||||
const int stride) {
|
||||
const uint16x8_t s = vld1q_u16(*dest);
|
||||
const int16x8_t res0 = vqaddq_s16(a, vreinterpretq_s16_u16(s));
|
||||
const int16x8_t res1 = vminq_s16(res0, max);
|
||||
const uint16x8_t d = vqshluq_n_s16(res1, 0);
|
||||
vst1q_u16(*dest, d);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void idct16x16_add_store_bd8(int16x8_t *const out, uint16_t *dest,
|
||||
const int stride) {
|
||||
// Add the result to dest
|
||||
const int16x8_t max = vdupq_n_s16((1 << 8) - 1);
|
||||
out[0] = vrshrq_n_s16(out[0], 6);
|
||||
out[1] = vrshrq_n_s16(out[1], 6);
|
||||
out[2] = vrshrq_n_s16(out[2], 6);
|
||||
out[3] = vrshrq_n_s16(out[3], 6);
|
||||
out[4] = vrshrq_n_s16(out[4], 6);
|
||||
out[5] = vrshrq_n_s16(out[5], 6);
|
||||
out[6] = vrshrq_n_s16(out[6], 6);
|
||||
out[7] = vrshrq_n_s16(out[7], 6);
|
||||
out[8] = vrshrq_n_s16(out[8], 6);
|
||||
out[9] = vrshrq_n_s16(out[9], 6);
|
||||
out[10] = vrshrq_n_s16(out[10], 6);
|
||||
out[11] = vrshrq_n_s16(out[11], 6);
|
||||
out[12] = vrshrq_n_s16(out[12], 6);
|
||||
out[13] = vrshrq_n_s16(out[13], 6);
|
||||
out[14] = vrshrq_n_s16(out[14], 6);
|
||||
out[15] = vrshrq_n_s16(out[15], 6);
|
||||
highbd_idct16x16_add8x1(out[0], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[1], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[2], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[3], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[4], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[5], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[6], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[7], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[8], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[9], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[10], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[11], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[12], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[13], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[14], max, &dest, stride);
|
||||
highbd_idct16x16_add8x1(out[15], max, &dest, stride);
|
||||
}
|
||||
|
||||
static INLINE void highbd_idct16x16_add8x1_bd8(const int16x8_t a,
|
||||
uint16_t **const dest,
|
||||
const int stride) {
|
||||
const uint16x8_t s = vld1q_u16(*dest);
|
||||
const int16x8_t res = vrsraq_n_s16(vreinterpretq_s16_u16(s), a, 6);
|
||||
const uint16x8_t d = vmovl_u8(vqmovun_s16(res));
|
||||
vst1q_u16(*dest, d);
|
||||
*dest += stride;
|
||||
}
|
||||
|
||||
static INLINE void highbd_add_and_store_bd8(const int16x8_t *const a,
|
||||
uint16_t *out, const int stride) {
|
||||
highbd_idct16x16_add8x1_bd8(a[0], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[1], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[2], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[3], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[4], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[5], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[6], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[7], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[8], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[9], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[10], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[11], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[12], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[13], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[14], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[15], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[16], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[17], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[18], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[19], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[20], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[21], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[22], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[23], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[24], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[25], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[26], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[27], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[28], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[29], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[30], &out, stride);
|
||||
highbd_idct16x16_add8x1_bd8(a[31], &out, stride);
|
||||
}
|
||||
|
||||
void vpx_idct16x16_256_add_half1d(const void *const input, int16_t *output,
|
||||
void *const dest, const int stride,
|
||||
const int highbd_flag);
|
||||
|
||||
void vpx_idct16x16_38_add_half1d(const void *const input, int16_t *const output,
|
||||
void *const dest, const int stride,
|
||||
const int highbd_flag);
|
||||
|
||||
void vpx_idct16x16_10_add_half1d_pass1(const tran_low_t *input,
|
||||
int16_t *output);
|
||||
|
||||
void vpx_idct16x16_10_add_half1d_pass2(const int16_t *input,
|
||||
int16_t *const output, void *const dest,
|
||||
const int stride, const int highbd_flag);
|
||||
|
||||
void vpx_idct32_32_neon(const tran_low_t *input, uint8_t *dest,
|
||||
const int stride, const int highbd_flag);
|
||||
|
||||
void vpx_idct32_12_neon(const tran_low_t *const input, int16_t *output);
|
||||
void vpx_idct32_16_neon(const int16_t *const input, void *const output,
|
||||
const int stride, const int highbd_flag);
|
||||
|
||||
void vpx_idct32_6_neon(const tran_low_t *input, int16_t *output);
|
||||
void vpx_idct32_8_neon(const int16_t *input, void *const output, int stride,
|
||||
const int highbd_flag);
|
||||
|
||||
#endif // VPX_VPX_DSP_ARM_IDCT_NEON_H_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,630 @@
|
||||
;
|
||||
; Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
|
||||
EXPORT |vpx_v_predictor_4x4_neon|
|
||||
EXPORT |vpx_v_predictor_8x8_neon|
|
||||
EXPORT |vpx_v_predictor_16x16_neon|
|
||||
EXPORT |vpx_v_predictor_32x32_neon|
|
||||
EXPORT |vpx_h_predictor_4x4_neon|
|
||||
EXPORT |vpx_h_predictor_8x8_neon|
|
||||
EXPORT |vpx_h_predictor_16x16_neon|
|
||||
EXPORT |vpx_h_predictor_32x32_neon|
|
||||
EXPORT |vpx_tm_predictor_4x4_neon|
|
||||
EXPORT |vpx_tm_predictor_8x8_neon|
|
||||
EXPORT |vpx_tm_predictor_16x16_neon|
|
||||
EXPORT |vpx_tm_predictor_32x32_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
;void vpx_v_predictor_4x4_neon(uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_v_predictor_4x4_neon| PROC
|
||||
vld1.32 {d0[0]}, [r2]
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
bx lr
|
||||
ENDP ; |vpx_v_predictor_4x4_neon|
|
||||
|
||||
;void vpx_v_predictor_8x8_neon(uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_v_predictor_8x8_neon| PROC
|
||||
vld1.8 {d0}, [r2]
|
||||
vst1.8 {d0}, [r0], r1
|
||||
vst1.8 {d0}, [r0], r1
|
||||
vst1.8 {d0}, [r0], r1
|
||||
vst1.8 {d0}, [r0], r1
|
||||
vst1.8 {d0}, [r0], r1
|
||||
vst1.8 {d0}, [r0], r1
|
||||
vst1.8 {d0}, [r0], r1
|
||||
vst1.8 {d0}, [r0], r1
|
||||
bx lr
|
||||
ENDP ; |vpx_v_predictor_8x8_neon|
|
||||
|
||||
;void vpx_v_predictor_16x16_neon(uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_v_predictor_16x16_neon| PROC
|
||||
vld1.8 {q0}, [r2]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vst1.8 {q0}, [r0], r1
|
||||
bx lr
|
||||
ENDP ; |vpx_v_predictor_16x16_neon|
|
||||
|
||||
;void vpx_v_predictor_32x32_neon(uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_v_predictor_32x32_neon| PROC
|
||||
vld1.8 {q0, q1}, [r2]
|
||||
mov r2, #2
|
||||
loop_v
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
vst1.8 {q0, q1}, [r0], r1
|
||||
subs r2, r2, #1
|
||||
bgt loop_v
|
||||
bx lr
|
||||
ENDP ; |vpx_v_predictor_32x32_neon|
|
||||
|
||||
;void vpx_h_predictor_4x4_neon(uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_h_predictor_4x4_neon| PROC
|
||||
vld1.32 {d1[0]}, [r3]
|
||||
vdup.8 d0, d1[0]
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
vdup.8 d0, d1[1]
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
vdup.8 d0, d1[2]
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
vdup.8 d0, d1[3]
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
bx lr
|
||||
ENDP ; |vpx_h_predictor_4x4_neon|
|
||||
|
||||
;void vpx_h_predictor_8x8_neon(uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_h_predictor_8x8_neon| PROC
|
||||
vld1.64 {d1}, [r3]
|
||||
vdup.8 d0, d1[0]
|
||||
vst1.64 {d0}, [r0], r1
|
||||
vdup.8 d0, d1[1]
|
||||
vst1.64 {d0}, [r0], r1
|
||||
vdup.8 d0, d1[2]
|
||||
vst1.64 {d0}, [r0], r1
|
||||
vdup.8 d0, d1[3]
|
||||
vst1.64 {d0}, [r0], r1
|
||||
vdup.8 d0, d1[4]
|
||||
vst1.64 {d0}, [r0], r1
|
||||
vdup.8 d0, d1[5]
|
||||
vst1.64 {d0}, [r0], r1
|
||||
vdup.8 d0, d1[6]
|
||||
vst1.64 {d0}, [r0], r1
|
||||
vdup.8 d0, d1[7]
|
||||
vst1.64 {d0}, [r0], r1
|
||||
bx lr
|
||||
ENDP ; |vpx_h_predictor_8x8_neon|
|
||||
|
||||
;void vpx_h_predictor_16x16_neon(uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_h_predictor_16x16_neon| PROC
|
||||
vld1.8 {q1}, [r3]
|
||||
vdup.8 q0, d2[0]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[1]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[2]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[3]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[4]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[5]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[6]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[7]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[0]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[1]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[2]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[3]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[4]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[5]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[6]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[7]
|
||||
vst1.8 {q0}, [r0], r1
|
||||
bx lr
|
||||
ENDP ; |vpx_h_predictor_16x16_neon|
|
||||
|
||||
;void vpx_h_predictor_32x32_neon(uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_h_predictor_32x32_neon| PROC
|
||||
sub r1, r1, #16
|
||||
mov r2, #2
|
||||
loop_h
|
||||
vld1.8 {q1}, [r3]!
|
||||
vdup.8 q0, d2[0]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[1]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[2]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[3]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[4]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[5]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[6]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d2[7]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[0]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[1]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[2]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[3]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[4]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[5]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[6]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
vdup.8 q0, d3[7]
|
||||
vst1.8 {q0}, [r0]!
|
||||
vst1.8 {q0}, [r0], r1
|
||||
subs r2, r2, #1
|
||||
bgt loop_h
|
||||
bx lr
|
||||
ENDP ; |vpx_h_predictor_32x32_neon|
|
||||
|
||||
;void vpx_tm_predictor_4x4_neon (uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_tm_predictor_4x4_neon| PROC
|
||||
; Load ytop_left = above[-1];
|
||||
sub r12, r2, #1
|
||||
vld1.u8 {d0[]}, [r12]
|
||||
|
||||
; Load above 4 pixels
|
||||
vld1.32 {d2[0]}, [r2]
|
||||
|
||||
; Compute above - ytop_left
|
||||
vsubl.u8 q3, d2, d0
|
||||
|
||||
; Load left row by row and compute left + (above - ytop_left)
|
||||
; 1st row and 2nd row
|
||||
vld1.u8 {d2[]}, [r3]!
|
||||
vld1.u8 {d4[]}, [r3]!
|
||||
vmovl.u8 q1, d2
|
||||
vmovl.u8 q2, d4
|
||||
vadd.s16 q1, q1, q3
|
||||
vadd.s16 q2, q2, q3
|
||||
vqmovun.s16 d0, q1
|
||||
vqmovun.s16 d1, q2
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
vst1.32 {d1[0]}, [r0], r1
|
||||
|
||||
; 3rd row and 4th row
|
||||
vld1.u8 {d2[]}, [r3]!
|
||||
vld1.u8 {d4[]}, [r3]
|
||||
vmovl.u8 q1, d2
|
||||
vmovl.u8 q2, d4
|
||||
vadd.s16 q1, q1, q3
|
||||
vadd.s16 q2, q2, q3
|
||||
vqmovun.s16 d0, q1
|
||||
vqmovun.s16 d1, q2
|
||||
vst1.32 {d0[0]}, [r0], r1
|
||||
vst1.32 {d1[0]}, [r0], r1
|
||||
bx lr
|
||||
ENDP ; |vpx_tm_predictor_4x4_neon|
|
||||
|
||||
;void vpx_tm_predictor_8x8_neon (uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_tm_predictor_8x8_neon| PROC
|
||||
; Load ytop_left = above[-1];
|
||||
sub r12, r2, #1
|
||||
vld1.8 {d0[]}, [r12]
|
||||
|
||||
; preload 8 left
|
||||
vld1.8 {d30}, [r3]
|
||||
|
||||
; Load above 8 pixels
|
||||
vld1.64 {d2}, [r2]
|
||||
|
||||
vmovl.u8 q10, d30
|
||||
|
||||
; Compute above - ytop_left
|
||||
vsubl.u8 q3, d2, d0
|
||||
|
||||
; Load left row by row and compute left + (above - ytop_left)
|
||||
; 1st row and 2nd row
|
||||
vdup.16 q0, d20[0]
|
||||
vdup.16 q1, d20[1]
|
||||
vadd.s16 q0, q3, q0
|
||||
vadd.s16 q1, q3, q1
|
||||
|
||||
; 3rd row and 4th row
|
||||
vdup.16 q8, d20[2]
|
||||
vdup.16 q9, d20[3]
|
||||
vadd.s16 q8, q3, q8
|
||||
vadd.s16 q9, q3, q9
|
||||
|
||||
vqmovun.s16 d0, q0
|
||||
vqmovun.s16 d1, q1
|
||||
vqmovun.s16 d2, q8
|
||||
vqmovun.s16 d3, q9
|
||||
|
||||
vst1.64 {d0}, [r0], r1
|
||||
vst1.64 {d1}, [r0], r1
|
||||
vst1.64 {d2}, [r0], r1
|
||||
vst1.64 {d3}, [r0], r1
|
||||
|
||||
; 5th row and 6th row
|
||||
vdup.16 q0, d21[0]
|
||||
vdup.16 q1, d21[1]
|
||||
vadd.s16 q0, q3, q0
|
||||
vadd.s16 q1, q3, q1
|
||||
|
||||
; 7th row and 8th row
|
||||
vdup.16 q8, d21[2]
|
||||
vdup.16 q9, d21[3]
|
||||
vadd.s16 q8, q3, q8
|
||||
vadd.s16 q9, q3, q9
|
||||
|
||||
vqmovun.s16 d0, q0
|
||||
vqmovun.s16 d1, q1
|
||||
vqmovun.s16 d2, q8
|
||||
vqmovun.s16 d3, q9
|
||||
|
||||
vst1.64 {d0}, [r0], r1
|
||||
vst1.64 {d1}, [r0], r1
|
||||
vst1.64 {d2}, [r0], r1
|
||||
vst1.64 {d3}, [r0], r1
|
||||
|
||||
bx lr
|
||||
ENDP ; |vpx_tm_predictor_8x8_neon|
|
||||
|
||||
;void vpx_tm_predictor_16x16_neon (uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_tm_predictor_16x16_neon| PROC
|
||||
; Load ytop_left = above[-1];
|
||||
sub r12, r2, #1
|
||||
vld1.8 {d0[]}, [r12]
|
||||
|
||||
; Load above 8 pixels
|
||||
vld1.8 {q1}, [r2]
|
||||
|
||||
; preload 8 left into r12
|
||||
vld1.8 {d18}, [r3]!
|
||||
|
||||
; Compute above - ytop_left
|
||||
vsubl.u8 q2, d2, d0
|
||||
vsubl.u8 q3, d3, d0
|
||||
|
||||
vmovl.u8 q10, d18
|
||||
|
||||
; Load left row by row and compute left + (above - ytop_left)
|
||||
; Process 8 rows in each single loop and loop 2 times to process 16 rows.
|
||||
mov r2, #2
|
||||
|
||||
loop_16x16_neon
|
||||
; Process two rows.
|
||||
vdup.16 q0, d20[0]
|
||||
vdup.16 q8, d20[1]
|
||||
vadd.s16 q1, q0, q2
|
||||
vadd.s16 q0, q0, q3
|
||||
vadd.s16 q11, q8, q2
|
||||
vadd.s16 q8, q8, q3
|
||||
vqmovun.s16 d2, q1
|
||||
vqmovun.s16 d3, q0
|
||||
vqmovun.s16 d22, q11
|
||||
vqmovun.s16 d23, q8
|
||||
vdup.16 q0, d20[2] ; proload next 2 rows data
|
||||
vdup.16 q8, d20[3]
|
||||
vst1.64 {d2,d3}, [r0], r1
|
||||
vst1.64 {d22,d23}, [r0], r1
|
||||
|
||||
; Process two rows.
|
||||
vadd.s16 q1, q0, q2
|
||||
vadd.s16 q0, q0, q3
|
||||
vadd.s16 q11, q8, q2
|
||||
vadd.s16 q8, q8, q3
|
||||
vqmovun.s16 d2, q1
|
||||
vqmovun.s16 d3, q0
|
||||
vqmovun.s16 d22, q11
|
||||
vqmovun.s16 d23, q8
|
||||
vdup.16 q0, d21[0] ; proload next 2 rows data
|
||||
vdup.16 q8, d21[1]
|
||||
vst1.64 {d2,d3}, [r0], r1
|
||||
vst1.64 {d22,d23}, [r0], r1
|
||||
|
||||
vadd.s16 q1, q0, q2
|
||||
vadd.s16 q0, q0, q3
|
||||
vadd.s16 q11, q8, q2
|
||||
vadd.s16 q8, q8, q3
|
||||
vqmovun.s16 d2, q1
|
||||
vqmovun.s16 d3, q0
|
||||
vqmovun.s16 d22, q11
|
||||
vqmovun.s16 d23, q8
|
||||
vdup.16 q0, d21[2] ; proload next 2 rows data
|
||||
vdup.16 q8, d21[3]
|
||||
vst1.64 {d2,d3}, [r0], r1
|
||||
vst1.64 {d22,d23}, [r0], r1
|
||||
|
||||
|
||||
vadd.s16 q1, q0, q2
|
||||
vadd.s16 q0, q0, q3
|
||||
vadd.s16 q11, q8, q2
|
||||
vadd.s16 q8, q8, q3
|
||||
vqmovun.s16 d2, q1
|
||||
vqmovun.s16 d3, q0
|
||||
vqmovun.s16 d22, q11
|
||||
vqmovun.s16 d23, q8
|
||||
vld1.8 {d18}, [r3]! ; preload 8 left into r12
|
||||
vmovl.u8 q10, d18
|
||||
vst1.64 {d2,d3}, [r0], r1
|
||||
vst1.64 {d22,d23}, [r0], r1
|
||||
|
||||
subs r2, r2, #1
|
||||
bgt loop_16x16_neon
|
||||
|
||||
bx lr
|
||||
ENDP ; |vpx_tm_predictor_16x16_neon|
|
||||
|
||||
;void vpx_tm_predictor_32x32_neon (uint8_t *dst, ptrdiff_t y_stride,
|
||||
; const uint8_t *above,
|
||||
; const uint8_t *left)
|
||||
; r0 uint8_t *dst
|
||||
; r1 ptrdiff_t y_stride
|
||||
; r2 const uint8_t *above
|
||||
; r3 const uint8_t *left
|
||||
|
||||
|vpx_tm_predictor_32x32_neon| PROC
|
||||
; Load ytop_left = above[-1];
|
||||
sub r12, r2, #1
|
||||
vld1.8 {d0[]}, [r12]
|
||||
|
||||
; Load above 32 pixels
|
||||
vld1.8 {q1}, [r2]!
|
||||
vld1.8 {q2}, [r2]
|
||||
|
||||
; preload 8 left pixels
|
||||
vld1.8 {d26}, [r3]!
|
||||
|
||||
; Compute above - ytop_left
|
||||
vsubl.u8 q8, d2, d0
|
||||
vsubl.u8 q9, d3, d0
|
||||
vsubl.u8 q10, d4, d0
|
||||
vsubl.u8 q11, d5, d0
|
||||
|
||||
vmovl.u8 q3, d26
|
||||
|
||||
; Load left row by row and compute left + (above - ytop_left)
|
||||
; Process 8 rows in each single loop and loop 4 times to process 32 rows.
|
||||
mov r2, #4
|
||||
|
||||
loop_32x32_neon
|
||||
; Process two rows.
|
||||
vdup.16 q0, d6[0]
|
||||
vdup.16 q2, d6[1]
|
||||
vadd.s16 q12, q0, q8
|
||||
vadd.s16 q13, q0, q9
|
||||
vadd.s16 q14, q0, q10
|
||||
vadd.s16 q15, q0, q11
|
||||
vqmovun.s16 d0, q12
|
||||
vqmovun.s16 d1, q13
|
||||
vadd.s16 q12, q2, q8
|
||||
vadd.s16 q13, q2, q9
|
||||
vqmovun.s16 d2, q14
|
||||
vqmovun.s16 d3, q15
|
||||
vadd.s16 q14, q2, q10
|
||||
vadd.s16 q15, q2, q11
|
||||
vst1.64 {d0-d3}, [r0], r1
|
||||
vqmovun.s16 d24, q12
|
||||
vqmovun.s16 d25, q13
|
||||
vqmovun.s16 d26, q14
|
||||
vqmovun.s16 d27, q15
|
||||
vdup.16 q1, d6[2]
|
||||
vdup.16 q2, d6[3]
|
||||
vst1.64 {d24-d27}, [r0], r1
|
||||
|
||||
; Process two rows.
|
||||
vadd.s16 q12, q1, q8
|
||||
vadd.s16 q13, q1, q9
|
||||
vadd.s16 q14, q1, q10
|
||||
vadd.s16 q15, q1, q11
|
||||
vqmovun.s16 d0, q12
|
||||
vqmovun.s16 d1, q13
|
||||
vadd.s16 q12, q2, q8
|
||||
vadd.s16 q13, q2, q9
|
||||
vqmovun.s16 d2, q14
|
||||
vqmovun.s16 d3, q15
|
||||
vadd.s16 q14, q2, q10
|
||||
vadd.s16 q15, q2, q11
|
||||
vst1.64 {d0-d3}, [r0], r1
|
||||
vqmovun.s16 d24, q12
|
||||
vqmovun.s16 d25, q13
|
||||
vqmovun.s16 d26, q14
|
||||
vqmovun.s16 d27, q15
|
||||
vdup.16 q0, d7[0]
|
||||
vdup.16 q2, d7[1]
|
||||
vst1.64 {d24-d27}, [r0], r1
|
||||
|
||||
; Process two rows.
|
||||
vadd.s16 q12, q0, q8
|
||||
vadd.s16 q13, q0, q9
|
||||
vadd.s16 q14, q0, q10
|
||||
vadd.s16 q15, q0, q11
|
||||
vqmovun.s16 d0, q12
|
||||
vqmovun.s16 d1, q13
|
||||
vadd.s16 q12, q2, q8
|
||||
vadd.s16 q13, q2, q9
|
||||
vqmovun.s16 d2, q14
|
||||
vqmovun.s16 d3, q15
|
||||
vadd.s16 q14, q2, q10
|
||||
vadd.s16 q15, q2, q11
|
||||
vst1.64 {d0-d3}, [r0], r1
|
||||
vqmovun.s16 d24, q12
|
||||
vqmovun.s16 d25, q13
|
||||
vqmovun.s16 d26, q14
|
||||
vqmovun.s16 d27, q15
|
||||
vdup.16 q0, d7[2]
|
||||
vdup.16 q2, d7[3]
|
||||
vst1.64 {d24-d27}, [r0], r1
|
||||
|
||||
; Process two rows.
|
||||
vadd.s16 q12, q0, q8
|
||||
vadd.s16 q13, q0, q9
|
||||
vadd.s16 q14, q0, q10
|
||||
vadd.s16 q15, q0, q11
|
||||
vqmovun.s16 d0, q12
|
||||
vqmovun.s16 d1, q13
|
||||
vadd.s16 q12, q2, q8
|
||||
vadd.s16 q13, q2, q9
|
||||
vqmovun.s16 d2, q14
|
||||
vqmovun.s16 d3, q15
|
||||
vadd.s16 q14, q2, q10
|
||||
vadd.s16 q15, q2, q11
|
||||
vst1.64 {d0-d3}, [r0], r1
|
||||
vqmovun.s16 d24, q12
|
||||
vqmovun.s16 d25, q13
|
||||
vld1.8 {d0}, [r3]! ; preload 8 left pixels
|
||||
vqmovun.s16 d26, q14
|
||||
vqmovun.s16 d27, q15
|
||||
vmovl.u8 q3, d0
|
||||
vst1.64 {d24-d27}, [r0], r1
|
||||
|
||||
subs r2, r2, #1
|
||||
bgt loop_32x32_neon
|
||||
|
||||
bx lr
|
||||
ENDP ; |vpx_tm_predictor_32x32_neon|
|
||||
|
||||
END
|
||||
@@ -0,0 +1,666 @@
|
||||
;
|
||||
; Copyright (c) 2013 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
|
||||
EXPORT |vpx_lpf_horizontal_16_neon|
|
||||
EXPORT |vpx_lpf_horizontal_16_dual_neon|
|
||||
EXPORT |vpx_lpf_vertical_16_neon|
|
||||
EXPORT |vpx_lpf_vertical_16_dual_neon|
|
||||
ARM
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
; void mb_lpf_horizontal_edge(uint8_t *s, int p,
|
||||
; const uint8_t *blimit,
|
||||
; const uint8_t *limit,
|
||||
; const uint8_t *thresh,
|
||||
; int count)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p, /* pitch */
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh,
|
||||
; r12 int count
|
||||
|mb_lpf_horizontal_edge| PROC
|
||||
push {r4-r8, lr}
|
||||
vpush {d8-d15}
|
||||
ldr r4, [sp, #88] ; load thresh
|
||||
|
||||
h_count
|
||||
vld1.8 {d16[]}, [r2] ; load *blimit
|
||||
vld1.8 {d17[]}, [r3] ; load *limit
|
||||
vld1.8 {d18[]}, [r4] ; load *thresh
|
||||
|
||||
sub r8, r0, r1, lsl #3 ; move src pointer down by 8 lines
|
||||
|
||||
vld1.u8 {d0}, [r8@64], r1 ; p7
|
||||
vld1.u8 {d1}, [r8@64], r1 ; p6
|
||||
vld1.u8 {d2}, [r8@64], r1 ; p5
|
||||
vld1.u8 {d3}, [r8@64], r1 ; p4
|
||||
vld1.u8 {d4}, [r8@64], r1 ; p3
|
||||
vld1.u8 {d5}, [r8@64], r1 ; p2
|
||||
vld1.u8 {d6}, [r8@64], r1 ; p1
|
||||
vld1.u8 {d7}, [r8@64], r1 ; p0
|
||||
vld1.u8 {d8}, [r8@64], r1 ; q0
|
||||
vld1.u8 {d9}, [r8@64], r1 ; q1
|
||||
vld1.u8 {d10}, [r8@64], r1 ; q2
|
||||
vld1.u8 {d11}, [r8@64], r1 ; q3
|
||||
vld1.u8 {d12}, [r8@64], r1 ; q4
|
||||
vld1.u8 {d13}, [r8@64], r1 ; q5
|
||||
vld1.u8 {d14}, [r8@64], r1 ; q6
|
||||
vld1.u8 {d15}, [r8@64], r1 ; q7
|
||||
|
||||
bl vpx_wide_mbfilter_neon
|
||||
|
||||
tst r7, #1
|
||||
beq h_mbfilter
|
||||
|
||||
; flat && mask were not set for any of the channels. Just store the values
|
||||
; from filter.
|
||||
sub r8, r0, r1, lsl #1
|
||||
|
||||
vst1.u8 {d25}, [r8@64], r1 ; store op1
|
||||
vst1.u8 {d24}, [r8@64], r1 ; store op0
|
||||
vst1.u8 {d23}, [r8@64], r1 ; store oq0
|
||||
vst1.u8 {d26}, [r8@64], r1 ; store oq1
|
||||
|
||||
b h_next
|
||||
|
||||
h_mbfilter
|
||||
tst r7, #2
|
||||
beq h_wide_mbfilter
|
||||
|
||||
; flat2 was not set for any of the channels. Just store the values from
|
||||
; mbfilter.
|
||||
sub r8, r0, r1, lsl #1
|
||||
sub r8, r8, r1
|
||||
|
||||
vst1.u8 {d18}, [r8@64], r1 ; store op2
|
||||
vst1.u8 {d19}, [r8@64], r1 ; store op1
|
||||
vst1.u8 {d20}, [r8@64], r1 ; store op0
|
||||
vst1.u8 {d21}, [r8@64], r1 ; store oq0
|
||||
vst1.u8 {d22}, [r8@64], r1 ; store oq1
|
||||
vst1.u8 {d23}, [r8@64], r1 ; store oq2
|
||||
|
||||
b h_next
|
||||
|
||||
h_wide_mbfilter
|
||||
sub r8, r0, r1, lsl #3
|
||||
add r8, r8, r1
|
||||
|
||||
vst1.u8 {d16}, [r8@64], r1 ; store op6
|
||||
vst1.u8 {d24}, [r8@64], r1 ; store op5
|
||||
vst1.u8 {d25}, [r8@64], r1 ; store op4
|
||||
vst1.u8 {d26}, [r8@64], r1 ; store op3
|
||||
vst1.u8 {d27}, [r8@64], r1 ; store op2
|
||||
vst1.u8 {d18}, [r8@64], r1 ; store op1
|
||||
vst1.u8 {d19}, [r8@64], r1 ; store op0
|
||||
vst1.u8 {d20}, [r8@64], r1 ; store oq0
|
||||
vst1.u8 {d21}, [r8@64], r1 ; store oq1
|
||||
vst1.u8 {d22}, [r8@64], r1 ; store oq2
|
||||
vst1.u8 {d23}, [r8@64], r1 ; store oq3
|
||||
vst1.u8 {d1}, [r8@64], r1 ; store oq4
|
||||
vst1.u8 {d2}, [r8@64], r1 ; store oq5
|
||||
vst1.u8 {d3}, [r8@64], r1 ; store oq6
|
||||
|
||||
h_next
|
||||
add r0, r0, #8
|
||||
subs r12, r12, #1
|
||||
bne h_count
|
||||
|
||||
vpop {d8-d15}
|
||||
pop {r4-r8, pc}
|
||||
|
||||
ENDP ; |mb_lpf_horizontal_edge|
|
||||
|
||||
; void vpx_lpf_horizontal_16_neon(uint8_t *s, int pitch,
|
||||
; const uint8_t *blimit,
|
||||
; const uint8_t *limit,
|
||||
; const uint8_t *thresh)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int pitch,
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh
|
||||
|vpx_lpf_horizontal_16_neon| PROC
|
||||
mov r12, #1
|
||||
b mb_lpf_horizontal_edge
|
||||
ENDP ; |vpx_lpf_horizontal_16_neon|
|
||||
|
||||
; void vpx_lpf_horizontal_16_dual_neon(uint8_t *s, int pitch,
|
||||
; const uint8_t *blimit,
|
||||
; const uint8_t *limit,
|
||||
; const uint8_t *thresh)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int pitch,
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh
|
||||
|vpx_lpf_horizontal_16_dual_neon| PROC
|
||||
mov r12, #2
|
||||
b mb_lpf_horizontal_edge
|
||||
ENDP ; |vpx_lpf_horizontal_16_dual_neon|
|
||||
|
||||
; void mb_lpf_vertical_edge_w(uint8_t *s, int p, const uint8_t *blimit,
|
||||
; const uint8_t *limit, const uint8_t *thresh,
|
||||
; int count) {
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p, /* pitch */
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh,
|
||||
; r12 int count
|
||||
|mb_lpf_vertical_edge_w| PROC
|
||||
push {r4-r8, lr}
|
||||
vpush {d8-d15}
|
||||
ldr r4, [sp, #88] ; load thresh
|
||||
|
||||
v_count
|
||||
vld1.8 {d16[]}, [r2] ; load *blimit
|
||||
vld1.8 {d17[]}, [r3] ; load *limit
|
||||
vld1.8 {d18[]}, [r4] ; load *thresh
|
||||
|
||||
sub r8, r0, #8
|
||||
|
||||
vld1.8 {d0}, [r8@64], r1
|
||||
vld1.8 {d8}, [r0@64], r1
|
||||
vld1.8 {d1}, [r8@64], r1
|
||||
vld1.8 {d9}, [r0@64], r1
|
||||
vld1.8 {d2}, [r8@64], r1
|
||||
vld1.8 {d10}, [r0@64], r1
|
||||
vld1.8 {d3}, [r8@64], r1
|
||||
vld1.8 {d11}, [r0@64], r1
|
||||
vld1.8 {d4}, [r8@64], r1
|
||||
vld1.8 {d12}, [r0@64], r1
|
||||
vld1.8 {d5}, [r8@64], r1
|
||||
vld1.8 {d13}, [r0@64], r1
|
||||
vld1.8 {d6}, [r8@64], r1
|
||||
vld1.8 {d14}, [r0@64], r1
|
||||
vld1.8 {d7}, [r8@64], r1
|
||||
vld1.8 {d15}, [r0@64], r1
|
||||
|
||||
sub r0, r0, r1, lsl #3
|
||||
|
||||
vtrn.32 q0, q2
|
||||
vtrn.32 q1, q3
|
||||
vtrn.32 q4, q6
|
||||
vtrn.32 q5, q7
|
||||
|
||||
vtrn.16 q0, q1
|
||||
vtrn.16 q2, q3
|
||||
vtrn.16 q4, q5
|
||||
vtrn.16 q6, q7
|
||||
|
||||
vtrn.8 d0, d1
|
||||
vtrn.8 d2, d3
|
||||
vtrn.8 d4, d5
|
||||
vtrn.8 d6, d7
|
||||
|
||||
vtrn.8 d8, d9
|
||||
vtrn.8 d10, d11
|
||||
vtrn.8 d12, d13
|
||||
vtrn.8 d14, d15
|
||||
|
||||
bl vpx_wide_mbfilter_neon
|
||||
|
||||
tst r7, #1
|
||||
beq v_mbfilter
|
||||
|
||||
; flat && mask were not set for any of the channels. Just store the values
|
||||
; from filter.
|
||||
sub r0, #2
|
||||
|
||||
vswp d23, d25
|
||||
|
||||
vst4.8 {d23[0], d24[0], d25[0], d26[0]}, [r0], r1
|
||||
vst4.8 {d23[1], d24[1], d25[1], d26[1]}, [r0], r1
|
||||
vst4.8 {d23[2], d24[2], d25[2], d26[2]}, [r0], r1
|
||||
vst4.8 {d23[3], d24[3], d25[3], d26[3]}, [r0], r1
|
||||
vst4.8 {d23[4], d24[4], d25[4], d26[4]}, [r0], r1
|
||||
vst4.8 {d23[5], d24[5], d25[5], d26[5]}, [r0], r1
|
||||
vst4.8 {d23[6], d24[6], d25[6], d26[6]}, [r0], r1
|
||||
vst4.8 {d23[7], d24[7], d25[7], d26[7]}, [r0], r1
|
||||
add r0, #2
|
||||
|
||||
b v_next
|
||||
|
||||
v_mbfilter
|
||||
tst r7, #2
|
||||
beq v_wide_mbfilter
|
||||
|
||||
; flat2 was not set for any of the channels. Just store the values from
|
||||
; mbfilter.
|
||||
sub r8, r0, #3
|
||||
|
||||
vst3.8 {d18[0], d19[0], d20[0]}, [r8], r1
|
||||
vst3.8 {d21[0], d22[0], d23[0]}, [r0], r1
|
||||
vst3.8 {d18[1], d19[1], d20[1]}, [r8], r1
|
||||
vst3.8 {d21[1], d22[1], d23[1]}, [r0], r1
|
||||
vst3.8 {d18[2], d19[2], d20[2]}, [r8], r1
|
||||
vst3.8 {d21[2], d22[2], d23[2]}, [r0], r1
|
||||
vst3.8 {d18[3], d19[3], d20[3]}, [r8], r1
|
||||
vst3.8 {d21[3], d22[3], d23[3]}, [r0], r1
|
||||
vst3.8 {d18[4], d19[4], d20[4]}, [r8], r1
|
||||
vst3.8 {d21[4], d22[4], d23[4]}, [r0], r1
|
||||
vst3.8 {d18[5], d19[5], d20[5]}, [r8], r1
|
||||
vst3.8 {d21[5], d22[5], d23[5]}, [r0], r1
|
||||
vst3.8 {d18[6], d19[6], d20[6]}, [r8], r1
|
||||
vst3.8 {d21[6], d22[6], d23[6]}, [r0], r1
|
||||
vst3.8 {d18[7], d19[7], d20[7]}, [r8], r1
|
||||
vst3.8 {d21[7], d22[7], d23[7]}, [r0], r1
|
||||
|
||||
b v_next
|
||||
|
||||
v_wide_mbfilter
|
||||
sub r8, r0, #8
|
||||
|
||||
vtrn.32 d0, d26
|
||||
vtrn.32 d16, d27
|
||||
vtrn.32 d24, d18
|
||||
vtrn.32 d25, d19
|
||||
|
||||
vtrn.16 d0, d24
|
||||
vtrn.16 d16, d25
|
||||
vtrn.16 d26, d18
|
||||
vtrn.16 d27, d19
|
||||
|
||||
vtrn.8 d0, d16
|
||||
vtrn.8 d24, d25
|
||||
vtrn.8 d26, d27
|
||||
vtrn.8 d18, d19
|
||||
|
||||
vtrn.32 d20, d1
|
||||
vtrn.32 d21, d2
|
||||
vtrn.32 d22, d3
|
||||
vtrn.32 d23, d15
|
||||
|
||||
vtrn.16 d20, d22
|
||||
vtrn.16 d21, d23
|
||||
vtrn.16 d1, d3
|
||||
vtrn.16 d2, d15
|
||||
|
||||
vtrn.8 d20, d21
|
||||
vtrn.8 d22, d23
|
||||
vtrn.8 d1, d2
|
||||
vtrn.8 d3, d15
|
||||
|
||||
vst1.8 {d0}, [r8@64], r1
|
||||
vst1.8 {d20}, [r0@64], r1
|
||||
vst1.8 {d16}, [r8@64], r1
|
||||
vst1.8 {d21}, [r0@64], r1
|
||||
vst1.8 {d24}, [r8@64], r1
|
||||
vst1.8 {d22}, [r0@64], r1
|
||||
vst1.8 {d25}, [r8@64], r1
|
||||
vst1.8 {d23}, [r0@64], r1
|
||||
vst1.8 {d26}, [r8@64], r1
|
||||
vst1.8 {d1}, [r0@64], r1
|
||||
vst1.8 {d27}, [r8@64], r1
|
||||
vst1.8 {d2}, [r0@64], r1
|
||||
vst1.8 {d18}, [r8@64], r1
|
||||
vst1.8 {d3}, [r0@64], r1
|
||||
vst1.8 {d19}, [r8@64], r1
|
||||
vst1.8 {d15}, [r0@64], r1
|
||||
|
||||
v_next
|
||||
subs r12, #1
|
||||
bne v_count
|
||||
|
||||
vpop {d8-d15}
|
||||
pop {r4-r8, pc}
|
||||
|
||||
ENDP ; |mb_lpf_vertical_edge_w|
|
||||
|
||||
; void vpx_lpf_vertical_16_neon(uint8_t *s, int p, const uint8_t *blimit,
|
||||
; const uint8_t *limit, const uint8_t *thresh)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p, /* pitch */
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh
|
||||
|vpx_lpf_vertical_16_neon| PROC
|
||||
mov r12, #1
|
||||
b mb_lpf_vertical_edge_w
|
||||
ENDP ; |vpx_lpf_vertical_16_neon|
|
||||
|
||||
; void vpx_lpf_vertical_16_dual_neon(uint8_t *s, int p, const uint8_t *blimit,
|
||||
; const uint8_t *limit,
|
||||
; const uint8_t *thresh)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p, /* pitch */
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh
|
||||
|vpx_lpf_vertical_16_dual_neon| PROC
|
||||
mov r12, #2
|
||||
b mb_lpf_vertical_edge_w
|
||||
ENDP ; |vpx_lpf_vertical_16_dual_neon|
|
||||
|
||||
; void vpx_wide_mbfilter_neon();
|
||||
; This is a helper function for the loopfilters. The invidual functions do the
|
||||
; necessary load, transpose (if necessary) and store.
|
||||
;
|
||||
; r0-r3 PRESERVE
|
||||
; d16 blimit
|
||||
; d17 limit
|
||||
; d18 thresh
|
||||
; d0 p7
|
||||
; d1 p6
|
||||
; d2 p5
|
||||
; d3 p4
|
||||
; d4 p3
|
||||
; d5 p2
|
||||
; d6 p1
|
||||
; d7 p0
|
||||
; d8 q0
|
||||
; d9 q1
|
||||
; d10 q2
|
||||
; d11 q3
|
||||
; d12 q4
|
||||
; d13 q5
|
||||
; d14 q6
|
||||
; d15 q7
|
||||
|vpx_wide_mbfilter_neon| PROC
|
||||
mov r7, #0
|
||||
|
||||
; filter_mask
|
||||
vabd.u8 d19, d4, d5 ; abs(p3 - p2)
|
||||
vabd.u8 d20, d5, d6 ; abs(p2 - p1)
|
||||
vabd.u8 d21, d6, d7 ; abs(p1 - p0)
|
||||
vabd.u8 d22, d9, d8 ; abs(q1 - q0)
|
||||
vabd.u8 d23, d10, d9 ; abs(q2 - q1)
|
||||
vabd.u8 d24, d11, d10 ; abs(q3 - q2)
|
||||
|
||||
; only compare the largest value to limit
|
||||
vmax.u8 d19, d19, d20 ; max(abs(p3 - p2), abs(p2 - p1))
|
||||
vmax.u8 d20, d21, d22 ; max(abs(p1 - p0), abs(q1 - q0))
|
||||
vmax.u8 d23, d23, d24 ; max(abs(q2 - q1), abs(q3 - q2))
|
||||
vmax.u8 d19, d19, d20
|
||||
|
||||
vabd.u8 d24, d7, d8 ; abs(p0 - q0)
|
||||
|
||||
vmax.u8 d19, d19, d23
|
||||
|
||||
vabd.u8 d23, d6, d9 ; a = abs(p1 - q1)
|
||||
vqadd.u8 d24, d24, d24 ; b = abs(p0 - q0) * 2
|
||||
|
||||
; abs () > limit
|
||||
vcge.u8 d19, d17, d19
|
||||
|
||||
; flatmask4
|
||||
vabd.u8 d25, d7, d5 ; abs(p0 - p2)
|
||||
vabd.u8 d26, d8, d10 ; abs(q0 - q2)
|
||||
vabd.u8 d27, d4, d7 ; abs(p3 - p0)
|
||||
vabd.u8 d28, d11, d8 ; abs(q3 - q0)
|
||||
|
||||
; only compare the largest value to thresh
|
||||
vmax.u8 d25, d25, d26 ; max(abs(p0 - p2), abs(q0 - q2))
|
||||
vmax.u8 d26, d27, d28 ; max(abs(p3 - p0), abs(q3 - q0))
|
||||
vmax.u8 d25, d25, d26
|
||||
vmax.u8 d20, d20, d25
|
||||
|
||||
vshr.u8 d23, d23, #1 ; a = a / 2
|
||||
vqadd.u8 d24, d24, d23 ; a = b + a
|
||||
|
||||
vmov.u8 d30, #1
|
||||
vcge.u8 d24, d16, d24 ; (a > blimit * 2 + limit) * -1
|
||||
|
||||
vcge.u8 d20, d30, d20 ; flat
|
||||
|
||||
vand d19, d19, d24 ; mask
|
||||
|
||||
; hevmask
|
||||
vcgt.u8 d21, d21, d18 ; (abs(p1 - p0) > thresh)*-1
|
||||
vcgt.u8 d22, d22, d18 ; (abs(q1 - q0) > thresh)*-1
|
||||
vorr d21, d21, d22 ; hev
|
||||
|
||||
vand d16, d20, d19 ; flat && mask
|
||||
vmov r5, r6, d16
|
||||
|
||||
; flatmask5(1, p7, p6, p5, p4, p0, q0, q4, q5, q6, q7)
|
||||
vabd.u8 d22, d3, d7 ; abs(p4 - p0)
|
||||
vabd.u8 d23, d12, d8 ; abs(q4 - q0)
|
||||
vabd.u8 d24, d7, d2 ; abs(p0 - p5)
|
||||
vabd.u8 d25, d8, d13 ; abs(q0 - q5)
|
||||
vabd.u8 d26, d1, d7 ; abs(p6 - p0)
|
||||
vabd.u8 d27, d14, d8 ; abs(q6 - q0)
|
||||
vabd.u8 d28, d0, d7 ; abs(p7 - p0)
|
||||
vabd.u8 d29, d15, d8 ; abs(q7 - q0)
|
||||
|
||||
; only compare the largest value to thresh
|
||||
vmax.u8 d22, d22, d23 ; max(abs(p4 - p0), abs(q4 - q0))
|
||||
vmax.u8 d23, d24, d25 ; max(abs(p0 - p5), abs(q0 - q5))
|
||||
vmax.u8 d24, d26, d27 ; max(abs(p6 - p0), abs(q6 - q0))
|
||||
vmax.u8 d25, d28, d29 ; max(abs(p7 - p0), abs(q7 - q0))
|
||||
|
||||
vmax.u8 d26, d22, d23
|
||||
vmax.u8 d27, d24, d25
|
||||
vmax.u8 d23, d26, d27
|
||||
|
||||
vcge.u8 d18, d30, d23 ; flat2
|
||||
|
||||
vmov.u8 d22, #0x80
|
||||
|
||||
orrs r5, r5, r6 ; Check for 0
|
||||
orreq r7, r7, #1 ; Only do filter branch
|
||||
|
||||
vand d17, d18, d16 ; flat2 && flat && mask
|
||||
vmov r5, r6, d17
|
||||
|
||||
; mbfilter() function
|
||||
|
||||
; filter() function
|
||||
; convert to signed
|
||||
veor d23, d8, d22 ; qs0
|
||||
veor d24, d7, d22 ; ps0
|
||||
veor d25, d6, d22 ; ps1
|
||||
veor d26, d9, d22 ; qs1
|
||||
|
||||
vmov.u8 d27, #3
|
||||
|
||||
vsub.s8 d28, d23, d24 ; ( qs0 - ps0)
|
||||
vqsub.s8 d29, d25, d26 ; filter = clamp(ps1-qs1)
|
||||
vmull.s8 q15, d28, d27 ; 3 * ( qs0 - ps0)
|
||||
vand d29, d29, d21 ; filter &= hev
|
||||
vaddw.s8 q15, q15, d29 ; filter + 3 * (qs0 - ps0)
|
||||
vmov.u8 d29, #4
|
||||
|
||||
; filter = clamp(filter + 3 * ( qs0 - ps0))
|
||||
vqmovn.s16 d28, q15
|
||||
|
||||
vand d28, d28, d19 ; filter &= mask
|
||||
|
||||
vqadd.s8 d30, d28, d27 ; filter2 = clamp(filter+3)
|
||||
vqadd.s8 d29, d28, d29 ; filter1 = clamp(filter+4)
|
||||
vshr.s8 d30, d30, #3 ; filter2 >>= 3
|
||||
vshr.s8 d29, d29, #3 ; filter1 >>= 3
|
||||
|
||||
|
||||
vqadd.s8 d24, d24, d30 ; op0 = clamp(ps0 + filter2)
|
||||
vqsub.s8 d23, d23, d29 ; oq0 = clamp(qs0 - filter1)
|
||||
|
||||
; outer tap adjustments: ++filter1 >> 1
|
||||
vrshr.s8 d29, d29, #1
|
||||
vbic d29, d29, d21 ; filter &= ~hev
|
||||
|
||||
vqadd.s8 d25, d25, d29 ; op1 = clamp(ps1 + filter)
|
||||
vqsub.s8 d26, d26, d29 ; oq1 = clamp(qs1 - filter)
|
||||
|
||||
veor d24, d24, d22 ; *f_op0 = u^0x80
|
||||
veor d23, d23, d22 ; *f_oq0 = u^0x80
|
||||
veor d25, d25, d22 ; *f_op1 = u^0x80
|
||||
veor d26, d26, d22 ; *f_oq1 = u^0x80
|
||||
|
||||
tst r7, #1
|
||||
bxne lr
|
||||
|
||||
orrs r5, r5, r6 ; Check for 0
|
||||
orreq r7, r7, #2 ; Only do mbfilter branch
|
||||
|
||||
; mbfilter flat && mask branch
|
||||
; TODO(fgalligan): Can I decrease the cycles shifting to consective d's
|
||||
; and using vibt on the q's?
|
||||
vmov.u8 d29, #2
|
||||
vaddl.u8 q15, d7, d8 ; op2 = p0 + q0
|
||||
vmlal.u8 q15, d4, d27 ; op2 = p0 + q0 + p3 * 3
|
||||
vmlal.u8 q15, d5, d29 ; op2 = p0 + q0 + p3 * 3 + p2 * 2
|
||||
vaddl.u8 q10, d4, d5
|
||||
vaddw.u8 q15, d6 ; op2=p1 + p0 + q0 + p3 * 3 + p2 *2
|
||||
vaddl.u8 q14, d6, d9
|
||||
vqrshrn.u16 d18, q15, #3 ; r_op2
|
||||
|
||||
vsub.i16 q15, q10
|
||||
vaddl.u8 q10, d4, d6
|
||||
vadd.i16 q15, q14
|
||||
vaddl.u8 q14, d7, d10
|
||||
vqrshrn.u16 d19, q15, #3 ; r_op1
|
||||
|
||||
vsub.i16 q15, q10
|
||||
vadd.i16 q15, q14
|
||||
vaddl.u8 q14, d8, d11
|
||||
vqrshrn.u16 d20, q15, #3 ; r_op0
|
||||
|
||||
vsubw.u8 q15, d4 ; oq0 = op0 - p3
|
||||
vsubw.u8 q15, d7 ; oq0 -= p0
|
||||
vadd.i16 q15, q14
|
||||
vaddl.u8 q14, d9, d11
|
||||
vqrshrn.u16 d21, q15, #3 ; r_oq0
|
||||
|
||||
vsubw.u8 q15, d5 ; oq1 = oq0 - p2
|
||||
vsubw.u8 q15, d8 ; oq1 -= q0
|
||||
vadd.i16 q15, q14
|
||||
vaddl.u8 q14, d10, d11
|
||||
vqrshrn.u16 d22, q15, #3 ; r_oq1
|
||||
|
||||
vsubw.u8 q15, d6 ; oq2 = oq0 - p1
|
||||
vsubw.u8 q15, d9 ; oq2 -= q1
|
||||
vadd.i16 q15, q14
|
||||
vqrshrn.u16 d27, q15, #3 ; r_oq2
|
||||
|
||||
; Filter does not set op2 or oq2, so use p2 and q2.
|
||||
vbif d18, d5, d16 ; t_op2 |= p2 & ~(flat & mask)
|
||||
vbif d19, d25, d16 ; t_op1 |= f_op1 & ~(flat & mask)
|
||||
vbif d20, d24, d16 ; t_op0 |= f_op0 & ~(flat & mask)
|
||||
vbif d21, d23, d16 ; t_oq0 |= f_oq0 & ~(flat & mask)
|
||||
vbif d22, d26, d16 ; t_oq1 |= f_oq1 & ~(flat & mask)
|
||||
|
||||
vbit d23, d27, d16 ; t_oq2 |= r_oq2 & (flat & mask)
|
||||
vbif d23, d10, d16 ; t_oq2 |= q2 & ~(flat & mask)
|
||||
|
||||
tst r7, #2
|
||||
bxne lr
|
||||
|
||||
; wide_mbfilter flat2 && flat && mask branch
|
||||
vmov.u8 d16, #7
|
||||
vaddl.u8 q15, d7, d8 ; op6 = p0 + q0
|
||||
vaddl.u8 q12, d2, d3
|
||||
vaddl.u8 q13, d4, d5
|
||||
vaddl.u8 q14, d1, d6
|
||||
vmlal.u8 q15, d0, d16 ; op6 += p7 * 3
|
||||
vadd.i16 q12, q13
|
||||
vadd.i16 q15, q14
|
||||
vaddl.u8 q14, d2, d9
|
||||
vadd.i16 q15, q12
|
||||
vaddl.u8 q12, d0, d1
|
||||
vaddw.u8 q15, d1
|
||||
vaddl.u8 q13, d0, d2
|
||||
vadd.i16 q14, q15, q14
|
||||
vqrshrn.u16 d16, q15, #4 ; w_op6
|
||||
|
||||
vsub.i16 q15, q14, q12
|
||||
vaddl.u8 q14, d3, d10
|
||||
vqrshrn.u16 d24, q15, #4 ; w_op5
|
||||
|
||||
vsub.i16 q15, q13
|
||||
vaddl.u8 q13, d0, d3
|
||||
vadd.i16 q15, q14
|
||||
vaddl.u8 q14, d4, d11
|
||||
vqrshrn.u16 d25, q15, #4 ; w_op4
|
||||
|
||||
vadd.i16 q15, q14
|
||||
vaddl.u8 q14, d0, d4
|
||||
vsub.i16 q15, q13
|
||||
vsub.i16 q14, q15, q14
|
||||
vqrshrn.u16 d26, q15, #4 ; w_op3
|
||||
|
||||
vaddw.u8 q15, q14, d5 ; op2 += p2
|
||||
vaddl.u8 q14, d0, d5
|
||||
vaddw.u8 q15, d12 ; op2 += q4
|
||||
vbif d26, d4, d17 ; op3 |= p3 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d27, q15, #4 ; w_op2
|
||||
|
||||
vsub.i16 q15, q14
|
||||
vaddl.u8 q14, d0, d6
|
||||
vaddw.u8 q15, d6 ; op1 += p1
|
||||
vaddw.u8 q15, d13 ; op1 += q5
|
||||
vbif d27, d18, d17 ; op2 |= t_op2 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d18, q15, #4 ; w_op1
|
||||
|
||||
vsub.i16 q15, q14
|
||||
vaddl.u8 q14, d0, d7
|
||||
vaddw.u8 q15, d7 ; op0 += p0
|
||||
vaddw.u8 q15, d14 ; op0 += q6
|
||||
vbif d18, d19, d17 ; op1 |= t_op1 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d19, q15, #4 ; w_op0
|
||||
|
||||
vsub.i16 q15, q14
|
||||
vaddl.u8 q14, d1, d8
|
||||
vaddw.u8 q15, d8 ; oq0 += q0
|
||||
vaddw.u8 q15, d15 ; oq0 += q7
|
||||
vbif d19, d20, d17 ; op0 |= t_op0 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d20, q15, #4 ; w_oq0
|
||||
|
||||
vsub.i16 q15, q14
|
||||
vaddl.u8 q14, d2, d9
|
||||
vaddw.u8 q15, d9 ; oq1 += q1
|
||||
vaddl.u8 q4, d10, d15
|
||||
vaddw.u8 q15, d15 ; oq1 += q7
|
||||
vbif d20, d21, d17 ; oq0 |= t_oq0 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d21, q15, #4 ; w_oq1
|
||||
|
||||
vsub.i16 q15, q14
|
||||
vaddl.u8 q14, d3, d10
|
||||
vadd.i16 q15, q4
|
||||
vaddl.u8 q4, d11, d15
|
||||
vbif d21, d22, d17 ; oq1 |= t_oq1 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d22, q15, #4 ; w_oq2
|
||||
|
||||
vsub.i16 q15, q14
|
||||
vaddl.u8 q14, d4, d11
|
||||
vadd.i16 q15, q4
|
||||
vaddl.u8 q4, d12, d15
|
||||
vbif d22, d23, d17 ; oq2 |= t_oq2 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d23, q15, #4 ; w_oq3
|
||||
|
||||
vsub.i16 q15, q14
|
||||
vaddl.u8 q14, d5, d12
|
||||
vadd.i16 q15, q4
|
||||
vaddl.u8 q4, d13, d15
|
||||
vbif d16, d1, d17 ; op6 |= p6 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d1, q15, #4 ; w_oq4
|
||||
|
||||
vsub.i16 q15, q14
|
||||
vaddl.u8 q14, d6, d13
|
||||
vadd.i16 q15, q4
|
||||
vaddl.u8 q4, d14, d15
|
||||
vbif d24, d2, d17 ; op5 |= p5 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d2, q15, #4 ; w_oq5
|
||||
|
||||
vsub.i16 q15, q14
|
||||
vbif d25, d3, d17 ; op4 |= p4 & ~(f2 & f & m)
|
||||
vadd.i16 q15, q4
|
||||
vbif d23, d11, d17 ; oq3 |= q3 & ~(f2 & f & m)
|
||||
vqrshrn.u16 d3, q15, #4 ; w_oq6
|
||||
vbif d1, d12, d17 ; oq4 |= q4 & ~(f2 & f & m)
|
||||
vbif d2, d13, d17 ; oq5 |= q5 & ~(f2 & f & m)
|
||||
vbif d3, d14, d17 ; oq6 |= q6 & ~(f2 & f & m)
|
||||
|
||||
bx lr
|
||||
ENDP ; |vpx_wide_mbfilter_neon|
|
||||
|
||||
END
|
||||
@@ -0,0 +1,549 @@
|
||||
;
|
||||
; Copyright (c) 2013 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
|
||||
EXPORT |vpx_lpf_horizontal_4_neon|
|
||||
EXPORT |vpx_lpf_vertical_4_neon|
|
||||
EXPORT |vpx_lpf_horizontal_4_dual_neon|
|
||||
EXPORT |vpx_lpf_vertical_4_dual_neon|
|
||||
ARM
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
; Currently vpx only works on iterations 8 at a time. The vp8 loop filter
|
||||
; works on 16 iterations at a time.
|
||||
;
|
||||
; void vpx_lpf_horizontal_4_neon(uint8_t *s,
|
||||
; int p /* pitch */,
|
||||
; const uint8_t *blimit,
|
||||
; const uint8_t *limit,
|
||||
; const uint8_t *thresh)
|
||||
;
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p, /* pitch */
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh,
|
||||
|vpx_lpf_horizontal_4_neon| PROC
|
||||
push {lr}
|
||||
|
||||
vld1.8 {d0[]}, [r2] ; duplicate *blimit
|
||||
ldr r2, [sp, #4] ; load thresh
|
||||
add r1, r1, r1 ; double pitch
|
||||
|
||||
vld1.8 {d1[]}, [r3] ; duplicate *limit
|
||||
vld1.8 {d2[]}, [r2] ; duplicate *thresh
|
||||
|
||||
sub r2, r0, r1, lsl #1 ; move src pointer down by 4 lines
|
||||
add r3, r2, r1, lsr #1 ; set to 3 lines down
|
||||
|
||||
vld1.u8 {d3}, [r2@64], r1 ; p3
|
||||
vld1.u8 {d4}, [r3@64], r1 ; p2
|
||||
vld1.u8 {d5}, [r2@64], r1 ; p1
|
||||
vld1.u8 {d6}, [r3@64], r1 ; p0
|
||||
vld1.u8 {d7}, [r2@64], r1 ; q0
|
||||
vld1.u8 {d16}, [r3@64], r1 ; q1
|
||||
vld1.u8 {d17}, [r2@64] ; q2
|
||||
vld1.u8 {d18}, [r3@64] ; q3
|
||||
|
||||
sub r2, r2, r1, lsl #1
|
||||
sub r3, r3, r1, lsl #1
|
||||
|
||||
bl filter4_8
|
||||
|
||||
vst1.u8 {d4}, [r2@64], r1 ; store op1
|
||||
vst1.u8 {d5}, [r3@64], r1 ; store op0
|
||||
vst1.u8 {d6}, [r2@64], r1 ; store oq0
|
||||
vst1.u8 {d7}, [r3@64], r1 ; store oq1
|
||||
|
||||
pop {pc}
|
||||
ENDP ; |vpx_lpf_horizontal_4_neon|
|
||||
|
||||
; Currently vpx only works on iterations 8 at a time. The vp8 loop filter
|
||||
; works on 16 iterations at a time.
|
||||
;
|
||||
; void vpx_lpf_vertical_4_neon(uint8_t *s,
|
||||
; int p /* pitch */,
|
||||
; const uint8_t *blimit,
|
||||
; const uint8_t *limit,
|
||||
; const uint8_t *thresh)
|
||||
;
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p, /* pitch */
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh,
|
||||
|vpx_lpf_vertical_4_neon| PROC
|
||||
push {lr}
|
||||
|
||||
vld1.8 {d0[]}, [r2] ; duplicate *blimit
|
||||
vld1.8 {d1[]}, [r3] ; duplicate *limit
|
||||
|
||||
ldr r3, [sp, #4] ; load thresh
|
||||
sub r2, r0, #4 ; move s pointer down by 4 columns
|
||||
|
||||
vld1.8 {d2[]}, [r3] ; duplicate *thresh
|
||||
|
||||
vld1.u8 {d3}, [r2], r1 ; load s data
|
||||
vld1.u8 {d4}, [r2], r1
|
||||
vld1.u8 {d5}, [r2], r1
|
||||
vld1.u8 {d6}, [r2], r1
|
||||
vld1.u8 {d7}, [r2], r1
|
||||
vld1.u8 {d16}, [r2], r1
|
||||
vld1.u8 {d17}, [r2], r1
|
||||
vld1.u8 {d18}, [r2]
|
||||
|
||||
;transpose to 8x16 matrix
|
||||
vtrn.32 d3, d7
|
||||
vtrn.32 d4, d16
|
||||
vtrn.32 d5, d17
|
||||
vtrn.32 d6, d18
|
||||
|
||||
vtrn.16 d3, d5
|
||||
vtrn.16 d4, d6
|
||||
vtrn.16 d7, d17
|
||||
vtrn.16 d16, d18
|
||||
|
||||
vtrn.8 d3, d4
|
||||
vtrn.8 d5, d6
|
||||
vtrn.8 d7, d16
|
||||
vtrn.8 d17, d18
|
||||
|
||||
bl filter4_8
|
||||
|
||||
sub r0, r0, #2
|
||||
|
||||
;store op1, op0, oq0, oq1
|
||||
vst4.8 {d4[0], d5[0], d6[0], d7[0]}, [r0], r1
|
||||
vst4.8 {d4[1], d5[1], d6[1], d7[1]}, [r0], r1
|
||||
vst4.8 {d4[2], d5[2], d6[2], d7[2]}, [r0], r1
|
||||
vst4.8 {d4[3], d5[3], d6[3], d7[3]}, [r0], r1
|
||||
vst4.8 {d4[4], d5[4], d6[4], d7[4]}, [r0], r1
|
||||
vst4.8 {d4[5], d5[5], d6[5], d7[5]}, [r0], r1
|
||||
vst4.8 {d4[6], d5[6], d6[6], d7[6]}, [r0], r1
|
||||
vst4.8 {d4[7], d5[7], d6[7], d7[7]}, [r0]
|
||||
|
||||
pop {pc}
|
||||
ENDP ; |vpx_lpf_vertical_4_neon|
|
||||
|
||||
; void filter4_8();
|
||||
; This is a helper function for the loopfilters. The invidual functions do the
|
||||
; necessary load, transpose (if necessary) and store. The function does not use
|
||||
; registers d8-d15.
|
||||
;
|
||||
; Inputs:
|
||||
; r0-r3, r12 PRESERVE
|
||||
; d0 blimit
|
||||
; d1 limit
|
||||
; d2 thresh
|
||||
; d3 p3
|
||||
; d4 p2
|
||||
; d5 p1
|
||||
; d6 p0
|
||||
; d7 q0
|
||||
; d16 q1
|
||||
; d17 q2
|
||||
; d18 q3
|
||||
;
|
||||
; Outputs:
|
||||
; d4 op1
|
||||
; d5 op0
|
||||
; d6 oq0
|
||||
; d7 oq1
|
||||
|filter4_8| PROC
|
||||
; filter_mask
|
||||
vabd.u8 d19, d3, d4 ; m1 = abs(p3 - p2)
|
||||
vabd.u8 d20, d4, d5 ; m2 = abs(p2 - p1)
|
||||
vabd.u8 d21, d5, d6 ; m3 = abs(p1 - p0)
|
||||
vabd.u8 d22, d16, d7 ; m4 = abs(q1 - q0)
|
||||
vabd.u8 d3, d17, d16 ; m5 = abs(q2 - q1)
|
||||
vabd.u8 d4, d18, d17 ; m6 = abs(q3 - q2)
|
||||
|
||||
; only compare the largest value to limit
|
||||
vmax.u8 d19, d19, d20 ; m1 = max(m1, m2)
|
||||
vmax.u8 d20, d21, d22 ; m2 = max(m3, m4)
|
||||
|
||||
vabd.u8 d17, d6, d7 ; abs(p0 - q0)
|
||||
|
||||
vmax.u8 d3, d3, d4 ; m3 = max(m5, m6)
|
||||
|
||||
vmov.u8 d18, #0x80
|
||||
|
||||
vmax.u8 d23, d19, d20 ; m1 = max(m1, m2)
|
||||
|
||||
; hevmask
|
||||
vcgt.u8 d21, d21, d2 ; (abs(p1 - p0) > thresh)*-1
|
||||
vcgt.u8 d22, d22, d2 ; (abs(q1 - q0) > thresh)*-1
|
||||
vmax.u8 d23, d23, d3 ; m1 = max(m1, m3)
|
||||
|
||||
vabd.u8 d28, d5, d16 ; a = abs(p1 - q1)
|
||||
vqadd.u8 d17, d17, d17 ; b = abs(p0 - q0) * 2
|
||||
|
||||
veor d7, d7, d18 ; qs0
|
||||
|
||||
vcge.u8 d23, d1, d23 ; abs(m1) > limit
|
||||
|
||||
; filter() function
|
||||
; convert to signed
|
||||
|
||||
vshr.u8 d28, d28, #1 ; a = a / 2
|
||||
veor d6, d6, d18 ; ps0
|
||||
|
||||
veor d5, d5, d18 ; ps1
|
||||
vqadd.u8 d17, d17, d28 ; a = b + a
|
||||
|
||||
veor d16, d16, d18 ; qs1
|
||||
|
||||
vmov.u8 d19, #3
|
||||
|
||||
vsub.s8 d28, d7, d6 ; ( qs0 - ps0)
|
||||
|
||||
vcge.u8 d17, d0, d17 ; a > blimit
|
||||
|
||||
vqsub.s8 d27, d5, d16 ; filter = clamp(ps1-qs1)
|
||||
vorr d22, d21, d22 ; hevmask
|
||||
|
||||
vmull.s8 q12, d28, d19 ; 3 * ( qs0 - ps0)
|
||||
|
||||
vand d27, d27, d22 ; filter &= hev
|
||||
vand d23, d23, d17 ; filter_mask
|
||||
|
||||
vaddw.s8 q12, q12, d27 ; filter + 3 * (qs0 - ps0)
|
||||
|
||||
vmov.u8 d17, #4
|
||||
|
||||
; filter = clamp(filter + 3 * ( qs0 - ps0))
|
||||
vqmovn.s16 d27, q12
|
||||
|
||||
vand d27, d27, d23 ; filter &= mask
|
||||
|
||||
vqadd.s8 d28, d27, d19 ; filter2 = clamp(filter+3)
|
||||
vqadd.s8 d27, d27, d17 ; filter1 = clamp(filter+4)
|
||||
vshr.s8 d28, d28, #3 ; filter2 >>= 3
|
||||
vshr.s8 d27, d27, #3 ; filter1 >>= 3
|
||||
|
||||
vqadd.s8 d19, d6, d28 ; u = clamp(ps0 + filter2)
|
||||
vqsub.s8 d26, d7, d27 ; u = clamp(qs0 - filter1)
|
||||
|
||||
; outer tap adjustments
|
||||
vrshr.s8 d27, d27, #1 ; filter = ++filter1 >> 1
|
||||
|
||||
veor d6, d26, d18 ; *oq0 = u^0x80
|
||||
|
||||
vbic d27, d27, d22 ; filter &= ~hev
|
||||
|
||||
vqadd.s8 d21, d5, d27 ; u = clamp(ps1 + filter)
|
||||
vqsub.s8 d20, d16, d27 ; u = clamp(qs1 - filter)
|
||||
|
||||
veor d5, d19, d18 ; *op0 = u^0x80
|
||||
veor d4, d21, d18 ; *op1 = u^0x80
|
||||
veor d7, d20, d18 ; *oq1 = u^0x80
|
||||
|
||||
bx lr
|
||||
ENDP ; |filter4_8|
|
||||
|
||||
;void vpx_lpf_horizontal_4_dual_neon(uint8_t *s, int p,
|
||||
; const uint8_t *blimit0,
|
||||
; const uint8_t *limit0,
|
||||
; const uint8_t *thresh0,
|
||||
; const uint8_t *blimit1,
|
||||
; const uint8_t *limit1,
|
||||
; const uint8_t *thresh1)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p,
|
||||
; r2 const uint8_t *blimit0,
|
||||
; r3 const uint8_t *limit0,
|
||||
; sp const uint8_t *thresh0,
|
||||
; sp+4 const uint8_t *blimit1,
|
||||
; sp+8 const uint8_t *limit1,
|
||||
; sp+12 const uint8_t *thresh1,
|
||||
|
||||
|vpx_lpf_horizontal_4_dual_neon| PROC
|
||||
push {lr}
|
||||
|
||||
ldr r12, [sp, #4] ; load thresh0
|
||||
vld1.8 {d0}, [r2] ; load blimit0 to first half q
|
||||
vld1.8 {d2}, [r3] ; load limit0 to first half q
|
||||
|
||||
add r1, r1, r1 ; double pitch
|
||||
ldr r2, [sp, #8] ; load blimit1
|
||||
|
||||
vld1.8 {d4}, [r12] ; load thresh0 to first half q
|
||||
|
||||
ldr r3, [sp, #12] ; load limit1
|
||||
ldr r12, [sp, #16] ; load thresh1
|
||||
vld1.8 {d1}, [r2] ; load blimit1 to 2nd half q
|
||||
|
||||
sub r2, r0, r1, lsl #1 ; s[-4 * p]
|
||||
|
||||
vld1.8 {d3}, [r3] ; load limit1 to 2nd half q
|
||||
vld1.8 {d5}, [r12] ; load thresh1 to 2nd half q
|
||||
|
||||
vpush {d8-d15} ; save neon registers
|
||||
|
||||
add r3, r2, r1, lsr #1 ; s[-3 * p]
|
||||
|
||||
vld1.u8 {q3}, [r2@64], r1 ; p3
|
||||
vld1.u8 {q4}, [r3@64], r1 ; p2
|
||||
vld1.u8 {q5}, [r2@64], r1 ; p1
|
||||
vld1.u8 {q6}, [r3@64], r1 ; p0
|
||||
vld1.u8 {q7}, [r2@64], r1 ; q0
|
||||
vld1.u8 {q8}, [r3@64], r1 ; q1
|
||||
vld1.u8 {q9}, [r2@64] ; q2
|
||||
vld1.u8 {q10}, [r3@64] ; q3
|
||||
|
||||
sub r2, r2, r1, lsl #1
|
||||
sub r3, r3, r1, lsl #1
|
||||
|
||||
bl filter4_16
|
||||
|
||||
vst1.u8 {q5}, [r2@64], r1 ; store op1
|
||||
vst1.u8 {q6}, [r3@64], r1 ; store op0
|
||||
vst1.u8 {q7}, [r2@64], r1 ; store oq0
|
||||
vst1.u8 {q8}, [r3@64], r1 ; store oq1
|
||||
|
||||
vpop {d8-d15} ; restore neon registers
|
||||
|
||||
pop {pc}
|
||||
ENDP ; |vpx_lpf_horizontal_4_dual_neon|
|
||||
|
||||
;void vpx_lpf_vertical_4_dual_neon(uint8_t *s, int p,
|
||||
; const uint8_t *blimit0,
|
||||
; const uint8_t *limit0,
|
||||
; const uint8_t *thresh0,
|
||||
; const uint8_t *blimit1,
|
||||
; const uint8_t *limit1,
|
||||
; const uint8_t *thresh1)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p,
|
||||
; r2 const uint8_t *blimit0,
|
||||
; r3 const uint8_t *limit0,
|
||||
; sp const uint8_t *thresh0,
|
||||
; sp+4 const uint8_t *blimit1,
|
||||
; sp+8 const uint8_t *limit1,
|
||||
; sp+12 const uint8_t *thresh1,
|
||||
|
||||
|vpx_lpf_vertical_4_dual_neon| PROC
|
||||
push {lr}
|
||||
|
||||
ldr r12, [sp, #4] ; load thresh0
|
||||
vld1.8 {d0}, [r2] ; load blimit0 to first half q
|
||||
vld1.8 {d2}, [r3] ; load limit0 to first half q
|
||||
|
||||
ldr r2, [sp, #8] ; load blimit1
|
||||
|
||||
vld1.8 {d4}, [r12] ; load thresh0 to first half q
|
||||
|
||||
ldr r3, [sp, #12] ; load limit1
|
||||
ldr r12, [sp, #16] ; load thresh1
|
||||
vld1.8 {d1}, [r2] ; load blimit1 to 2nd half q
|
||||
|
||||
sub r2, r0, #4 ; s[-4]
|
||||
|
||||
vld1.8 {d3}, [r3] ; load limit1 to 2nd half q
|
||||
vld1.8 {d5}, [r12] ; load thresh1 to 2nd half q
|
||||
|
||||
vpush {d8-d15} ; save neon registers
|
||||
|
||||
vld1.u8 {d6}, [r2], r1 ; 00 01 02 03 04 05 06 07
|
||||
vld1.u8 {d8}, [r2], r1 ; 10 11 12 13 14 15 16 17
|
||||
vld1.u8 {d10}, [r2], r1 ; 20 21 22 23 24 25 26 27
|
||||
vld1.u8 {d12}, [r2], r1 ; 30 31 32 33 34 35 36 37
|
||||
vld1.u8 {d14}, [r2], r1 ; 40 41 42 43 44 45 46 47
|
||||
vld1.u8 {d16}, [r2], r1 ; 50 51 52 53 54 55 56 57
|
||||
vld1.u8 {d18}, [r2], r1 ; 60 61 62 63 64 65 66 67
|
||||
vld1.u8 {d20}, [r2], r1 ; 70 71 72 73 74 75 76 77
|
||||
vld1.u8 {d7}, [r2], r1 ; 80 81 82 83 84 85 86 87
|
||||
vld1.u8 {d9}, [r2], r1 ; 90 91 92 93 94 95 96 97
|
||||
vld1.u8 {d11}, [r2], r1 ; A0 A1 A2 A3 A4 A5 A6 A7
|
||||
vld1.u8 {d13}, [r2], r1 ; B0 B1 B2 B3 B4 B5 B6 B7
|
||||
vld1.u8 {d15}, [r2], r1 ; C0 C1 C2 C3 C4 C5 C6 C7
|
||||
vld1.u8 {d17}, [r2], r1 ; D0 D1 D2 D3 D4 D5 D6 D7
|
||||
vld1.u8 {d19}, [r2], r1 ; E0 E1 E2 E3 E4 E5 E6 E7
|
||||
vld1.u8 {d21}, [r2] ; F0 F1 F2 F3 F4 F5 F6 F7
|
||||
|
||||
vtrn.8 q3, q4 ; q3 : 00 10 02 12 04 14 06 16 80 90 82 92 84 94 86 96
|
||||
; q4 : 01 11 03 13 05 15 07 17 81 91 83 93 85 95 87 97
|
||||
vtrn.8 q5, q6 ; q5 : 20 30 22 32 24 34 26 36 A0 B0 A2 B2 A4 B4 A6 B6
|
||||
; q6 : 21 31 23 33 25 35 27 37 A1 B1 A3 B3 A5 B5 A7 B7
|
||||
vtrn.8 q7, q8 ; q7 : 40 50 42 52 44 54 46 56 C0 D0 C2 D2 C4 D4 C6 D6
|
||||
; q8 : 41 51 43 53 45 55 47 57 C1 D1 C3 D3 C5 D5 C7 D7
|
||||
vtrn.8 q9, q10 ; q9 : 60 70 62 72 64 74 66 76 E0 F0 E2 F2 E4 F4 E6 F6
|
||||
; q10: 61 71 63 73 65 75 67 77 E1 F1 E3 F3 E5 F5 E7 F7
|
||||
|
||||
vtrn.16 q3, q5 ; q3 : 00 10 20 30 04 14 24 34 80 90 A0 B0 84 94 A4 B4
|
||||
; q5 : 02 12 22 32 06 16 26 36 82 92 A2 B2 86 96 A6 B6
|
||||
vtrn.16 q4, q6 ; q4 : 01 11 21 31 05 15 25 35 81 91 A1 B1 85 95 A5 B5
|
||||
; q6 : 03 13 23 33 07 17 27 37 83 93 A3 B3 87 97 A7 B7
|
||||
vtrn.16 q7, q9 ; q7 : 40 50 60 70 44 54 64 74 C0 D0 E0 F0 C4 D4 E4 F4
|
||||
; q9 : 42 52 62 72 46 56 66 76 C2 D2 E2 F2 C6 D6 E6 F6
|
||||
vtrn.16 q8, q10 ; q8 : 41 51 61 71 45 55 65 75 C1 D1 E1 F1 C5 D5 E5 F5
|
||||
; q10: 43 53 63 73 47 57 67 77 C3 D3 E3 F3 C7 D7 E7 F7
|
||||
|
||||
vtrn.32 q3, q7 ; q3 : 00 10 20 30 40 50 60 70 80 90 A0 B0 C0 D0 E0 F0
|
||||
; q7 : 04 14 24 34 44 54 64 74 84 94 A4 B4 C4 D4 E4 F4
|
||||
vtrn.32 q5, q9 ; q5 : 02 12 22 32 42 52 62 72 82 92 A2 B2 C2 D2 E2 F2
|
||||
; q9 : 06 16 26 36 46 56 66 76 86 96 A6 B6 C6 D6 E6 F6
|
||||
vtrn.32 q4, q8 ; q4 : 01 11 21 31 41 51 61 71 81 91 A1 B1 C1 D1 E1 F1
|
||||
; q8 : 05 15 25 35 45 55 65 75 85 95 A5 B5 C5 D5 E5 F5
|
||||
vtrn.32 q6, q10 ; q6 : 03 13 23 33 43 53 63 73 83 93 A3 B3 C3 D3 E3 F3
|
||||
; q10: 07 17 27 37 47 57 67 77 87 97 A7 B7 C7 D7 E7 F7
|
||||
|
||||
bl filter4_16
|
||||
|
||||
sub r0, #2
|
||||
|
||||
vmov d0, d11
|
||||
vmov d1, d13
|
||||
vmov d2, d15
|
||||
vmov d3, d17
|
||||
vmov d11, d12
|
||||
vmov d12, d14
|
||||
vmov d13, d16
|
||||
vst4.8 {d10[0], d11[0], d12[0], d13[0]}, [r0], r1
|
||||
vst4.8 {d10[1], d11[1], d12[1], d13[1]}, [r0], r1
|
||||
vst4.8 {d10[2], d11[2], d12[2], d13[2]}, [r0], r1
|
||||
vst4.8 {d10[3], d11[3], d12[3], d13[3]}, [r0], r1
|
||||
vst4.8 {d10[4], d11[4], d12[4], d13[4]}, [r0], r1
|
||||
vst4.8 {d10[5], d11[5], d12[5], d13[5]}, [r0], r1
|
||||
vst4.8 {d10[6], d11[6], d12[6], d13[6]}, [r0], r1
|
||||
vst4.8 {d10[7], d11[7], d12[7], d13[7]}, [r0], r1
|
||||
vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0], r1
|
||||
vst4.8 {d0[1], d1[1], d2[1], d3[1]}, [r0], r1
|
||||
vst4.8 {d0[2], d1[2], d2[2], d3[2]}, [r0], r1
|
||||
vst4.8 {d0[3], d1[3], d2[3], d3[3]}, [r0], r1
|
||||
vst4.8 {d0[4], d1[4], d2[4], d3[4]}, [r0], r1
|
||||
vst4.8 {d0[5], d1[5], d2[5], d3[5]}, [r0], r1
|
||||
vst4.8 {d0[6], d1[6], d2[6], d3[6]}, [r0], r1
|
||||
vst4.8 {d0[7], d1[7], d2[7], d3[7]}, [r0]
|
||||
|
||||
vpop {d8-d15} ; restore neon registers
|
||||
|
||||
pop {pc}
|
||||
ENDP ; |vpx_lpf_vertical_4_dual_neon|
|
||||
|
||||
; void filter4_16();
|
||||
; This is a helper function for the loopfilters. The invidual functions do the
|
||||
; necessary load, transpose (if necessary) and store. This function uses
|
||||
; registers d8-d15, so the calling function must save those registers.
|
||||
;
|
||||
; r0-r3, r12 PRESERVE
|
||||
; q0 blimit
|
||||
; q1 limit
|
||||
; q2 thresh
|
||||
; q3 p3
|
||||
; q4 p2
|
||||
; q5 p1
|
||||
; q6 p0
|
||||
; q7 q0
|
||||
; q8 q1
|
||||
; q9 q2
|
||||
; q10 q3
|
||||
;
|
||||
; Outputs:
|
||||
; q5 op1
|
||||
; q6 op0
|
||||
; q7 oq0
|
||||
; q8 oq1
|
||||
|filter4_16| PROC
|
||||
|
||||
; filter_mask
|
||||
vabd.u8 q11, q3, q4 ; m1 = abs(p3 - p2)
|
||||
vabd.u8 q12, q4, q5 ; m2 = abs(p2 - p1)
|
||||
vabd.u8 q13, q5, q6 ; m3 = abs(p1 - p0)
|
||||
vabd.u8 q14, q8, q7 ; m4 = abs(q1 - q0)
|
||||
vabd.u8 q3, q9, q8 ; m5 = abs(q2 - q1)
|
||||
vabd.u8 q4, q10, q9 ; m6 = abs(q3 - q2)
|
||||
|
||||
; only compare the largest value to limit
|
||||
vmax.u8 q11, q11, q12 ; m7 = max(m1, m2)
|
||||
vmax.u8 q12, q13, q14 ; m8 = max(m3, m4)
|
||||
|
||||
vabd.u8 q9, q6, q7 ; abs(p0 - q0)
|
||||
|
||||
vmax.u8 q3, q3, q4 ; m9 = max(m5, m6)
|
||||
|
||||
vmov.u8 q10, #0x80
|
||||
|
||||
vmax.u8 q15, q11, q12 ; m10 = max(m7, m8)
|
||||
|
||||
vcgt.u8 q13, q13, q2 ; (abs(p1 - p0) > thresh)*-1
|
||||
vcgt.u8 q14, q14, q2 ; (abs(q1 - q0) > thresh)*-1
|
||||
vmax.u8 q15, q15, q3 ; m11 = max(m10, m9)
|
||||
|
||||
vabd.u8 q2, q5, q8 ; a = abs(p1 - q1)
|
||||
vqadd.u8 q9, q9, q9 ; b = abs(p0 - q0) * 2
|
||||
|
||||
veor q7, q7, q10 ; qs0
|
||||
|
||||
vcge.u8 q15, q1, q15 ; abs(m11) > limit
|
||||
|
||||
vshr.u8 q2, q2, #1 ; a = a / 2
|
||||
veor q6, q6, q10 ; ps0
|
||||
|
||||
veor q5, q5, q10 ; ps1
|
||||
vqadd.u8 q9, q9, q2 ; a = b + a
|
||||
|
||||
veor q8, q8, q10 ; qs1
|
||||
|
||||
vmov.u16 q4, #3
|
||||
|
||||
vsubl.s8 q2, d14, d12 ; ( qs0 - ps0)
|
||||
vsubl.s8 q11, d15, d13
|
||||
|
||||
vcge.u8 q9, q0, q9 ; a > blimit
|
||||
|
||||
vqsub.s8 q1, q5, q8 ; filter = clamp(ps1-qs1)
|
||||
vorr q14, q13, q14 ; hev
|
||||
|
||||
vmul.i16 q2, q2, q4 ; 3 * ( qs0 - ps0)
|
||||
vmul.i16 q11, q11, q4
|
||||
|
||||
vand q1, q1, q14 ; filter &= hev
|
||||
vand q15, q15, q9 ; mask
|
||||
|
||||
vmov.u8 q4, #3
|
||||
|
||||
vaddw.s8 q2, q2, d2 ; filter + 3 * (qs0 - ps0)
|
||||
vaddw.s8 q11, q11, d3
|
||||
|
||||
vmov.u8 q9, #4
|
||||
|
||||
; filter = clamp(filter + 3 * ( qs0 - ps0))
|
||||
vqmovn.s16 d2, q2
|
||||
vqmovn.s16 d3, q11
|
||||
vand q1, q1, q15 ; filter &= mask
|
||||
|
||||
vqadd.s8 q2, q1, q4 ; filter2 = clamp(filter+3)
|
||||
vqadd.s8 q1, q1, q9 ; filter1 = clamp(filter+4)
|
||||
vshr.s8 q2, q2, #3 ; filter2 >>= 3
|
||||
vshr.s8 q1, q1, #3 ; filter1 >>= 3
|
||||
|
||||
|
||||
vqadd.s8 q11, q6, q2 ; u = clamp(ps0 + filter2)
|
||||
vqsub.s8 q0, q7, q1 ; u = clamp(qs0 - filter1)
|
||||
|
||||
; outer tap adjustments
|
||||
vrshr.s8 q1, q1, #1 ; filter = ++filter1 >> 1
|
||||
|
||||
veor q7, q0, q10 ; *oq0 = u^0x80
|
||||
|
||||
vbic q1, q1, q14 ; filter &= ~hev
|
||||
|
||||
vqadd.s8 q13, q5, q1 ; u = clamp(ps1 + filter)
|
||||
vqsub.s8 q12, q8, q1 ; u = clamp(qs1 - filter)
|
||||
|
||||
veor q6, q11, q10 ; *op0 = u^0x80
|
||||
veor q5, q13, q10 ; *op1 = u^0x80
|
||||
veor q8, q12, q10 ; *oq1 = u^0x80
|
||||
|
||||
bx lr
|
||||
ENDP ; |filter4_16|
|
||||
|
||||
END
|
||||
@@ -0,0 +1,491 @@
|
||||
;
|
||||
; Copyright (c) 2013 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
|
||||
EXPORT |vpx_lpf_horizontal_8_neon|
|
||||
EXPORT |vpx_lpf_horizontal_8_dual_neon|
|
||||
EXPORT |vpx_lpf_vertical_8_neon|
|
||||
EXPORT |vpx_lpf_vertical_8_dual_neon|
|
||||
ARM
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
; Currently vpx only works on iterations 8 at a time. The vp8 loop filter
|
||||
; works on 16 iterations at a time.
|
||||
;
|
||||
; void vpx_lpf_horizontal_8_neon(uint8_t *s, int p,
|
||||
; const uint8_t *blimit,
|
||||
; const uint8_t *limit,
|
||||
; const uint8_t *thresh)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p, /* pitch */
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh,
|
||||
|vpx_lpf_horizontal_8_neon| PROC
|
||||
push {r4-r5, lr}
|
||||
|
||||
vld1.8 {d0[]}, [r2] ; duplicate *blimit
|
||||
ldr r2, [sp, #12] ; load thresh
|
||||
add r1, r1, r1 ; double pitch
|
||||
|
||||
vld1.8 {d1[]}, [r3] ; duplicate *limit
|
||||
vld1.8 {d2[]}, [r2] ; duplicate *thresh
|
||||
|
||||
sub r3, r0, r1, lsl #1 ; move src pointer down by 4 lines
|
||||
add r2, r3, r1, lsr #1 ; set to 3 lines down
|
||||
|
||||
vld1.u8 {d3}, [r3@64], r1 ; p3
|
||||
vld1.u8 {d4}, [r2@64], r1 ; p2
|
||||
vld1.u8 {d5}, [r3@64], r1 ; p1
|
||||
vld1.u8 {d6}, [r2@64], r1 ; p0
|
||||
vld1.u8 {d7}, [r3@64], r1 ; q0
|
||||
vld1.u8 {d16}, [r2@64], r1 ; q1
|
||||
vld1.u8 {d17}, [r3@64] ; q2
|
||||
vld1.u8 {d18}, [r2@64], r1 ; q3
|
||||
|
||||
sub r3, r3, r1, lsl #1
|
||||
sub r2, r2, r1, lsl #2
|
||||
|
||||
bl vpx_mbloop_filter_neon
|
||||
|
||||
vst1.u8 {d0}, [r2@64], r1 ; store op2
|
||||
vst1.u8 {d1}, [r3@64], r1 ; store op1
|
||||
vst1.u8 {d2}, [r2@64], r1 ; store op0
|
||||
vst1.u8 {d3}, [r3@64], r1 ; store oq0
|
||||
vst1.u8 {d4}, [r2@64], r1 ; store oq1
|
||||
vst1.u8 {d5}, [r3@64], r1 ; store oq2
|
||||
|
||||
pop {r4-r5, pc}
|
||||
|
||||
ENDP ; |vpx_lpf_horizontal_8_neon|
|
||||
|
||||
;void vpx_lpf_horizontal_8_dual_neon(uint8_t *s,
|
||||
; int p,
|
||||
; const uint8_t *blimit0,
|
||||
; const uint8_t *limit0,
|
||||
; const uint8_t *thresh0,
|
||||
; const uint8_t *blimit1,
|
||||
; const uint8_t *limit1,
|
||||
; const uint8_t *thresh1)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int p, /* pitch */
|
||||
; r2 const uint8_t *blimit0,
|
||||
; r3 const uint8_t *limit0,
|
||||
; sp const uint8_t *thresh0,
|
||||
; sp + 4 const uint8_t *blimit1,
|
||||
; sp + 8 const uint8_t *limit1,
|
||||
; sp + 12 const uint8_t *thresh1,
|
||||
|vpx_lpf_horizontal_8_dual_neon| PROC
|
||||
push {r0-r1, lr}
|
||||
ldr lr, [sp, #12]
|
||||
push {lr} ; thresh0
|
||||
bl vpx_lpf_horizontal_8_neon
|
||||
|
||||
ldr r2, [sp, #20] ; blimit1
|
||||
ldr r3, [sp, #24] ; limit1
|
||||
ldr lr, [sp, #28]
|
||||
str lr, [sp, #16] ; thresh1
|
||||
add sp, #4
|
||||
pop {r0-r1, lr}
|
||||
add r0, #8 ; s + 8
|
||||
b vpx_lpf_horizontal_8_neon
|
||||
ENDP ; |vpx_lpf_horizontal_8_dual_neon|
|
||||
|
||||
; void vpx_lpf_vertical_8_neon(uint8_t *s,
|
||||
; int pitch,
|
||||
; const uint8_t *blimit,
|
||||
; const uint8_t *limit,
|
||||
; const uint8_t *thresh)
|
||||
;
|
||||
; r0 uint8_t *s,
|
||||
; r1 int pitch,
|
||||
; r2 const uint8_t *blimit,
|
||||
; r3 const uint8_t *limit,
|
||||
; sp const uint8_t *thresh,
|
||||
|vpx_lpf_vertical_8_neon| PROC
|
||||
push {r4-r5, lr}
|
||||
|
||||
vld1.8 {d0[]}, [r2] ; duplicate *blimit
|
||||
vld1.8 {d1[]}, [r3] ; duplicate *limit
|
||||
|
||||
ldr r3, [sp, #12] ; load thresh
|
||||
sub r2, r0, #4 ; move s pointer down by 4 columns
|
||||
|
||||
vld1.8 {d2[]}, [r3] ; duplicate *thresh
|
||||
|
||||
vld1.u8 {d3}, [r2], r1 ; load s data
|
||||
vld1.u8 {d4}, [r2], r1
|
||||
vld1.u8 {d5}, [r2], r1
|
||||
vld1.u8 {d6}, [r2], r1
|
||||
vld1.u8 {d7}, [r2], r1
|
||||
vld1.u8 {d16}, [r2], r1
|
||||
vld1.u8 {d17}, [r2], r1
|
||||
vld1.u8 {d18}, [r2]
|
||||
|
||||
;transpose to 8x16 matrix
|
||||
vtrn.32 d3, d7
|
||||
vtrn.32 d4, d16
|
||||
vtrn.32 d5, d17
|
||||
vtrn.32 d6, d18
|
||||
|
||||
vtrn.16 d3, d5
|
||||
vtrn.16 d4, d6
|
||||
vtrn.16 d7, d17
|
||||
vtrn.16 d16, d18
|
||||
|
||||
vtrn.8 d3, d4
|
||||
vtrn.8 d5, d6
|
||||
vtrn.8 d7, d16
|
||||
vtrn.8 d17, d18
|
||||
|
||||
sub r2, r0, #3
|
||||
add r3, r0, #1
|
||||
|
||||
bl vpx_mbloop_filter_neon
|
||||
|
||||
;store op2, op1, op0, oq0
|
||||
vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r2], r1
|
||||
vst4.8 {d0[1], d1[1], d2[1], d3[1]}, [r2], r1
|
||||
vst4.8 {d0[2], d1[2], d2[2], d3[2]}, [r2], r1
|
||||
vst4.8 {d0[3], d1[3], d2[3], d3[3]}, [r2], r1
|
||||
vst4.8 {d0[4], d1[4], d2[4], d3[4]}, [r2], r1
|
||||
vst4.8 {d0[5], d1[5], d2[5], d3[5]}, [r2], r1
|
||||
vst4.8 {d0[6], d1[6], d2[6], d3[6]}, [r2], r1
|
||||
vst4.8 {d0[7], d1[7], d2[7], d3[7]}, [r2]
|
||||
|
||||
;store oq1, oq2
|
||||
vst2.8 {d4[0], d5[0]}, [r3], r1
|
||||
vst2.8 {d4[1], d5[1]}, [r3], r1
|
||||
vst2.8 {d4[2], d5[2]}, [r3], r1
|
||||
vst2.8 {d4[3], d5[3]}, [r3], r1
|
||||
vst2.8 {d4[4], d5[4]}, [r3], r1
|
||||
vst2.8 {d4[5], d5[5]}, [r3], r1
|
||||
vst2.8 {d4[6], d5[6]}, [r3], r1
|
||||
vst2.8 {d4[7], d5[7]}, [r3]
|
||||
|
||||
pop {r4-r5, pc}
|
||||
ENDP ; |vpx_lpf_vertical_8_neon|
|
||||
|
||||
;void vpx_lpf_vertical_8_dual_neon(uint8_t *s,
|
||||
; int pitch,
|
||||
; const uint8_t *blimit0,
|
||||
; const uint8_t *limit0,
|
||||
; const uint8_t *thresh0,
|
||||
; const uint8_t *blimit1,
|
||||
; const uint8_t *limit1,
|
||||
; const uint8_t *thresh1)
|
||||
; r0 uint8_t *s,
|
||||
; r1 int pitch
|
||||
; r2 const uint8_t *blimit0,
|
||||
; r3 const uint8_t *limit0,
|
||||
; sp const uint8_t *thresh0,
|
||||
; sp + 4 const uint8_t *blimit1,
|
||||
; sp + 8 const uint8_t *limit1,
|
||||
; sp + 12 const uint8_t *thresh1,
|
||||
|vpx_lpf_vertical_8_dual_neon| PROC
|
||||
push {r0-r1, lr}
|
||||
ldr lr, [sp, #12]
|
||||
push {lr} ; thresh0
|
||||
bl vpx_lpf_vertical_8_neon
|
||||
|
||||
ldr r2, [sp, #20] ; blimit1
|
||||
ldr r3, [sp, #24] ; limit1
|
||||
ldr lr, [sp, #28]
|
||||
str lr, [sp, #16] ; thresh1
|
||||
add sp, #4
|
||||
pop {r0-r1, lr}
|
||||
add r0, r0, r1, lsl #3 ; s + 8 * pitch
|
||||
b vpx_lpf_vertical_8_neon
|
||||
ENDP ; |vpx_lpf_vertical_8_dual_neon|
|
||||
|
||||
; void vpx_mbloop_filter_neon();
|
||||
; This is a helper function for the loopfilters. The invidual functions do the
|
||||
; necessary load, transpose (if necessary) and store. The function does not use
|
||||
; registers d8-d15.
|
||||
;
|
||||
; Inputs:
|
||||
; r0-r3, r12 PRESERVE
|
||||
; d0 blimit
|
||||
; d1 limit
|
||||
; d2 thresh
|
||||
; d3 p3
|
||||
; d4 p2
|
||||
; d5 p1
|
||||
; d6 p0
|
||||
; d7 q0
|
||||
; d16 q1
|
||||
; d17 q2
|
||||
; d18 q3
|
||||
;
|
||||
; Outputs:
|
||||
; d0 op2
|
||||
; d1 op1
|
||||
; d2 op0
|
||||
; d3 oq0
|
||||
; d4 oq1
|
||||
; d5 oq2
|
||||
|vpx_mbloop_filter_neon| PROC
|
||||
; filter_mask
|
||||
vabd.u8 d19, d3, d4 ; m1 = abs(p3 - p2)
|
||||
vabd.u8 d20, d4, d5 ; m2 = abs(p2 - p1)
|
||||
vabd.u8 d21, d5, d6 ; m3 = abs(p1 - p0)
|
||||
vabd.u8 d22, d16, d7 ; m4 = abs(q1 - q0)
|
||||
vabd.u8 d23, d17, d16 ; m5 = abs(q2 - q1)
|
||||
vabd.u8 d24, d18, d17 ; m6 = abs(q3 - q2)
|
||||
|
||||
; only compare the largest value to limit
|
||||
vmax.u8 d19, d19, d20 ; m1 = max(m1, m2)
|
||||
vmax.u8 d20, d21, d22 ; m2 = max(m3, m4)
|
||||
|
||||
vabd.u8 d25, d6, d4 ; m7 = abs(p0 - p2)
|
||||
|
||||
vmax.u8 d23, d23, d24 ; m3 = max(m5, m6)
|
||||
|
||||
vabd.u8 d26, d7, d17 ; m8 = abs(q0 - q2)
|
||||
|
||||
vmax.u8 d19, d19, d20
|
||||
|
||||
vabd.u8 d24, d6, d7 ; m9 = abs(p0 - q0)
|
||||
vabd.u8 d27, d3, d6 ; m10 = abs(p3 - p0)
|
||||
vabd.u8 d28, d18, d7 ; m11 = abs(q3 - q0)
|
||||
|
||||
vmax.u8 d19, d19, d23
|
||||
|
||||
vabd.u8 d23, d5, d16 ; a = abs(p1 - q1)
|
||||
vqadd.u8 d24, d24, d24 ; b = abs(p0 - q0) * 2
|
||||
|
||||
; abs () > limit
|
||||
vcge.u8 d19, d1, d19
|
||||
|
||||
; only compare the largest value to thresh
|
||||
vmax.u8 d25, d25, d26 ; m4 = max(m7, m8)
|
||||
vmax.u8 d26, d27, d28 ; m5 = max(m10, m11)
|
||||
|
||||
vshr.u8 d23, d23, #1 ; a = a / 2
|
||||
|
||||
vmax.u8 d25, d25, d26 ; m4 = max(m4, m5)
|
||||
|
||||
vqadd.u8 d24, d24, d23 ; a = b + a
|
||||
|
||||
vmax.u8 d20, d20, d25 ; m2 = max(m2, m4)
|
||||
|
||||
vmov.u8 d23, #1
|
||||
vcge.u8 d24, d0, d24 ; a > blimit
|
||||
|
||||
vcgt.u8 d21, d21, d2 ; (abs(p1 - p0) > thresh)*-1
|
||||
|
||||
vcge.u8 d20, d23, d20 ; flat
|
||||
|
||||
vand d19, d19, d24 ; mask
|
||||
|
||||
vcgt.u8 d23, d22, d2 ; (abs(q1 - q0) > thresh)*-1
|
||||
|
||||
vand d20, d20, d19 ; flat & mask
|
||||
|
||||
vmov.u8 d22, #0x80
|
||||
|
||||
vorr d23, d21, d23 ; hev
|
||||
|
||||
; This instruction will truncate the "flat & mask" masks down to 4 bits
|
||||
; each to fit into one 32 bit arm register. The values are stored in
|
||||
; q10.64[0].
|
||||
vshrn.u16 d30, q10, #4
|
||||
vmov.u32 r4, d30[0] ; flat & mask 4bits
|
||||
|
||||
adds r5, r4, #1 ; Check for all 1's
|
||||
|
||||
; If mask and flat are 1's for all vectors, then we only need to execute
|
||||
; the power branch for all vectors.
|
||||
beq power_branch_only
|
||||
|
||||
cmp r4, #0 ; Check for 0, set flag for later
|
||||
|
||||
; mbfilter() function
|
||||
; filter() function
|
||||
; convert to signed
|
||||
veor d21, d7, d22 ; qs0
|
||||
veor d24, d6, d22 ; ps0
|
||||
veor d25, d5, d22 ; ps1
|
||||
veor d26, d16, d22 ; qs1
|
||||
|
||||
vmov.u8 d27, #3
|
||||
|
||||
vsub.s8 d28, d21, d24 ; ( qs0 - ps0)
|
||||
|
||||
vqsub.s8 d29, d25, d26 ; filter = clamp(ps1-qs1)
|
||||
|
||||
vmull.s8 q15, d28, d27 ; 3 * ( qs0 - ps0)
|
||||
|
||||
vand d29, d29, d23 ; filter &= hev
|
||||
|
||||
vaddw.s8 q15, q15, d29 ; filter + 3 * (qs0 - ps0)
|
||||
|
||||
vmov.u8 d29, #4
|
||||
|
||||
; filter = clamp(filter + 3 * ( qs0 - ps0))
|
||||
vqmovn.s16 d28, q15
|
||||
|
||||
vand d28, d28, d19 ; filter &= mask
|
||||
|
||||
vqadd.s8 d30, d28, d27 ; filter2 = clamp(filter+3)
|
||||
vqadd.s8 d29, d28, d29 ; filter1 = clamp(filter+4)
|
||||
vshr.s8 d30, d30, #3 ; filter2 >>= 3
|
||||
vshr.s8 d29, d29, #3 ; filter1 >>= 3
|
||||
|
||||
vqadd.s8 d24, d24, d30 ; op0 = clamp(ps0 + filter2)
|
||||
vqsub.s8 d21, d21, d29 ; oq0 = clamp(qs0 - filter1)
|
||||
|
||||
; outer tap adjustments: ++filter1 >> 1
|
||||
vrshr.s8 d29, d29, #1
|
||||
vbic d29, d29, d23 ; filter &= ~hev
|
||||
|
||||
vqadd.s8 d25, d25, d29 ; op1 = clamp(ps1 + filter)
|
||||
vqsub.s8 d26, d26, d29 ; oq1 = clamp(qs1 - filter)
|
||||
|
||||
; If mask and flat are 0's for all vectors, then we only need to execute
|
||||
; the filter branch for all vectors.
|
||||
beq filter_branch_only
|
||||
|
||||
; If mask and flat are mixed then we must perform both branches and
|
||||
; combine the data.
|
||||
veor d24, d24, d22 ; *f_op0 = u^0x80
|
||||
veor d21, d21, d22 ; *f_oq0 = u^0x80
|
||||
veor d25, d25, d22 ; *f_op1 = u^0x80
|
||||
veor d26, d26, d22 ; *f_oq1 = u^0x80
|
||||
|
||||
; At this point we have already executed the filter branch. The filter
|
||||
; branch does not set op2 or oq2, so use p2 and q2. Execute the power
|
||||
; branch and combine the data.
|
||||
vmov.u8 d23, #2
|
||||
vaddl.u8 q14, d6, d7 ; r_op2 = p0 + q0
|
||||
vmlal.u8 q14, d3, d27 ; r_op2 += p3 * 3
|
||||
vmlal.u8 q14, d4, d23 ; r_op2 += p2 * 2
|
||||
|
||||
vbif d0, d4, d20 ; op2 |= p2 & ~(flat & mask)
|
||||
|
||||
vaddw.u8 q14, d5 ; r_op2 += p1
|
||||
|
||||
vbif d1, d25, d20 ; op1 |= f_op1 & ~(flat & mask)
|
||||
|
||||
vqrshrn.u16 d30, q14, #3 ; r_op2
|
||||
|
||||
vsubw.u8 q14, d3 ; r_op1 = r_op2 - p3
|
||||
vsubw.u8 q14, d4 ; r_op1 -= p2
|
||||
vaddw.u8 q14, d5 ; r_op1 += p1
|
||||
vaddw.u8 q14, d16 ; r_op1 += q1
|
||||
|
||||
vbif d2, d24, d20 ; op0 |= f_op0 & ~(flat & mask)
|
||||
|
||||
vqrshrn.u16 d31, q14, #3 ; r_op1
|
||||
|
||||
vsubw.u8 q14, d3 ; r_op0 = r_op1 - p3
|
||||
vsubw.u8 q14, d5 ; r_op0 -= p1
|
||||
vaddw.u8 q14, d6 ; r_op0 += p0
|
||||
vaddw.u8 q14, d17 ; r_op0 += q2
|
||||
|
||||
vbit d0, d30, d20 ; op2 |= r_op2 & (flat & mask)
|
||||
|
||||
vqrshrn.u16 d23, q14, #3 ; r_op0
|
||||
|
||||
vsubw.u8 q14, d3 ; r_oq0 = r_op0 - p3
|
||||
vsubw.u8 q14, d6 ; r_oq0 -= p0
|
||||
vaddw.u8 q14, d7 ; r_oq0 += q0
|
||||
|
||||
vbit d1, d31, d20 ; op1 |= r_op1 & (flat & mask)
|
||||
|
||||
vaddw.u8 q14, d18 ; oq0 += q3
|
||||
|
||||
vbit d2, d23, d20 ; op0 |= r_op0 & (flat & mask)
|
||||
|
||||
vqrshrn.u16 d22, q14, #3 ; r_oq0
|
||||
|
||||
vsubw.u8 q14, d4 ; r_oq1 = r_oq0 - p2
|
||||
vsubw.u8 q14, d7 ; r_oq1 -= q0
|
||||
vaddw.u8 q14, d16 ; r_oq1 += q1
|
||||
|
||||
vbif d3, d21, d20 ; oq0 |= f_oq0 & ~(flat & mask)
|
||||
|
||||
vaddw.u8 q14, d18 ; r_oq1 += q3
|
||||
|
||||
vbif d4, d26, d20 ; oq1 |= f_oq1 & ~(flat & mask)
|
||||
|
||||
vqrshrn.u16 d6, q14, #3 ; r_oq1
|
||||
|
||||
vsubw.u8 q14, d5 ; r_oq2 = r_oq1 - p1
|
||||
vsubw.u8 q14, d16 ; r_oq2 -= q1
|
||||
vaddw.u8 q14, d17 ; r_oq2 += q2
|
||||
vaddw.u8 q14, d18 ; r_oq2 += q3
|
||||
|
||||
vbif d5, d17, d20 ; oq2 |= q2 & ~(flat & mask)
|
||||
|
||||
vqrshrn.u16 d7, q14, #3 ; r_oq2
|
||||
|
||||
vbit d3, d22, d20 ; oq0 |= r_oq0 & (flat & mask)
|
||||
vbit d4, d6, d20 ; oq1 |= r_oq1 & (flat & mask)
|
||||
vbit d5, d7, d20 ; oq2 |= r_oq2 & (flat & mask)
|
||||
|
||||
bx lr
|
||||
|
||||
power_branch_only
|
||||
vmov.u8 d27, #3
|
||||
vmov.u8 d21, #2
|
||||
vaddl.u8 q14, d6, d7 ; op2 = p0 + q0
|
||||
vmlal.u8 q14, d3, d27 ; op2 += p3 * 3
|
||||
vmlal.u8 q14, d4, d21 ; op2 += p2 * 2
|
||||
vaddw.u8 q14, d5 ; op2 += p1
|
||||
vqrshrn.u16 d0, q14, #3 ; op2
|
||||
|
||||
vsubw.u8 q14, d3 ; op1 = op2 - p3
|
||||
vsubw.u8 q14, d4 ; op1 -= p2
|
||||
vaddw.u8 q14, d5 ; op1 += p1
|
||||
vaddw.u8 q14, d16 ; op1 += q1
|
||||
vqrshrn.u16 d1, q14, #3 ; op1
|
||||
|
||||
vsubw.u8 q14, d3 ; op0 = op1 - p3
|
||||
vsubw.u8 q14, d5 ; op0 -= p1
|
||||
vaddw.u8 q14, d6 ; op0 += p0
|
||||
vaddw.u8 q14, d17 ; op0 += q2
|
||||
vqrshrn.u16 d2, q14, #3 ; op0
|
||||
|
||||
vsubw.u8 q14, d3 ; oq0 = op0 - p3
|
||||
vsubw.u8 q14, d6 ; oq0 -= p0
|
||||
vaddw.u8 q14, d7 ; oq0 += q0
|
||||
vaddw.u8 q14, d18 ; oq0 += q3
|
||||
vqrshrn.u16 d3, q14, #3 ; oq0
|
||||
|
||||
vsubw.u8 q14, d4 ; oq1 = oq0 - p2
|
||||
vsubw.u8 q14, d7 ; oq1 -= q0
|
||||
vaddw.u8 q14, d16 ; oq1 += q1
|
||||
vaddw.u8 q14, d18 ; oq1 += q3
|
||||
vqrshrn.u16 d4, q14, #3 ; oq1
|
||||
|
||||
vsubw.u8 q14, d5 ; oq2 = oq1 - p1
|
||||
vsubw.u8 q14, d16 ; oq2 -= q1
|
||||
vaddw.u8 q14, d17 ; oq2 += q2
|
||||
vaddw.u8 q14, d18 ; oq2 += q3
|
||||
vqrshrn.u16 d5, q14, #3 ; oq2
|
||||
|
||||
bx lr
|
||||
|
||||
filter_branch_only
|
||||
; TODO(fgalligan): See if we can rearange registers so we do not need to
|
||||
; do the 2 vswp.
|
||||
vswp d0, d4 ; op2
|
||||
vswp d5, d17 ; oq2
|
||||
veor d2, d24, d22 ; *op0 = u^0x80
|
||||
veor d3, d21, d22 ; *oq0 = u^0x80
|
||||
veor d1, d25, d22 ; *op1 = u^0x80
|
||||
veor d4, d26, d22 ; *oq1 = u^0x80
|
||||
|
||||
bx lr
|
||||
|
||||
ENDP ; |vpx_mbloop_filter_neon|
|
||||
|
||||
END
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,184 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_ARM_MEM_NEON_H_
|
||||
#define VPX_VPX_DSP_ARM_MEM_NEON_H_
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
|
||||
static INLINE int16x4_t create_s16x4_neon(const int16_t c0, const int16_t c1,
|
||||
const int16_t c2, const int16_t c3) {
|
||||
return vcreate_s16((uint16_t)c0 | ((uint32_t)c1 << 16) |
|
||||
((int64_t)(uint16_t)c2 << 32) | ((int64_t)c3 << 48));
|
||||
}
|
||||
|
||||
static INLINE int32x2_t create_s32x2_neon(const int32_t c0, const int32_t c1) {
|
||||
return vcreate_s32((uint32_t)c0 | ((int64_t)(uint32_t)c1 << 32));
|
||||
}
|
||||
|
||||
static INLINE int32x4_t create_s32x4_neon(const int32_t c0, const int32_t c1,
|
||||
const int32_t c2, const int32_t c3) {
|
||||
return vcombine_s32(create_s32x2_neon(c0, c1), create_s32x2_neon(c2, c3));
|
||||
}
|
||||
|
||||
// Helper functions used to load tran_low_t into int16, narrowing if necessary.
|
||||
static INLINE int16x8x2_t load_tran_low_to_s16x2q(const tran_low_t *buf) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
const int32x4x2_t v0 = vld2q_s32(buf);
|
||||
const int32x4x2_t v1 = vld2q_s32(buf + 8);
|
||||
const int16x4_t s0 = vmovn_s32(v0.val[0]);
|
||||
const int16x4_t s1 = vmovn_s32(v0.val[1]);
|
||||
const int16x4_t s2 = vmovn_s32(v1.val[0]);
|
||||
const int16x4_t s3 = vmovn_s32(v1.val[1]);
|
||||
int16x8x2_t res;
|
||||
res.val[0] = vcombine_s16(s0, s2);
|
||||
res.val[1] = vcombine_s16(s1, s3);
|
||||
return res;
|
||||
#else
|
||||
return vld2q_s16(buf);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE int16x8_t load_tran_low_to_s16q(const tran_low_t *buf) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
const int32x4_t v0 = vld1q_s32(buf);
|
||||
const int32x4_t v1 = vld1q_s32(buf + 4);
|
||||
const int16x4_t s0 = vmovn_s32(v0);
|
||||
const int16x4_t s1 = vmovn_s32(v1);
|
||||
return vcombine_s16(s0, s1);
|
||||
#else
|
||||
return vld1q_s16(buf);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE int16x4_t load_tran_low_to_s16d(const tran_low_t *buf) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
const int32x4_t v0 = vld1q_s32(buf);
|
||||
return vmovn_s32(v0);
|
||||
#else
|
||||
return vld1_s16(buf);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE void store_s16q_to_tran_low(tran_low_t *buf, const int16x8_t a) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
const int32x4_t v0 = vmovl_s16(vget_low_s16(a));
|
||||
const int32x4_t v1 = vmovl_s16(vget_high_s16(a));
|
||||
vst1q_s32(buf, v0);
|
||||
vst1q_s32(buf + 4, v1);
|
||||
#else
|
||||
vst1q_s16(buf, a);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Propagate type information to the compiler. Without this the compiler may
|
||||
// assume the required alignment of uint32_t (4 bytes) and add alignment hints
|
||||
// to the memory access.
|
||||
//
|
||||
// This is used for functions operating on uint8_t which wish to load or store 4
|
||||
// values at a time but which may not be on 4 byte boundaries.
|
||||
static INLINE void uint32_to_mem(uint8_t *buf, uint32_t a) {
|
||||
memcpy(buf, &a, 4);
|
||||
}
|
||||
|
||||
// Load 2 sets of 4 bytes when alignment is not guaranteed.
|
||||
static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) {
|
||||
uint32_t a;
|
||||
uint32x2_t a_u32 = vdup_n_u32(0);
|
||||
if (stride == 4) return vld1_u8(buf);
|
||||
memcpy(&a, buf, 4);
|
||||
buf += stride;
|
||||
a_u32 = vset_lane_u32(a, a_u32, 0);
|
||||
memcpy(&a, buf, 4);
|
||||
a_u32 = vset_lane_u32(a, a_u32, 1);
|
||||
return vreinterpret_u8_u32(a_u32);
|
||||
}
|
||||
|
||||
// Store 2 sets of 4 bytes when alignment is not guaranteed.
|
||||
static INLINE void store_unaligned_u8(uint8_t *buf, int stride,
|
||||
const uint8x8_t a) {
|
||||
const uint32x2_t a_u32 = vreinterpret_u32_u8(a);
|
||||
if (stride == 4) {
|
||||
vst1_u8(buf, a);
|
||||
return;
|
||||
}
|
||||
uint32_to_mem(buf, vget_lane_u32(a_u32, 0));
|
||||
buf += stride;
|
||||
uint32_to_mem(buf, vget_lane_u32(a_u32, 1));
|
||||
}
|
||||
|
||||
// Load 4 sets of 4 bytes when alignment is not guaranteed.
|
||||
static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) {
|
||||
uint32_t a;
|
||||
uint32x4_t a_u32 = vdupq_n_u32(0);
|
||||
if (stride == 4) return vld1q_u8(buf);
|
||||
memcpy(&a, buf, 4);
|
||||
buf += stride;
|
||||
a_u32 = vsetq_lane_u32(a, a_u32, 0);
|
||||
memcpy(&a, buf, 4);
|
||||
buf += stride;
|
||||
a_u32 = vsetq_lane_u32(a, a_u32, 1);
|
||||
memcpy(&a, buf, 4);
|
||||
buf += stride;
|
||||
a_u32 = vsetq_lane_u32(a, a_u32, 2);
|
||||
memcpy(&a, buf, 4);
|
||||
buf += stride;
|
||||
a_u32 = vsetq_lane_u32(a, a_u32, 3);
|
||||
return vreinterpretq_u8_u32(a_u32);
|
||||
}
|
||||
|
||||
// Store 4 sets of 4 bytes when alignment is not guaranteed.
|
||||
static INLINE void store_unaligned_u8q(uint8_t *buf, int stride,
|
||||
const uint8x16_t a) {
|
||||
const uint32x4_t a_u32 = vreinterpretq_u32_u8(a);
|
||||
if (stride == 4) {
|
||||
vst1q_u8(buf, a);
|
||||
return;
|
||||
}
|
||||
uint32_to_mem(buf, vgetq_lane_u32(a_u32, 0));
|
||||
buf += stride;
|
||||
uint32_to_mem(buf, vgetq_lane_u32(a_u32, 1));
|
||||
buf += stride;
|
||||
uint32_to_mem(buf, vgetq_lane_u32(a_u32, 2));
|
||||
buf += stride;
|
||||
uint32_to_mem(buf, vgetq_lane_u32(a_u32, 3));
|
||||
}
|
||||
|
||||
// Load 2 sets of 4 bytes when alignment is guaranteed.
|
||||
static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) {
|
||||
uint32x2_t a = vdup_n_u32(0);
|
||||
|
||||
assert(!((intptr_t)buf % sizeof(uint32_t)));
|
||||
assert(!(stride % sizeof(uint32_t)));
|
||||
|
||||
a = vld1_lane_u32((const uint32_t *)buf, a, 0);
|
||||
buf += stride;
|
||||
a = vld1_lane_u32((const uint32_t *)buf, a, 1);
|
||||
return vreinterpret_u8_u32(a);
|
||||
}
|
||||
|
||||
// Store 2 sets of 4 bytes when alignment is guaranteed.
|
||||
static INLINE void store_u8(uint8_t *buf, int stride, const uint8x8_t a) {
|
||||
uint32x2_t a_u32 = vreinterpret_u32_u8(a);
|
||||
|
||||
assert(!((intptr_t)buf % sizeof(uint32_t)));
|
||||
assert(!(stride % sizeof(uint32_t)));
|
||||
|
||||
vst1_lane_u32((uint32_t *)buf, a_u32, 0);
|
||||
buf += stride;
|
||||
vst1_lane_u32((uint32_t *)buf, a_u32, 1);
|
||||
}
|
||||
#endif // VPX_VPX_DSP_ARM_MEM_NEON_H_
|
||||
@@ -0,0 +1,317 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
|
||||
static INLINE void calculate_dqcoeff_and_store(const int16x8_t qcoeff,
|
||||
const int16x8_t dequant,
|
||||
tran_low_t *dqcoeff) {
|
||||
const int32x4_t dqcoeff_0 =
|
||||
vmull_s16(vget_low_s16(qcoeff), vget_low_s16(dequant));
|
||||
const int32x4_t dqcoeff_1 =
|
||||
vmull_s16(vget_high_s16(qcoeff), vget_high_s16(dequant));
|
||||
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
vst1q_s32(dqcoeff, dqcoeff_0);
|
||||
vst1q_s32(dqcoeff + 4, dqcoeff_1);
|
||||
#else
|
||||
vst1q_s16(dqcoeff, vcombine_s16(vmovn_s32(dqcoeff_0), vmovn_s32(dqcoeff_1)));
|
||||
#endif // CONFIG_VP9_HIGHBITDEPTH
|
||||
}
|
||||
|
||||
void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
|
||||
int skip_block, const int16_t *zbin_ptr,
|
||||
const int16_t *round_ptr, const int16_t *quant_ptr,
|
||||
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
|
||||
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
|
||||
uint16_t *eob_ptr, const int16_t *scan,
|
||||
const int16_t *iscan) {
|
||||
const int16x8_t one = vdupq_n_s16(1);
|
||||
const int16x8_t neg_one = vdupq_n_s16(-1);
|
||||
uint16x8_t eob_max;
|
||||
(void)scan;
|
||||
(void)skip_block;
|
||||
assert(!skip_block);
|
||||
|
||||
// Process first 8 values which include a dc component.
|
||||
{
|
||||
// Only the first element of each vector is DC.
|
||||
const int16x8_t zbin = vld1q_s16(zbin_ptr);
|
||||
const int16x8_t round = vld1q_s16(round_ptr);
|
||||
const int16x8_t quant = vld1q_s16(quant_ptr);
|
||||
const int16x8_t quant_shift = vld1q_s16(quant_shift_ptr);
|
||||
const int16x8_t dequant = vld1q_s16(dequant_ptr);
|
||||
// Add one because the eob does not index from 0.
|
||||
const uint16x8_t v_iscan =
|
||||
vreinterpretq_u16_s16(vaddq_s16(vld1q_s16(iscan), one));
|
||||
|
||||
const int16x8_t coeff = load_tran_low_to_s16q(coeff_ptr);
|
||||
const int16x8_t coeff_sign = vshrq_n_s16(coeff, 15);
|
||||
const int16x8_t coeff_abs = vabsq_s16(coeff);
|
||||
|
||||
const int16x8_t zbin_mask =
|
||||
vreinterpretq_s16_u16(vcgeq_s16(coeff_abs, zbin));
|
||||
|
||||
const int16x8_t rounded = vqaddq_s16(coeff_abs, round);
|
||||
|
||||
// (round * quant * 2) >> 16 >> 1 == (round * quant) >> 16
|
||||
int16x8_t qcoeff = vshrq_n_s16(vqdmulhq_s16(rounded, quant), 1);
|
||||
|
||||
qcoeff = vaddq_s16(qcoeff, rounded);
|
||||
|
||||
// (qcoeff * quant_shift * 2) >> 16 >> 1 == (qcoeff * quant_shift) >> 16
|
||||
qcoeff = vshrq_n_s16(vqdmulhq_s16(qcoeff, quant_shift), 1);
|
||||
|
||||
// Restore the sign bit.
|
||||
qcoeff = veorq_s16(qcoeff, coeff_sign);
|
||||
qcoeff = vsubq_s16(qcoeff, coeff_sign);
|
||||
|
||||
qcoeff = vandq_s16(qcoeff, zbin_mask);
|
||||
|
||||
// Set non-zero elements to -1 and use that to extract values for eob.
|
||||
eob_max = vandq_u16(vtstq_s16(qcoeff, neg_one), v_iscan);
|
||||
|
||||
coeff_ptr += 8;
|
||||
iscan += 8;
|
||||
|
||||
store_s16q_to_tran_low(qcoeff_ptr, qcoeff);
|
||||
qcoeff_ptr += 8;
|
||||
|
||||
calculate_dqcoeff_and_store(qcoeff, dequant, dqcoeff_ptr);
|
||||
dqcoeff_ptr += 8;
|
||||
}
|
||||
|
||||
n_coeffs -= 8;
|
||||
|
||||
{
|
||||
const int16x8_t zbin = vdupq_n_s16(zbin_ptr[1]);
|
||||
const int16x8_t round = vdupq_n_s16(round_ptr[1]);
|
||||
const int16x8_t quant = vdupq_n_s16(quant_ptr[1]);
|
||||
const int16x8_t quant_shift = vdupq_n_s16(quant_shift_ptr[1]);
|
||||
const int16x8_t dequant = vdupq_n_s16(dequant_ptr[1]);
|
||||
|
||||
do {
|
||||
// Add one because the eob is not its index.
|
||||
const uint16x8_t v_iscan =
|
||||
vreinterpretq_u16_s16(vaddq_s16(vld1q_s16(iscan), one));
|
||||
|
||||
const int16x8_t coeff = load_tran_low_to_s16q(coeff_ptr);
|
||||
const int16x8_t coeff_sign = vshrq_n_s16(coeff, 15);
|
||||
const int16x8_t coeff_abs = vabsq_s16(coeff);
|
||||
|
||||
const int16x8_t zbin_mask =
|
||||
vreinterpretq_s16_u16(vcgeq_s16(coeff_abs, zbin));
|
||||
|
||||
const int16x8_t rounded = vqaddq_s16(coeff_abs, round);
|
||||
|
||||
// (round * quant * 2) >> 16 >> 1 == (round * quant) >> 16
|
||||
int16x8_t qcoeff = vshrq_n_s16(vqdmulhq_s16(rounded, quant), 1);
|
||||
|
||||
qcoeff = vaddq_s16(qcoeff, rounded);
|
||||
|
||||
// (qcoeff * quant_shift * 2) >> 16 >> 1 == (qcoeff * quant_shift) >> 16
|
||||
qcoeff = vshrq_n_s16(vqdmulhq_s16(qcoeff, quant_shift), 1);
|
||||
|
||||
// Restore the sign bit.
|
||||
qcoeff = veorq_s16(qcoeff, coeff_sign);
|
||||
qcoeff = vsubq_s16(qcoeff, coeff_sign);
|
||||
|
||||
qcoeff = vandq_s16(qcoeff, zbin_mask);
|
||||
|
||||
// Set non-zero elements to -1 and use that to extract values for eob.
|
||||
eob_max =
|
||||
vmaxq_u16(eob_max, vandq_u16(vtstq_s16(qcoeff, neg_one), v_iscan));
|
||||
|
||||
coeff_ptr += 8;
|
||||
iscan += 8;
|
||||
|
||||
store_s16q_to_tran_low(qcoeff_ptr, qcoeff);
|
||||
qcoeff_ptr += 8;
|
||||
|
||||
calculate_dqcoeff_and_store(qcoeff, dequant, dqcoeff_ptr);
|
||||
dqcoeff_ptr += 8;
|
||||
|
||||
n_coeffs -= 8;
|
||||
} while (n_coeffs > 0);
|
||||
}
|
||||
|
||||
#ifdef __aarch64__
|
||||
*eob_ptr = vmaxvq_u16(eob_max);
|
||||
#else
|
||||
{
|
||||
const uint16x4_t eob_max_0 =
|
||||
vmax_u16(vget_low_u16(eob_max), vget_high_u16(eob_max));
|
||||
const uint16x4_t eob_max_1 = vpmax_u16(eob_max_0, eob_max_0);
|
||||
const uint16x4_t eob_max_2 = vpmax_u16(eob_max_1, eob_max_1);
|
||||
vst1_lane_u16(eob_ptr, eob_max_2, 0);
|
||||
}
|
||||
#endif // __aarch64__
|
||||
}
|
||||
|
||||
static INLINE int32x4_t extract_sign_bit(int32x4_t a) {
|
||||
return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a), 31));
|
||||
}
|
||||
|
||||
static INLINE void calculate_dqcoeff_and_store_32x32(const int16x8_t qcoeff,
|
||||
const int16x8_t dequant,
|
||||
tran_low_t *dqcoeff) {
|
||||
int32x4_t dqcoeff_0 = vmull_s16(vget_low_s16(qcoeff), vget_low_s16(dequant));
|
||||
int32x4_t dqcoeff_1 =
|
||||
vmull_s16(vget_high_s16(qcoeff), vget_high_s16(dequant));
|
||||
|
||||
// Add 1 if negative to round towards zero because the C uses division.
|
||||
dqcoeff_0 = vaddq_s32(dqcoeff_0, extract_sign_bit(dqcoeff_0));
|
||||
dqcoeff_1 = vaddq_s32(dqcoeff_1, extract_sign_bit(dqcoeff_1));
|
||||
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
dqcoeff_0 = vshrq_n_s32(dqcoeff_0, 1);
|
||||
dqcoeff_1 = vshrq_n_s32(dqcoeff_1, 1);
|
||||
vst1q_s32(dqcoeff, dqcoeff_0);
|
||||
vst1q_s32(dqcoeff + 4, dqcoeff_1);
|
||||
#else
|
||||
vst1q_s16(dqcoeff,
|
||||
vcombine_s16(vshrn_n_s32(dqcoeff_0, 1), vshrn_n_s32(dqcoeff_1, 1)));
|
||||
#endif // CONFIG_VP9_HIGHBITDEPTH
|
||||
}
|
||||
|
||||
// Main difference is that zbin values are halved before comparison and dqcoeff
|
||||
// values are divided by 2. zbin is rounded but dqcoeff is not.
|
||||
void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
|
||||
int skip_block, const int16_t *zbin_ptr,
|
||||
const int16_t *round_ptr,
|
||||
const int16_t *quant_ptr,
|
||||
const int16_t *quant_shift_ptr,
|
||||
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
|
||||
const int16_t *dequant_ptr, uint16_t *eob_ptr,
|
||||
const int16_t *scan, const int16_t *iscan) {
|
||||
const int16x8_t one = vdupq_n_s16(1);
|
||||
const int16x8_t neg_one = vdupq_n_s16(-1);
|
||||
uint16x8_t eob_max;
|
||||
int i;
|
||||
(void)scan;
|
||||
(void)n_coeffs; // Because we will always calculate 32*32.
|
||||
(void)skip_block;
|
||||
assert(!skip_block);
|
||||
|
||||
// Process first 8 values which include a dc component.
|
||||
{
|
||||
// Only the first element of each vector is DC.
|
||||
const int16x8_t zbin = vrshrq_n_s16(vld1q_s16(zbin_ptr), 1);
|
||||
const int16x8_t round = vrshrq_n_s16(vld1q_s16(round_ptr), 1);
|
||||
const int16x8_t quant = vld1q_s16(quant_ptr);
|
||||
const int16x8_t quant_shift = vld1q_s16(quant_shift_ptr);
|
||||
const int16x8_t dequant = vld1q_s16(dequant_ptr);
|
||||
// Add one because the eob does not index from 0.
|
||||
const uint16x8_t v_iscan =
|
||||
vreinterpretq_u16_s16(vaddq_s16(vld1q_s16(iscan), one));
|
||||
|
||||
const int16x8_t coeff = load_tran_low_to_s16q(coeff_ptr);
|
||||
const int16x8_t coeff_sign = vshrq_n_s16(coeff, 15);
|
||||
const int16x8_t coeff_abs = vabsq_s16(coeff);
|
||||
|
||||
const int16x8_t zbin_mask =
|
||||
vreinterpretq_s16_u16(vcgeq_s16(coeff_abs, zbin));
|
||||
|
||||
const int16x8_t rounded = vqaddq_s16(coeff_abs, round);
|
||||
|
||||
// (round * quant * 2) >> 16 >> 1 == (round * quant) >> 16
|
||||
int16x8_t qcoeff = vshrq_n_s16(vqdmulhq_s16(rounded, quant), 1);
|
||||
|
||||
qcoeff = vaddq_s16(qcoeff, rounded);
|
||||
|
||||
// (qcoeff * quant_shift * 2) >> 16 == (qcoeff * quant_shift) >> 15
|
||||
qcoeff = vqdmulhq_s16(qcoeff, quant_shift);
|
||||
|
||||
// Restore the sign bit.
|
||||
qcoeff = veorq_s16(qcoeff, coeff_sign);
|
||||
qcoeff = vsubq_s16(qcoeff, coeff_sign);
|
||||
|
||||
qcoeff = vandq_s16(qcoeff, zbin_mask);
|
||||
|
||||
// Set non-zero elements to -1 and use that to extract values for eob.
|
||||
eob_max = vandq_u16(vtstq_s16(qcoeff, neg_one), v_iscan);
|
||||
|
||||
coeff_ptr += 8;
|
||||
iscan += 8;
|
||||
|
||||
store_s16q_to_tran_low(qcoeff_ptr, qcoeff);
|
||||
qcoeff_ptr += 8;
|
||||
|
||||
calculate_dqcoeff_and_store_32x32(qcoeff, dequant, dqcoeff_ptr);
|
||||
dqcoeff_ptr += 8;
|
||||
}
|
||||
|
||||
{
|
||||
const int16x8_t zbin = vrshrq_n_s16(vdupq_n_s16(zbin_ptr[1]), 1);
|
||||
const int16x8_t round = vrshrq_n_s16(vdupq_n_s16(round_ptr[1]), 1);
|
||||
const int16x8_t quant = vdupq_n_s16(quant_ptr[1]);
|
||||
const int16x8_t quant_shift = vdupq_n_s16(quant_shift_ptr[1]);
|
||||
const int16x8_t dequant = vdupq_n_s16(dequant_ptr[1]);
|
||||
|
||||
for (i = 1; i < 32 * 32 / 8; ++i) {
|
||||
// Add one because the eob is not its index.
|
||||
const uint16x8_t v_iscan =
|
||||
vreinterpretq_u16_s16(vaddq_s16(vld1q_s16(iscan), one));
|
||||
|
||||
const int16x8_t coeff = load_tran_low_to_s16q(coeff_ptr);
|
||||
const int16x8_t coeff_sign = vshrq_n_s16(coeff, 15);
|
||||
const int16x8_t coeff_abs = vabsq_s16(coeff);
|
||||
|
||||
const int16x8_t zbin_mask =
|
||||
vreinterpretq_s16_u16(vcgeq_s16(coeff_abs, zbin));
|
||||
|
||||
const int16x8_t rounded = vqaddq_s16(coeff_abs, round);
|
||||
|
||||
// (round * quant * 2) >> 16 >> 1 == (round * quant) >> 16
|
||||
int16x8_t qcoeff = vshrq_n_s16(vqdmulhq_s16(rounded, quant), 1);
|
||||
|
||||
qcoeff = vaddq_s16(qcoeff, rounded);
|
||||
|
||||
// (qcoeff * quant_shift * 2) >> 16 == (qcoeff * quant_shift) >> 15
|
||||
qcoeff = vqdmulhq_s16(qcoeff, quant_shift);
|
||||
|
||||
// Restore the sign bit.
|
||||
qcoeff = veorq_s16(qcoeff, coeff_sign);
|
||||
qcoeff = vsubq_s16(qcoeff, coeff_sign);
|
||||
|
||||
qcoeff = vandq_s16(qcoeff, zbin_mask);
|
||||
|
||||
// Set non-zero elements to -1 and use that to extract values for eob.
|
||||
eob_max =
|
||||
vmaxq_u16(eob_max, vandq_u16(vtstq_s16(qcoeff, neg_one), v_iscan));
|
||||
|
||||
coeff_ptr += 8;
|
||||
iscan += 8;
|
||||
|
||||
store_s16q_to_tran_low(qcoeff_ptr, qcoeff);
|
||||
qcoeff_ptr += 8;
|
||||
|
||||
calculate_dqcoeff_and_store_32x32(qcoeff, dequant, dqcoeff_ptr);
|
||||
dqcoeff_ptr += 8;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __aarch64__
|
||||
*eob_ptr = vmaxvq_u16(eob_max);
|
||||
#else
|
||||
{
|
||||
const uint16x4_t eob_max_0 =
|
||||
vmax_u16(vget_low_u16(eob_max), vget_high_u16(eob_max));
|
||||
const uint16x4_t eob_max_1 = vpmax_u16(eob_max_0, eob_max_0);
|
||||
const uint16x4_t eob_max_2 = vpmax_u16(eob_max_1, eob_max_1);
|
||||
vst1_lane_u16(eob_ptr, eob_max_2, 0);
|
||||
}
|
||||
#endif // __aarch64__
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
/*
|
||||
* Copyright (c) 2015 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include <assert.h>
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/sum_neon.h"
|
||||
|
||||
static INLINE uint8x8_t load_unaligned_2_buffers(const void *const buf0,
|
||||
const void *const buf1) {
|
||||
uint32_t a;
|
||||
uint32x2_t aa = vdup_n_u32(0);
|
||||
memcpy(&a, buf0, 4);
|
||||
aa = vset_lane_u32(a, aa, 0);
|
||||
memcpy(&a, buf1, 4);
|
||||
aa = vset_lane_u32(a, aa, 1);
|
||||
return vreinterpret_u8_u32(aa);
|
||||
}
|
||||
|
||||
static INLINE void sad4x_4d(const uint8_t *const src_ptr, const int src_stride,
|
||||
const uint8_t *const ref_array[4],
|
||||
const int ref_stride, const int height,
|
||||
uint32_t *const res) {
|
||||
int i;
|
||||
uint16x8_t abs[2] = { vdupq_n_u16(0), vdupq_n_u16(0) };
|
||||
uint16x4_t a[2];
|
||||
uint32x4_t r;
|
||||
|
||||
assert(!((intptr_t)src_ptr % sizeof(uint32_t)));
|
||||
assert(!(src_stride % sizeof(uint32_t)));
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x8_t s = vreinterpret_u8_u32(
|
||||
vld1_dup_u32((const uint32_t *)(src_ptr + i * src_stride)));
|
||||
const uint8x8_t ref01 = load_unaligned_2_buffers(
|
||||
ref_array[0] + i * ref_stride, ref_array[1] + i * ref_stride);
|
||||
const uint8x8_t ref23 = load_unaligned_2_buffers(
|
||||
ref_array[2] + i * ref_stride, ref_array[3] + i * ref_stride);
|
||||
abs[0] = vabal_u8(abs[0], s, ref01);
|
||||
abs[1] = vabal_u8(abs[1], s, ref23);
|
||||
}
|
||||
|
||||
a[0] = vpadd_u16(vget_low_u16(abs[0]), vget_high_u16(abs[0]));
|
||||
a[1] = vpadd_u16(vget_low_u16(abs[1]), vget_high_u16(abs[1]));
|
||||
r = vpaddlq_u16(vcombine_u16(a[0], a[1]));
|
||||
vst1q_u32(res, r);
|
||||
}
|
||||
|
||||
void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 4, res);
|
||||
}
|
||||
|
||||
void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 8, res);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Can handle 512 pixels' sad sum (such as 16x32 or 32x16)
|
||||
static INLINE void sad_512_pel_final_neon(const uint16x8_t *sum /*[4]*/,
|
||||
uint32_t *const res) {
|
||||
const uint16x4_t a0 = vadd_u16(vget_low_u16(sum[0]), vget_high_u16(sum[0]));
|
||||
const uint16x4_t a1 = vadd_u16(vget_low_u16(sum[1]), vget_high_u16(sum[1]));
|
||||
const uint16x4_t a2 = vadd_u16(vget_low_u16(sum[2]), vget_high_u16(sum[2]));
|
||||
const uint16x4_t a3 = vadd_u16(vget_low_u16(sum[3]), vget_high_u16(sum[3]));
|
||||
const uint16x4_t b0 = vpadd_u16(a0, a1);
|
||||
const uint16x4_t b1 = vpadd_u16(a2, a3);
|
||||
const uint32x4_t r = vpaddlq_u16(vcombine_u16(b0, b1));
|
||||
vst1q_u32(res, r);
|
||||
}
|
||||
|
||||
// Can handle 1024 pixels' sad sum (such as 32x32)
|
||||
static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/,
|
||||
uint32_t *const res) {
|
||||
const uint16x4_t a0 = vpadd_u16(vget_low_u16(sum[0]), vget_high_u16(sum[0]));
|
||||
const uint16x4_t a1 = vpadd_u16(vget_low_u16(sum[1]), vget_high_u16(sum[1]));
|
||||
const uint16x4_t a2 = vpadd_u16(vget_low_u16(sum[2]), vget_high_u16(sum[2]));
|
||||
const uint16x4_t a3 = vpadd_u16(vget_low_u16(sum[3]), vget_high_u16(sum[3]));
|
||||
const uint32x4_t b0 = vpaddlq_u16(vcombine_u16(a0, a1));
|
||||
const uint32x4_t b1 = vpaddlq_u16(vcombine_u16(a2, a3));
|
||||
const uint32x2_t c0 = vpadd_u32(vget_low_u32(b0), vget_high_u32(b0));
|
||||
const uint32x2_t c1 = vpadd_u32(vget_low_u32(b1), vget_high_u32(b1));
|
||||
vst1q_u32(res, vcombine_u32(c0, c1));
|
||||
}
|
||||
|
||||
// Can handle 2048 pixels' sad sum (such as 32x64 or 64x32)
|
||||
static INLINE void sad_2048_pel_final_neon(const uint16x8_t *sum /*[4]*/,
|
||||
uint32_t *const res) {
|
||||
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
|
||||
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
|
||||
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
|
||||
const uint32x4_t a3 = vpaddlq_u16(sum[3]);
|
||||
const uint32x2_t b0 = vadd_u32(vget_low_u32(a0), vget_high_u32(a0));
|
||||
const uint32x2_t b1 = vadd_u32(vget_low_u32(a1), vget_high_u32(a1));
|
||||
const uint32x2_t b2 = vadd_u32(vget_low_u32(a2), vget_high_u32(a2));
|
||||
const uint32x2_t b3 = vadd_u32(vget_low_u32(a3), vget_high_u32(a3));
|
||||
const uint32x2_t c0 = vpadd_u32(b0, b1);
|
||||
const uint32x2_t c1 = vpadd_u32(b2, b3);
|
||||
vst1q_u32(res, vcombine_u32(c0, c1));
|
||||
}
|
||||
|
||||
// Can handle 4096 pixels' sad sum (such as 64x64)
|
||||
static INLINE void sad_4096_pel_final_neon(const uint16x8_t *sum /*[8]*/,
|
||||
uint32_t *const res) {
|
||||
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
|
||||
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
|
||||
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
|
||||
const uint32x4_t a3 = vpaddlq_u16(sum[3]);
|
||||
const uint32x4_t a4 = vpaddlq_u16(sum[4]);
|
||||
const uint32x4_t a5 = vpaddlq_u16(sum[5]);
|
||||
const uint32x4_t a6 = vpaddlq_u16(sum[6]);
|
||||
const uint32x4_t a7 = vpaddlq_u16(sum[7]);
|
||||
const uint32x4_t b0 = vaddq_u32(a0, a1);
|
||||
const uint32x4_t b1 = vaddq_u32(a2, a3);
|
||||
const uint32x4_t b2 = vaddq_u32(a4, a5);
|
||||
const uint32x4_t b3 = vaddq_u32(a6, a7);
|
||||
const uint32x2_t c0 = vadd_u32(vget_low_u32(b0), vget_high_u32(b0));
|
||||
const uint32x2_t c1 = vadd_u32(vget_low_u32(b1), vget_high_u32(b1));
|
||||
const uint32x2_t c2 = vadd_u32(vget_low_u32(b2), vget_high_u32(b2));
|
||||
const uint32x2_t c3 = vadd_u32(vget_low_u32(b3), vget_high_u32(b3));
|
||||
const uint32x2_t d0 = vpadd_u32(c0, c1);
|
||||
const uint32x2_t d1 = vpadd_u32(c2, c3);
|
||||
vst1q_u32(res, vcombine_u32(d0, d1));
|
||||
}
|
||||
|
||||
static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res, const int height) {
|
||||
int i, j;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
|
||||
vdupq_n_u16(0) };
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x8_t s = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
for (j = 0; j < 4; ++j) {
|
||||
const uint8x8_t b_u8 = vld1_u8(ref_loop[j]);
|
||||
ref_loop[j] += ref_stride;
|
||||
sum[j] = vabal_u8(sum[j], s, b_u8);
|
||||
}
|
||||
}
|
||||
|
||||
sad_512_pel_final_neon(sum, res);
|
||||
}
|
||||
|
||||
void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 4);
|
||||
}
|
||||
|
||||
void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8);
|
||||
}
|
||||
|
||||
void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr,
|
||||
uint16x8_t *const sum) {
|
||||
const uint8x16_t r = vld1q_u8(ref_ptr);
|
||||
*sum = vabal_u8(*sum, vget_low_u8(src_ptr), vget_low_u8(r));
|
||||
*sum = vabal_u8(*sum, vget_high_u8(src_ptr), vget_high_u8(r));
|
||||
}
|
||||
|
||||
static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res, const int height) {
|
||||
int i, j;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
|
||||
vdupq_n_u16(0) };
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x16_t s = vld1q_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
for (j = 0; j < 4; ++j) {
|
||||
sad16_neon(ref_loop[j], s, &sum[j]);
|
||||
ref_loop[j] += ref_stride;
|
||||
}
|
||||
}
|
||||
|
||||
sad_512_pel_final_neon(sum, res);
|
||||
}
|
||||
|
||||
void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8);
|
||||
}
|
||||
|
||||
void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16);
|
||||
}
|
||||
|
||||
void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
const int height, uint16x8_t *const sum) {
|
||||
int i;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
|
||||
sum[0] = sum[1] = sum[2] = sum[3] = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
uint8x16_t s;
|
||||
|
||||
s = vld1q_u8(src_ptr + 0 * 16);
|
||||
sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 1 * 16);
|
||||
sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]);
|
||||
|
||||
src_ptr += src_stride;
|
||||
ref_loop[0] += ref_stride;
|
||||
ref_loop[1] += ref_stride;
|
||||
ref_loop[2] += ref_stride;
|
||||
ref_loop[3] += ref_stride;
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
uint16x8_t sum[4];
|
||||
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 16, sum);
|
||||
sad_512_pel_final_neon(sum, res);
|
||||
}
|
||||
|
||||
void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
uint16x8_t sum[4];
|
||||
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 32, sum);
|
||||
sad_1024_pel_final_neon(sum, res);
|
||||
}
|
||||
|
||||
void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
uint16x8_t sum[4];
|
||||
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 64, sum);
|
||||
sad_2048_pel_final_neon(sum, res);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
int i;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
|
||||
vdupq_n_u16(0) };
|
||||
|
||||
for (i = 0; i < 32; ++i) {
|
||||
uint8x16_t s;
|
||||
|
||||
s = vld1q_u8(src_ptr + 0 * 16);
|
||||
sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 1 * 16);
|
||||
sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 2 * 16);
|
||||
sad16_neon(ref_loop[0] + 2 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 2 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 2 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 2 * 16, s, &sum[3]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 3 * 16);
|
||||
sad16_neon(ref_loop[0] + 3 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 3 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 3 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 3 * 16, s, &sum[3]);
|
||||
|
||||
src_ptr += src_stride;
|
||||
ref_loop[0] += ref_stride;
|
||||
ref_loop[1] += ref_stride;
|
||||
ref_loop[2] += ref_stride;
|
||||
ref_loop[3] += ref_stride;
|
||||
}
|
||||
|
||||
sad_2048_pel_final_neon(sum, res);
|
||||
}
|
||||
|
||||
void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
int i;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
uint16x8_t sum[8] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
|
||||
vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
|
||||
vdupq_n_u16(0), vdupq_n_u16(0) };
|
||||
|
||||
for (i = 0; i < 64; ++i) {
|
||||
uint8x16_t s;
|
||||
|
||||
s = vld1q_u8(src_ptr + 0 * 16);
|
||||
sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 0 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[2] + 0 * 16, s, &sum[4]);
|
||||
sad16_neon(ref_loop[3] + 0 * 16, s, &sum[6]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 1 * 16);
|
||||
sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 1 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[2] + 1 * 16, s, &sum[4]);
|
||||
sad16_neon(ref_loop[3] + 1 * 16, s, &sum[6]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 2 * 16);
|
||||
sad16_neon(ref_loop[0] + 2 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[1] + 2 * 16, s, &sum[3]);
|
||||
sad16_neon(ref_loop[2] + 2 * 16, s, &sum[5]);
|
||||
sad16_neon(ref_loop[3] + 2 * 16, s, &sum[7]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 3 * 16);
|
||||
sad16_neon(ref_loop[0] + 3 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[1] + 3 * 16, s, &sum[3]);
|
||||
sad16_neon(ref_loop[2] + 3 * 16, s, &sum[5]);
|
||||
sad16_neon(ref_loop[3] + 3 * 16, s, &sum[7]);
|
||||
|
||||
src_ptr += src_stride;
|
||||
ref_loop[0] += ref_stride;
|
||||
ref_loop[1] += ref_stride;
|
||||
ref_loop[2] += ref_stride;
|
||||
ref_loop[3] += ref_stride;
|
||||
}
|
||||
|
||||
sad_4096_pel_final_neon(sum, res);
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/sum_neon.h"
|
||||
|
||||
uint32_t vpx_sad4x4_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride) {
|
||||
const uint8x16_t src_u8 = load_unaligned_u8q(src_ptr, src_stride);
|
||||
const uint8x16_t ref_u8 = load_unaligned_u8q(ref_ptr, ref_stride);
|
||||
uint16x8_t abs = vabdl_u8(vget_low_u8(src_u8), vget_low_u8(ref_u8));
|
||||
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(ref_u8));
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
|
||||
}
|
||||
|
||||
uint32_t vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const uint8_t *second_pred) {
|
||||
const uint8x16_t src_u8 = load_unaligned_u8q(src_ptr, src_stride);
|
||||
const uint8x16_t ref_u8 = load_unaligned_u8q(ref_ptr, ref_stride);
|
||||
const uint8x16_t second_pred_u8 = vld1q_u8(second_pred);
|
||||
const uint8x16_t avg = vrhaddq_u8(ref_u8, second_pred_u8);
|
||||
uint16x8_t abs = vabdl_u8(vget_low_u8(src_u8), vget_low_u8(avg));
|
||||
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(avg));
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
|
||||
}
|
||||
|
||||
uint32_t vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride) {
|
||||
int i;
|
||||
uint16x8_t abs = vdupq_n_u16(0);
|
||||
for (i = 0; i < 8; i += 4) {
|
||||
const uint8x16_t src_u8 = load_unaligned_u8q(src_ptr, src_stride);
|
||||
const uint8x16_t ref_u8 = load_unaligned_u8q(ref_ptr, ref_stride);
|
||||
src_ptr += 4 * src_stride;
|
||||
ref_ptr += 4 * ref_stride;
|
||||
abs = vabal_u8(abs, vget_low_u8(src_u8), vget_low_u8(ref_u8));
|
||||
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(ref_u8));
|
||||
}
|
||||
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
|
||||
}
|
||||
|
||||
uint32_t vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const uint8_t *second_pred) {
|
||||
int i;
|
||||
uint16x8_t abs = vdupq_n_u16(0);
|
||||
for (i = 0; i < 8; i += 4) {
|
||||
const uint8x16_t src_u8 = load_unaligned_u8q(src_ptr, src_stride);
|
||||
const uint8x16_t ref_u8 = load_unaligned_u8q(ref_ptr, ref_stride);
|
||||
const uint8x16_t second_pred_u8 = vld1q_u8(second_pred);
|
||||
const uint8x16_t avg = vrhaddq_u8(ref_u8, second_pred_u8);
|
||||
src_ptr += 4 * src_stride;
|
||||
ref_ptr += 4 * ref_stride;
|
||||
second_pred += 16;
|
||||
abs = vabal_u8(abs, vget_low_u8(src_u8), vget_low_u8(avg));
|
||||
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(avg));
|
||||
}
|
||||
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t sad8x(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const int height) {
|
||||
int i;
|
||||
uint16x8_t abs = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x8_t a_u8 = vld1_u8(src_ptr);
|
||||
const uint8x8_t b_u8 = vld1_u8(ref_ptr);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
abs = vabal_u8(abs, a_u8, b_u8);
|
||||
}
|
||||
return abs;
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t sad8x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const uint8_t *second_pred,
|
||||
const int height) {
|
||||
int i;
|
||||
uint16x8_t abs = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x8_t a_u8 = vld1_u8(src_ptr);
|
||||
const uint8x8_t b_u8 = vld1_u8(ref_ptr);
|
||||
const uint8x8_t c_u8 = vld1_u8(second_pred);
|
||||
const uint8x8_t avg = vrhadd_u8(b_u8, c_u8);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
second_pred += 8;
|
||||
abs = vabal_u8(abs, a_u8, avg);
|
||||
}
|
||||
return abs;
|
||||
}
|
||||
|
||||
#define sad8xN(n) \
|
||||
uint32_t vpx_sad8x##n##_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride) { \
|
||||
const uint16x8_t abs = sad8x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
} \
|
||||
\
|
||||
uint32_t vpx_sad8x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride, \
|
||||
const uint8_t *second_pred) { \
|
||||
const uint16x8_t abs = \
|
||||
sad8x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
}
|
||||
|
||||
sad8xN(4);
|
||||
sad8xN(8);
|
||||
sad8xN(16);
|
||||
|
||||
static INLINE uint16x8_t sad16x(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const int height) {
|
||||
int i;
|
||||
uint16x8_t abs = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x16_t a_u8 = vld1q_u8(src_ptr);
|
||||
const uint8x16_t b_u8 = vld1q_u8(ref_ptr);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
abs = vabal_u8(abs, vget_low_u8(a_u8), vget_low_u8(b_u8));
|
||||
abs = vabal_u8(abs, vget_high_u8(a_u8), vget_high_u8(b_u8));
|
||||
}
|
||||
return abs;
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t sad16x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const uint8_t *second_pred,
|
||||
const int height) {
|
||||
int i;
|
||||
uint16x8_t abs = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x16_t a_u8 = vld1q_u8(src_ptr);
|
||||
const uint8x16_t b_u8 = vld1q_u8(ref_ptr);
|
||||
const uint8x16_t c_u8 = vld1q_u8(second_pred);
|
||||
const uint8x16_t avg = vrhaddq_u8(b_u8, c_u8);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
second_pred += 16;
|
||||
abs = vabal_u8(abs, vget_low_u8(a_u8), vget_low_u8(avg));
|
||||
abs = vabal_u8(abs, vget_high_u8(a_u8), vget_high_u8(avg));
|
||||
}
|
||||
return abs;
|
||||
}
|
||||
|
||||
#define sad16xN(n) \
|
||||
uint32_t vpx_sad16x##n##_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride) { \
|
||||
const uint16x8_t abs = \
|
||||
sad16x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
} \
|
||||
\
|
||||
uint32_t vpx_sad16x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride, \
|
||||
const uint8_t *second_pred) { \
|
||||
const uint16x8_t abs = \
|
||||
sad16x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
}
|
||||
|
||||
sad16xN(8);
|
||||
sad16xN(16);
|
||||
sad16xN(32);
|
||||
|
||||
static INLINE uint16x8_t sad32x(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const int height) {
|
||||
int i;
|
||||
uint16x8_t abs = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x16_t a_lo = vld1q_u8(src_ptr);
|
||||
const uint8x16_t a_hi = vld1q_u8(src_ptr + 16);
|
||||
const uint8x16_t b_lo = vld1q_u8(ref_ptr);
|
||||
const uint8x16_t b_hi = vld1q_u8(ref_ptr + 16);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
abs = vabal_u8(abs, vget_low_u8(a_lo), vget_low_u8(b_lo));
|
||||
abs = vabal_u8(abs, vget_high_u8(a_lo), vget_high_u8(b_lo));
|
||||
abs = vabal_u8(abs, vget_low_u8(a_hi), vget_low_u8(b_hi));
|
||||
abs = vabal_u8(abs, vget_high_u8(a_hi), vget_high_u8(b_hi));
|
||||
}
|
||||
return abs;
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t sad32x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const uint8_t *second_pred,
|
||||
const int height) {
|
||||
int i;
|
||||
uint16x8_t abs = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x16_t a_lo = vld1q_u8(src_ptr);
|
||||
const uint8x16_t a_hi = vld1q_u8(src_ptr + 16);
|
||||
const uint8x16_t b_lo = vld1q_u8(ref_ptr);
|
||||
const uint8x16_t b_hi = vld1q_u8(ref_ptr + 16);
|
||||
const uint8x16_t c_lo = vld1q_u8(second_pred);
|
||||
const uint8x16_t c_hi = vld1q_u8(second_pred + 16);
|
||||
const uint8x16_t avg_lo = vrhaddq_u8(b_lo, c_lo);
|
||||
const uint8x16_t avg_hi = vrhaddq_u8(b_hi, c_hi);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
second_pred += 32;
|
||||
abs = vabal_u8(abs, vget_low_u8(a_lo), vget_low_u8(avg_lo));
|
||||
abs = vabal_u8(abs, vget_high_u8(a_lo), vget_high_u8(avg_lo));
|
||||
abs = vabal_u8(abs, vget_low_u8(a_hi), vget_low_u8(avg_hi));
|
||||
abs = vabal_u8(abs, vget_high_u8(a_hi), vget_high_u8(avg_hi));
|
||||
}
|
||||
return abs;
|
||||
}
|
||||
|
||||
#define sad32xN(n) \
|
||||
uint32_t vpx_sad32x##n##_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride) { \
|
||||
const uint16x8_t abs = \
|
||||
sad32x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
} \
|
||||
\
|
||||
uint32_t vpx_sad32x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride, \
|
||||
const uint8_t *second_pred) { \
|
||||
const uint16x8_t abs = \
|
||||
sad32x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
}
|
||||
|
||||
sad32xN(16);
|
||||
sad32xN(32);
|
||||
sad32xN(64);
|
||||
|
||||
static INLINE uint32x4_t sad64x(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const int height) {
|
||||
int i;
|
||||
uint16x8_t abs_0 = vdupq_n_u16(0);
|
||||
uint16x8_t abs_1 = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x16_t a_0 = vld1q_u8(src_ptr);
|
||||
const uint8x16_t a_1 = vld1q_u8(src_ptr + 16);
|
||||
const uint8x16_t a_2 = vld1q_u8(src_ptr + 32);
|
||||
const uint8x16_t a_3 = vld1q_u8(src_ptr + 48);
|
||||
const uint8x16_t b_0 = vld1q_u8(ref_ptr);
|
||||
const uint8x16_t b_1 = vld1q_u8(ref_ptr + 16);
|
||||
const uint8x16_t b_2 = vld1q_u8(ref_ptr + 32);
|
||||
const uint8x16_t b_3 = vld1q_u8(ref_ptr + 48);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
abs_0 = vabal_u8(abs_0, vget_low_u8(a_0), vget_low_u8(b_0));
|
||||
abs_0 = vabal_u8(abs_0, vget_high_u8(a_0), vget_high_u8(b_0));
|
||||
abs_0 = vabal_u8(abs_0, vget_low_u8(a_1), vget_low_u8(b_1));
|
||||
abs_0 = vabal_u8(abs_0, vget_high_u8(a_1), vget_high_u8(b_1));
|
||||
abs_1 = vabal_u8(abs_1, vget_low_u8(a_2), vget_low_u8(b_2));
|
||||
abs_1 = vabal_u8(abs_1, vget_high_u8(a_2), vget_high_u8(b_2));
|
||||
abs_1 = vabal_u8(abs_1, vget_low_u8(a_3), vget_low_u8(b_3));
|
||||
abs_1 = vabal_u8(abs_1, vget_high_u8(a_3), vget_high_u8(b_3));
|
||||
}
|
||||
|
||||
{
|
||||
const uint32x4_t sum = vpaddlq_u16(abs_0);
|
||||
return vpadalq_u16(sum, abs_1);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE uint32x4_t sad64x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
const uint8_t *second_pred,
|
||||
const int height) {
|
||||
int i;
|
||||
uint16x8_t abs_0 = vdupq_n_u16(0);
|
||||
uint16x8_t abs_1 = vdupq_n_u16(0);
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x16_t a_0 = vld1q_u8(src_ptr);
|
||||
const uint8x16_t a_1 = vld1q_u8(src_ptr + 16);
|
||||
const uint8x16_t a_2 = vld1q_u8(src_ptr + 32);
|
||||
const uint8x16_t a_3 = vld1q_u8(src_ptr + 48);
|
||||
const uint8x16_t b_0 = vld1q_u8(ref_ptr);
|
||||
const uint8x16_t b_1 = vld1q_u8(ref_ptr + 16);
|
||||
const uint8x16_t b_2 = vld1q_u8(ref_ptr + 32);
|
||||
const uint8x16_t b_3 = vld1q_u8(ref_ptr + 48);
|
||||
const uint8x16_t c_0 = vld1q_u8(second_pred);
|
||||
const uint8x16_t c_1 = vld1q_u8(second_pred + 16);
|
||||
const uint8x16_t c_2 = vld1q_u8(second_pred + 32);
|
||||
const uint8x16_t c_3 = vld1q_u8(second_pred + 48);
|
||||
const uint8x16_t avg_0 = vrhaddq_u8(b_0, c_0);
|
||||
const uint8x16_t avg_1 = vrhaddq_u8(b_1, c_1);
|
||||
const uint8x16_t avg_2 = vrhaddq_u8(b_2, c_2);
|
||||
const uint8x16_t avg_3 = vrhaddq_u8(b_3, c_3);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
second_pred += 64;
|
||||
abs_0 = vabal_u8(abs_0, vget_low_u8(a_0), vget_low_u8(avg_0));
|
||||
abs_0 = vabal_u8(abs_0, vget_high_u8(a_0), vget_high_u8(avg_0));
|
||||
abs_0 = vabal_u8(abs_0, vget_low_u8(a_1), vget_low_u8(avg_1));
|
||||
abs_0 = vabal_u8(abs_0, vget_high_u8(a_1), vget_high_u8(avg_1));
|
||||
abs_1 = vabal_u8(abs_1, vget_low_u8(a_2), vget_low_u8(avg_2));
|
||||
abs_1 = vabal_u8(abs_1, vget_high_u8(a_2), vget_high_u8(avg_2));
|
||||
abs_1 = vabal_u8(abs_1, vget_low_u8(a_3), vget_low_u8(avg_3));
|
||||
abs_1 = vabal_u8(abs_1, vget_high_u8(a_3), vget_high_u8(avg_3));
|
||||
}
|
||||
|
||||
{
|
||||
const uint32x4_t sum = vpaddlq_u16(abs_0);
|
||||
return vpadalq_u16(sum, abs_1);
|
||||
}
|
||||
}
|
||||
|
||||
#define sad64xN(n) \
|
||||
uint32_t vpx_sad64x##n##_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride) { \
|
||||
const uint32x4_t abs = \
|
||||
sad64x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
|
||||
return vget_lane_u32(horizontal_add_uint32x4(abs), 0); \
|
||||
} \
|
||||
\
|
||||
uint32_t vpx_sad64x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride, \
|
||||
const uint8_t *second_pred) { \
|
||||
const uint32x4_t abs = \
|
||||
sad64x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
|
||||
return vget_lane_u32(horizontal_add_uint32x4(abs), 0); \
|
||||
}
|
||||
|
||||
sad64xN(32);
|
||||
sad64xN(64);
|
||||
@@ -0,0 +1,34 @@
|
||||
;
|
||||
; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
|
||||
|
||||
EXPORT |vpx_push_neon|
|
||||
EXPORT |vpx_pop_neon|
|
||||
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_push_neon| PROC
|
||||
vstm r0!, {d8-d15}
|
||||
bx lr
|
||||
|
||||
ENDP
|
||||
|
||||
|vpx_pop_neon| PROC
|
||||
vldm r0!, {d8-d15}
|
||||
bx lr
|
||||
|
||||
ENDP
|
||||
|
||||
END
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "./vpx_config.h"
|
||||
|
||||
#include "vpx/vpx_integer.h"
|
||||
|
||||
#include "vpx_dsp/variance.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
|
||||
static const uint8_t bilinear_filters[8][2] = {
|
||||
{ 128, 0 }, { 112, 16 }, { 96, 32 }, { 80, 48 },
|
||||
{ 64, 64 }, { 48, 80 }, { 32, 96 }, { 16, 112 },
|
||||
};
|
||||
|
||||
// Process a block exactly 4 wide and a multiple of 2 high.
|
||||
static void var_filter_block2d_bil_w4(const uint8_t *src_ptr,
|
||||
uint8_t *output_ptr,
|
||||
unsigned int src_pixels_per_line,
|
||||
int pixel_step,
|
||||
unsigned int output_height,
|
||||
const uint8_t *filter) {
|
||||
const uint8x8_t f0 = vdup_n_u8(filter[0]);
|
||||
const uint8x8_t f1 = vdup_n_u8(filter[1]);
|
||||
unsigned int i;
|
||||
for (i = 0; i < output_height; i += 2) {
|
||||
const uint8x8_t src_0 = load_unaligned_u8(src_ptr, src_pixels_per_line);
|
||||
const uint8x8_t src_1 =
|
||||
load_unaligned_u8(src_ptr + pixel_step, src_pixels_per_line);
|
||||
const uint16x8_t a = vmull_u8(src_0, f0);
|
||||
const uint16x8_t b = vmlal_u8(a, src_1, f1);
|
||||
const uint8x8_t out = vrshrn_n_u16(b, FILTER_BITS);
|
||||
vst1_u8(output_ptr, out);
|
||||
src_ptr += 2 * src_pixels_per_line;
|
||||
output_ptr += 8;
|
||||
}
|
||||
}
|
||||
|
||||
// Process a block exactly 8 wide and any height.
|
||||
static void var_filter_block2d_bil_w8(const uint8_t *src_ptr,
|
||||
uint8_t *output_ptr,
|
||||
unsigned int src_pixels_per_line,
|
||||
int pixel_step,
|
||||
unsigned int output_height,
|
||||
const uint8_t *filter) {
|
||||
const uint8x8_t f0 = vdup_n_u8(filter[0]);
|
||||
const uint8x8_t f1 = vdup_n_u8(filter[1]);
|
||||
unsigned int i;
|
||||
for (i = 0; i < output_height; ++i) {
|
||||
const uint8x8_t src_0 = vld1_u8(&src_ptr[0]);
|
||||
const uint8x8_t src_1 = vld1_u8(&src_ptr[pixel_step]);
|
||||
const uint16x8_t a = vmull_u8(src_0, f0);
|
||||
const uint16x8_t b = vmlal_u8(a, src_1, f1);
|
||||
const uint8x8_t out = vrshrn_n_u16(b, FILTER_BITS);
|
||||
vst1_u8(output_ptr, out);
|
||||
src_ptr += src_pixels_per_line;
|
||||
output_ptr += 8;
|
||||
}
|
||||
}
|
||||
|
||||
// Process a block which is a mutiple of 16 wide and any height.
|
||||
static void var_filter_block2d_bil_w16(const uint8_t *src_ptr,
|
||||
uint8_t *output_ptr,
|
||||
unsigned int src_pixels_per_line,
|
||||
int pixel_step,
|
||||
unsigned int output_height,
|
||||
unsigned int output_width,
|
||||
const uint8_t *filter) {
|
||||
const uint8x8_t f0 = vdup_n_u8(filter[0]);
|
||||
const uint8x8_t f1 = vdup_n_u8(filter[1]);
|
||||
unsigned int i, j;
|
||||
for (i = 0; i < output_height; ++i) {
|
||||
for (j = 0; j < output_width; j += 16) {
|
||||
const uint8x16_t src_0 = vld1q_u8(&src_ptr[j]);
|
||||
const uint8x16_t src_1 = vld1q_u8(&src_ptr[j + pixel_step]);
|
||||
const uint16x8_t a = vmull_u8(vget_low_u8(src_0), f0);
|
||||
const uint16x8_t b = vmlal_u8(a, vget_low_u8(src_1), f1);
|
||||
const uint8x8_t out_lo = vrshrn_n_u16(b, FILTER_BITS);
|
||||
const uint16x8_t c = vmull_u8(vget_high_u8(src_0), f0);
|
||||
const uint16x8_t d = vmlal_u8(c, vget_high_u8(src_1), f1);
|
||||
const uint8x8_t out_hi = vrshrn_n_u16(d, FILTER_BITS);
|
||||
vst1q_u8(output_ptr + j, vcombine_u8(out_lo, out_hi));
|
||||
}
|
||||
src_ptr += src_pixels_per_line;
|
||||
output_ptr += output_width;
|
||||
}
|
||||
}
|
||||
|
||||
// 4xM filter writes an extra row to fdata because it processes two rows at a
|
||||
// time.
|
||||
#define sub_pixel_varianceNxM(n, m) \
|
||||
uint32_t vpx_sub_pixel_variance##n##x##m##_neon( \
|
||||
const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \
|
||||
const uint8_t *ref_ptr, int ref_stride, uint32_t *sse) { \
|
||||
uint8_t temp0[n * (m + (n == 4 ? 2 : 1))]; \
|
||||
uint8_t temp1[n * m]; \
|
||||
\
|
||||
if (n == 4) { \
|
||||
var_filter_block2d_bil_w4(src_ptr, temp0, src_stride, 1, (m + 2), \
|
||||
bilinear_filters[x_offset]); \
|
||||
var_filter_block2d_bil_w4(temp0, temp1, n, n, m, \
|
||||
bilinear_filters[y_offset]); \
|
||||
} else if (n == 8) { \
|
||||
var_filter_block2d_bil_w8(src_ptr, temp0, src_stride, 1, (m + 1), \
|
||||
bilinear_filters[x_offset]); \
|
||||
var_filter_block2d_bil_w8(temp0, temp1, n, n, m, \
|
||||
bilinear_filters[y_offset]); \
|
||||
} else { \
|
||||
var_filter_block2d_bil_w16(src_ptr, temp0, src_stride, 1, (m + 1), n, \
|
||||
bilinear_filters[x_offset]); \
|
||||
var_filter_block2d_bil_w16(temp0, temp1, n, n, m, n, \
|
||||
bilinear_filters[y_offset]); \
|
||||
} \
|
||||
return vpx_variance##n##x##m(temp1, n, ref_ptr, ref_stride, sse); \
|
||||
}
|
||||
|
||||
sub_pixel_varianceNxM(4, 4);
|
||||
sub_pixel_varianceNxM(4, 8);
|
||||
sub_pixel_varianceNxM(8, 4);
|
||||
sub_pixel_varianceNxM(8, 8);
|
||||
sub_pixel_varianceNxM(8, 16);
|
||||
sub_pixel_varianceNxM(16, 8);
|
||||
sub_pixel_varianceNxM(16, 16);
|
||||
sub_pixel_varianceNxM(16, 32);
|
||||
sub_pixel_varianceNxM(32, 16);
|
||||
sub_pixel_varianceNxM(32, 32);
|
||||
sub_pixel_varianceNxM(32, 64);
|
||||
sub_pixel_varianceNxM(64, 32);
|
||||
sub_pixel_varianceNxM(64, 64);
|
||||
|
||||
// 4xM filter writes an extra row to fdata because it processes two rows at a
|
||||
// time.
|
||||
#define sub_pixel_avg_varianceNxM(n, m) \
|
||||
uint32_t vpx_sub_pixel_avg_variance##n##x##m##_neon( \
|
||||
const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \
|
||||
const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, \
|
||||
const uint8_t *second_pred) { \
|
||||
uint8_t temp0[n * (m + (n == 4 ? 2 : 1))]; \
|
||||
uint8_t temp1[n * m]; \
|
||||
\
|
||||
if (n == 4) { \
|
||||
var_filter_block2d_bil_w4(src_ptr, temp0, src_stride, 1, (m + 2), \
|
||||
bilinear_filters[x_offset]); \
|
||||
var_filter_block2d_bil_w4(temp0, temp1, n, n, m, \
|
||||
bilinear_filters[y_offset]); \
|
||||
} else if (n == 8) { \
|
||||
var_filter_block2d_bil_w8(src_ptr, temp0, src_stride, 1, (m + 1), \
|
||||
bilinear_filters[x_offset]); \
|
||||
var_filter_block2d_bil_w8(temp0, temp1, n, n, m, \
|
||||
bilinear_filters[y_offset]); \
|
||||
} else { \
|
||||
var_filter_block2d_bil_w16(src_ptr, temp0, src_stride, 1, (m + 1), n, \
|
||||
bilinear_filters[x_offset]); \
|
||||
var_filter_block2d_bil_w16(temp0, temp1, n, n, m, n, \
|
||||
bilinear_filters[y_offset]); \
|
||||
} \
|
||||
\
|
||||
vpx_comp_avg_pred(temp0, second_pred, n, m, temp1, n); \
|
||||
\
|
||||
return vpx_variance##n##x##m(temp0, n, ref_ptr, ref_stride, sse); \
|
||||
}
|
||||
|
||||
sub_pixel_avg_varianceNxM(4, 4);
|
||||
sub_pixel_avg_varianceNxM(4, 8);
|
||||
sub_pixel_avg_varianceNxM(8, 4);
|
||||
sub_pixel_avg_varianceNxM(8, 8);
|
||||
sub_pixel_avg_varianceNxM(8, 16);
|
||||
sub_pixel_avg_varianceNxM(16, 8);
|
||||
sub_pixel_avg_varianceNxM(16, 16);
|
||||
sub_pixel_avg_varianceNxM(16, 32);
|
||||
sub_pixel_avg_varianceNxM(32, 16);
|
||||
sub_pixel_avg_varianceNxM(32, 32);
|
||||
sub_pixel_avg_varianceNxM(32, 64);
|
||||
sub_pixel_avg_varianceNxM(64, 32);
|
||||
sub_pixel_avg_varianceNxM(64, 64);
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
|
||||
void vpx_subtract_block_neon(int rows, int cols, int16_t *diff,
|
||||
ptrdiff_t diff_stride, const uint8_t *src,
|
||||
ptrdiff_t src_stride, const uint8_t *pred,
|
||||
ptrdiff_t pred_stride) {
|
||||
int r = rows, c;
|
||||
|
||||
if (cols > 16) {
|
||||
do {
|
||||
for (c = 0; c < cols; c += 32) {
|
||||
const uint8x16_t s0 = vld1q_u8(&src[c + 0]);
|
||||
const uint8x16_t s1 = vld1q_u8(&src[c + 16]);
|
||||
const uint8x16_t p0 = vld1q_u8(&pred[c + 0]);
|
||||
const uint8x16_t p1 = vld1q_u8(&pred[c + 16]);
|
||||
const uint16x8_t d0 = vsubl_u8(vget_low_u8(s0), vget_low_u8(p0));
|
||||
const uint16x8_t d1 = vsubl_u8(vget_high_u8(s0), vget_high_u8(p0));
|
||||
const uint16x8_t d2 = vsubl_u8(vget_low_u8(s1), vget_low_u8(p1));
|
||||
const uint16x8_t d3 = vsubl_u8(vget_high_u8(s1), vget_high_u8(p1));
|
||||
vst1q_s16(&diff[c + 0], vreinterpretq_s16_u16(d0));
|
||||
vst1q_s16(&diff[c + 8], vreinterpretq_s16_u16(d1));
|
||||
vst1q_s16(&diff[c + 16], vreinterpretq_s16_u16(d2));
|
||||
vst1q_s16(&diff[c + 24], vreinterpretq_s16_u16(d3));
|
||||
}
|
||||
diff += diff_stride;
|
||||
pred += pred_stride;
|
||||
src += src_stride;
|
||||
} while (--r);
|
||||
} else if (cols > 8) {
|
||||
do {
|
||||
const uint8x16_t s = vld1q_u8(&src[0]);
|
||||
const uint8x16_t p = vld1q_u8(&pred[0]);
|
||||
const uint16x8_t d0 = vsubl_u8(vget_low_u8(s), vget_low_u8(p));
|
||||
const uint16x8_t d1 = vsubl_u8(vget_high_u8(s), vget_high_u8(p));
|
||||
vst1q_s16(&diff[0], vreinterpretq_s16_u16(d0));
|
||||
vst1q_s16(&diff[8], vreinterpretq_s16_u16(d1));
|
||||
diff += diff_stride;
|
||||
pred += pred_stride;
|
||||
src += src_stride;
|
||||
} while (--r);
|
||||
} else if (cols > 4) {
|
||||
do {
|
||||
const uint8x8_t s = vld1_u8(&src[0]);
|
||||
const uint8x8_t p = vld1_u8(&pred[0]);
|
||||
const uint16x8_t v_diff = vsubl_u8(s, p);
|
||||
vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff));
|
||||
diff += diff_stride;
|
||||
pred += pred_stride;
|
||||
src += src_stride;
|
||||
} while (--r);
|
||||
} else {
|
||||
assert(cols == 4);
|
||||
do {
|
||||
const uint8x8_t s = load_unaligned_u8(src, (int)src_stride);
|
||||
const uint8x8_t p = load_unaligned_u8(pred, (int)pred_stride);
|
||||
const uint16x8_t d = vsubl_u8(s, p);
|
||||
vst1_s16(diff + 0 * diff_stride, vreinterpret_s16_u16(vget_low_u16(d)));
|
||||
vst1_s16(diff + 1 * diff_stride, vreinterpret_s16_u16(vget_high_u16(d)));
|
||||
diff += 2 * diff_stride;
|
||||
pred += 2 * pred_stride;
|
||||
src += 2 * src_stride;
|
||||
r -= 2;
|
||||
} while (r);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_ARM_SUM_NEON_H_
|
||||
#define VPX_VPX_DSP_ARM_SUM_NEON_H_
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
|
||||
static INLINE int32x2_t horizontal_add_int16x8(const int16x8_t a) {
|
||||
const int32x4_t b = vpaddlq_s16(a);
|
||||
const int64x2_t c = vpaddlq_s32(b);
|
||||
return vadd_s32(vreinterpret_s32_s64(vget_low_s64(c)),
|
||||
vreinterpret_s32_s64(vget_high_s64(c)));
|
||||
}
|
||||
|
||||
static INLINE uint32x2_t horizontal_add_uint16x8(const uint16x8_t a) {
|
||||
const uint32x4_t b = vpaddlq_u16(a);
|
||||
const uint64x2_t c = vpaddlq_u32(b);
|
||||
return vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
|
||||
vreinterpret_u32_u64(vget_high_u64(c)));
|
||||
}
|
||||
|
||||
static INLINE uint32x2_t horizontal_add_uint32x4(const uint32x4_t a) {
|
||||
const uint64x2_t b = vpaddlq_u32(a);
|
||||
return vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)),
|
||||
vreinterpret_u32_u64(vget_high_u64(b)));
|
||||
}
|
||||
#endif // VPX_VPX_DSP_ARM_SUM_NEON_H_
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include <assert.h>
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
|
||||
uint64_t vpx_sum_squares_2d_i16_neon(const int16_t *src, int stride, int size) {
|
||||
uint64x1_t s2;
|
||||
|
||||
if (size == 4) {
|
||||
int16x4_t s[4];
|
||||
int32x4_t s0;
|
||||
uint32x2_t s1;
|
||||
|
||||
s[0] = vld1_s16(src + 0 * stride);
|
||||
s[1] = vld1_s16(src + 1 * stride);
|
||||
s[2] = vld1_s16(src + 2 * stride);
|
||||
s[3] = vld1_s16(src + 3 * stride);
|
||||
s0 = vmull_s16(s[0], s[0]);
|
||||
s0 = vmlal_s16(s0, s[1], s[1]);
|
||||
s0 = vmlal_s16(s0, s[2], s[2]);
|
||||
s0 = vmlal_s16(s0, s[3], s[3]);
|
||||
s1 = vpadd_u32(vget_low_u32(vreinterpretq_u32_s32(s0)),
|
||||
vget_high_u32(vreinterpretq_u32_s32(s0)));
|
||||
s2 = vpaddl_u32(s1);
|
||||
} else {
|
||||
int r = size;
|
||||
uint64x2_t s1 = vdupq_n_u64(0);
|
||||
|
||||
do {
|
||||
int c = size;
|
||||
int32x4_t s0 = vdupq_n_s32(0);
|
||||
const int16_t *src_t = src;
|
||||
|
||||
do {
|
||||
int16x8_t s[8];
|
||||
|
||||
s[0] = vld1q_s16(src_t + 0 * stride);
|
||||
s[1] = vld1q_s16(src_t + 1 * stride);
|
||||
s[2] = vld1q_s16(src_t + 2 * stride);
|
||||
s[3] = vld1q_s16(src_t + 3 * stride);
|
||||
s[4] = vld1q_s16(src_t + 4 * stride);
|
||||
s[5] = vld1q_s16(src_t + 5 * stride);
|
||||
s[6] = vld1q_s16(src_t + 6 * stride);
|
||||
s[7] = vld1q_s16(src_t + 7 * stride);
|
||||
s0 = vmlal_s16(s0, vget_low_s16(s[0]), vget_low_s16(s[0]));
|
||||
s0 = vmlal_s16(s0, vget_low_s16(s[1]), vget_low_s16(s[1]));
|
||||
s0 = vmlal_s16(s0, vget_low_s16(s[2]), vget_low_s16(s[2]));
|
||||
s0 = vmlal_s16(s0, vget_low_s16(s[3]), vget_low_s16(s[3]));
|
||||
s0 = vmlal_s16(s0, vget_low_s16(s[4]), vget_low_s16(s[4]));
|
||||
s0 = vmlal_s16(s0, vget_low_s16(s[5]), vget_low_s16(s[5]));
|
||||
s0 = vmlal_s16(s0, vget_low_s16(s[6]), vget_low_s16(s[6]));
|
||||
s0 = vmlal_s16(s0, vget_low_s16(s[7]), vget_low_s16(s[7]));
|
||||
s0 = vmlal_s16(s0, vget_high_s16(s[0]), vget_high_s16(s[0]));
|
||||
s0 = vmlal_s16(s0, vget_high_s16(s[1]), vget_high_s16(s[1]));
|
||||
s0 = vmlal_s16(s0, vget_high_s16(s[2]), vget_high_s16(s[2]));
|
||||
s0 = vmlal_s16(s0, vget_high_s16(s[3]), vget_high_s16(s[3]));
|
||||
s0 = vmlal_s16(s0, vget_high_s16(s[4]), vget_high_s16(s[4]));
|
||||
s0 = vmlal_s16(s0, vget_high_s16(s[5]), vget_high_s16(s[5]));
|
||||
s0 = vmlal_s16(s0, vget_high_s16(s[6]), vget_high_s16(s[6]));
|
||||
s0 = vmlal_s16(s0, vget_high_s16(s[7]), vget_high_s16(s[7]));
|
||||
src_t += 8;
|
||||
c -= 8;
|
||||
} while (c);
|
||||
|
||||
s1 = vaddw_u32(s1, vget_low_u32(vreinterpretq_u32_s32(s0)));
|
||||
s1 = vaddw_u32(s1, vget_high_u32(vreinterpretq_u32_s32(s0)));
|
||||
src += 8 * stride;
|
||||
r -= 8;
|
||||
} while (r);
|
||||
|
||||
s2 = vadd_u64(vget_low_u64(s1), vget_high_u64(s1));
|
||||
}
|
||||
|
||||
return vget_lane_u64(s2, 0);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,380 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "./vpx_config.h"
|
||||
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/sum_neon.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
|
||||
// The variance helper functions use int16_t for sum. 8 values are accumulated
|
||||
// and then added (at which point they expand up to int32_t). To avoid overflow,
|
||||
// there can be no more than 32767 / 255 ~= 128 values accumulated in each
|
||||
// column. For a 32x32 buffer, this results in 32 / 8 = 4 values per row * 32
|
||||
// rows = 128. Asserts have been added to each function to warn against reaching
|
||||
// this limit.
|
||||
|
||||
// Process a block of width 4 four rows at a time.
|
||||
static void variance_neon_w4x4(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride, int h,
|
||||
uint32_t *sse, int *sum) {
|
||||
int i;
|
||||
int16x8_t sum_s16 = vdupq_n_s16(0);
|
||||
int32x4_t sse_lo_s32 = vdupq_n_s32(0);
|
||||
int32x4_t sse_hi_s32 = vdupq_n_s32(0);
|
||||
|
||||
// Since width is only 4, sum_s16 only loads a half row per loop.
|
||||
assert(h <= 256);
|
||||
|
||||
for (i = 0; i < h; i += 4) {
|
||||
const uint8x16_t a_u8 = load_unaligned_u8q(src_ptr, src_stride);
|
||||
const uint8x16_t b_u8 = load_unaligned_u8q(ref_ptr, ref_stride);
|
||||
const uint16x8_t diff_lo_u16 =
|
||||
vsubl_u8(vget_low_u8(a_u8), vget_low_u8(b_u8));
|
||||
const uint16x8_t diff_hi_u16 =
|
||||
vsubl_u8(vget_high_u8(a_u8), vget_high_u8(b_u8));
|
||||
|
||||
const int16x8_t diff_lo_s16 = vreinterpretq_s16_u16(diff_lo_u16);
|
||||
const int16x8_t diff_hi_s16 = vreinterpretq_s16_u16(diff_hi_u16);
|
||||
|
||||
sum_s16 = vaddq_s16(sum_s16, diff_lo_s16);
|
||||
sum_s16 = vaddq_s16(sum_s16, diff_hi_s16);
|
||||
|
||||
sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_lo_s16),
|
||||
vget_low_s16(diff_lo_s16));
|
||||
sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_high_s16(diff_lo_s16),
|
||||
vget_high_s16(diff_lo_s16));
|
||||
|
||||
sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_low_s16(diff_hi_s16),
|
||||
vget_low_s16(diff_hi_s16));
|
||||
sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_hi_s16),
|
||||
vget_high_s16(diff_hi_s16));
|
||||
|
||||
src_ptr += 4 * src_stride;
|
||||
ref_ptr += 4 * ref_stride;
|
||||
}
|
||||
|
||||
*sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
|
||||
*sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
|
||||
vaddq_s32(sse_lo_s32, sse_hi_s32))),
|
||||
0);
|
||||
}
|
||||
|
||||
// Process a block of any size where the width is divisible by 16.
|
||||
static void variance_neon_w16(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride, int w,
|
||||
int h, uint32_t *sse, int *sum) {
|
||||
int i, j;
|
||||
int16x8_t sum_s16 = vdupq_n_s16(0);
|
||||
int32x4_t sse_lo_s32 = vdupq_n_s32(0);
|
||||
int32x4_t sse_hi_s32 = vdupq_n_s32(0);
|
||||
|
||||
// The loop loads 16 values at a time but doubles them up when accumulating
|
||||
// into sum_s16.
|
||||
assert(w / 8 * h <= 128);
|
||||
|
||||
for (i = 0; i < h; ++i) {
|
||||
for (j = 0; j < w; j += 16) {
|
||||
const uint8x16_t a_u8 = vld1q_u8(src_ptr + j);
|
||||
const uint8x16_t b_u8 = vld1q_u8(ref_ptr + j);
|
||||
|
||||
const uint16x8_t diff_lo_u16 =
|
||||
vsubl_u8(vget_low_u8(a_u8), vget_low_u8(b_u8));
|
||||
const uint16x8_t diff_hi_u16 =
|
||||
vsubl_u8(vget_high_u8(a_u8), vget_high_u8(b_u8));
|
||||
|
||||
const int16x8_t diff_lo_s16 = vreinterpretq_s16_u16(diff_lo_u16);
|
||||
const int16x8_t diff_hi_s16 = vreinterpretq_s16_u16(diff_hi_u16);
|
||||
|
||||
sum_s16 = vaddq_s16(sum_s16, diff_lo_s16);
|
||||
sum_s16 = vaddq_s16(sum_s16, diff_hi_s16);
|
||||
|
||||
sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_lo_s16),
|
||||
vget_low_s16(diff_lo_s16));
|
||||
sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_high_s16(diff_lo_s16),
|
||||
vget_high_s16(diff_lo_s16));
|
||||
|
||||
sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_low_s16(diff_hi_s16),
|
||||
vget_low_s16(diff_hi_s16));
|
||||
sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_hi_s16),
|
||||
vget_high_s16(diff_hi_s16));
|
||||
}
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
}
|
||||
|
||||
*sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
|
||||
*sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
|
||||
vaddq_s32(sse_lo_s32, sse_hi_s32))),
|
||||
0);
|
||||
}
|
||||
|
||||
// Process a block of width 8 two rows at a time.
|
||||
static void variance_neon_w8x2(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride, int h,
|
||||
uint32_t *sse, int *sum) {
|
||||
int i = 0;
|
||||
int16x8_t sum_s16 = vdupq_n_s16(0);
|
||||
int32x4_t sse_lo_s32 = vdupq_n_s32(0);
|
||||
int32x4_t sse_hi_s32 = vdupq_n_s32(0);
|
||||
|
||||
// Each column has it's own accumulator entry in sum_s16.
|
||||
assert(h <= 128);
|
||||
|
||||
do {
|
||||
const uint8x8_t a_0_u8 = vld1_u8(src_ptr);
|
||||
const uint8x8_t a_1_u8 = vld1_u8(src_ptr + src_stride);
|
||||
const uint8x8_t b_0_u8 = vld1_u8(ref_ptr);
|
||||
const uint8x8_t b_1_u8 = vld1_u8(ref_ptr + ref_stride);
|
||||
const uint16x8_t diff_0_u16 = vsubl_u8(a_0_u8, b_0_u8);
|
||||
const uint16x8_t diff_1_u16 = vsubl_u8(a_1_u8, b_1_u8);
|
||||
const int16x8_t diff_0_s16 = vreinterpretq_s16_u16(diff_0_u16);
|
||||
const int16x8_t diff_1_s16 = vreinterpretq_s16_u16(diff_1_u16);
|
||||
sum_s16 = vaddq_s16(sum_s16, diff_0_s16);
|
||||
sum_s16 = vaddq_s16(sum_s16, diff_1_s16);
|
||||
sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_0_s16),
|
||||
vget_low_s16(diff_0_s16));
|
||||
sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_1_s16),
|
||||
vget_low_s16(diff_1_s16));
|
||||
sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_0_s16),
|
||||
vget_high_s16(diff_0_s16));
|
||||
sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_1_s16),
|
||||
vget_high_s16(diff_1_s16));
|
||||
src_ptr += src_stride + src_stride;
|
||||
ref_ptr += ref_stride + ref_stride;
|
||||
i += 2;
|
||||
} while (i < h);
|
||||
|
||||
*sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
|
||||
*sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
|
||||
vaddq_s32(sse_lo_s32, sse_hi_s32))),
|
||||
0);
|
||||
}
|
||||
|
||||
void vpx_get8x8var_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
unsigned int *sse, int *sum) {
|
||||
variance_neon_w8x2(src_ptr, src_stride, ref_ptr, ref_stride, 8, sse, sum);
|
||||
}
|
||||
|
||||
void vpx_get16x16var_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
unsigned int *sse, int *sum) {
|
||||
variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 16, 16, sse, sum);
|
||||
}
|
||||
|
||||
#define varianceNxM(n, m, shift) \
|
||||
unsigned int vpx_variance##n##x##m##_neon( \
|
||||
const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
|
||||
int ref_stride, unsigned int *sse) { \
|
||||
int sum; \
|
||||
if (n == 4) \
|
||||
variance_neon_w4x4(src_ptr, src_stride, ref_ptr, ref_stride, m, sse, \
|
||||
&sum); \
|
||||
else if (n == 8) \
|
||||
variance_neon_w8x2(src_ptr, src_stride, ref_ptr, ref_stride, m, sse, \
|
||||
&sum); \
|
||||
else \
|
||||
variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, n, m, sse, \
|
||||
&sum); \
|
||||
if (n * m < 16 * 16) \
|
||||
return *sse - ((sum * sum) >> shift); \
|
||||
else \
|
||||
return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \
|
||||
}
|
||||
|
||||
varianceNxM(4, 4, 4);
|
||||
varianceNxM(4, 8, 5);
|
||||
varianceNxM(8, 4, 5);
|
||||
varianceNxM(8, 8, 6);
|
||||
varianceNxM(8, 16, 7);
|
||||
varianceNxM(16, 8, 7);
|
||||
varianceNxM(16, 16, 8);
|
||||
varianceNxM(16, 32, 9);
|
||||
varianceNxM(32, 16, 9);
|
||||
varianceNxM(32, 32, 10);
|
||||
|
||||
unsigned int vpx_variance32x64_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
unsigned int *sse) {
|
||||
int sum1, sum2;
|
||||
uint32_t sse1, sse2;
|
||||
variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 32, 32, &sse1,
|
||||
&sum1);
|
||||
variance_neon_w16(src_ptr + (32 * src_stride), src_stride,
|
||||
ref_ptr + (32 * ref_stride), ref_stride, 32, 32, &sse2,
|
||||
&sum2);
|
||||
*sse = sse1 + sse2;
|
||||
sum1 += sum2;
|
||||
return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 11);
|
||||
}
|
||||
|
||||
unsigned int vpx_variance64x32_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
unsigned int *sse) {
|
||||
int sum1, sum2;
|
||||
uint32_t sse1, sse2;
|
||||
variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 64, 16, &sse1,
|
||||
&sum1);
|
||||
variance_neon_w16(src_ptr + (16 * src_stride), src_stride,
|
||||
ref_ptr + (16 * ref_stride), ref_stride, 64, 16, &sse2,
|
||||
&sum2);
|
||||
*sse = sse1 + sse2;
|
||||
sum1 += sum2;
|
||||
return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 11);
|
||||
}
|
||||
|
||||
unsigned int vpx_variance64x64_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
unsigned int *sse) {
|
||||
int sum1, sum2;
|
||||
uint32_t sse1, sse2;
|
||||
|
||||
variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 64, 16, &sse1,
|
||||
&sum1);
|
||||
variance_neon_w16(src_ptr + (16 * src_stride), src_stride,
|
||||
ref_ptr + (16 * ref_stride), ref_stride, 64, 16, &sse2,
|
||||
&sum2);
|
||||
sse1 += sse2;
|
||||
sum1 += sum2;
|
||||
|
||||
variance_neon_w16(src_ptr + (16 * 2 * src_stride), src_stride,
|
||||
ref_ptr + (16 * 2 * ref_stride), ref_stride, 64, 16, &sse2,
|
||||
&sum2);
|
||||
sse1 += sse2;
|
||||
sum1 += sum2;
|
||||
|
||||
variance_neon_w16(src_ptr + (16 * 3 * src_stride), src_stride,
|
||||
ref_ptr + (16 * 3 * ref_stride), ref_stride, 64, 16, &sse2,
|
||||
&sum2);
|
||||
*sse = sse1 + sse2;
|
||||
sum1 += sum2;
|
||||
return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 12);
|
||||
}
|
||||
|
||||
unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride,
|
||||
const unsigned char *ref_ptr, int ref_stride,
|
||||
unsigned int *sse) {
|
||||
int i;
|
||||
int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
|
||||
int64x1_t d0s64;
|
||||
uint8x16_t q0u8, q1u8, q2u8, q3u8;
|
||||
int32x4_t q7s32, q8s32, q9s32, q10s32;
|
||||
uint16x8_t q11u16, q12u16, q13u16, q14u16;
|
||||
int64x2_t q1s64;
|
||||
|
||||
q7s32 = vdupq_n_s32(0);
|
||||
q8s32 = vdupq_n_s32(0);
|
||||
q9s32 = vdupq_n_s32(0);
|
||||
q10s32 = vdupq_n_s32(0);
|
||||
|
||||
for (i = 0; i < 8; i++) { // mse16x16_neon_loop
|
||||
q0u8 = vld1q_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
q1u8 = vld1q_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
q2u8 = vld1q_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
q3u8 = vld1q_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
|
||||
q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
|
||||
q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
|
||||
q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
|
||||
q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
|
||||
|
||||
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
|
||||
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
|
||||
q7s32 = vmlal_s16(q7s32, d22s16, d22s16);
|
||||
q8s32 = vmlal_s16(q8s32, d23s16, d23s16);
|
||||
|
||||
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
|
||||
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
|
||||
q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
|
||||
q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
|
||||
|
||||
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
|
||||
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
|
||||
q7s32 = vmlal_s16(q7s32, d26s16, d26s16);
|
||||
q8s32 = vmlal_s16(q8s32, d27s16, d27s16);
|
||||
|
||||
d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
|
||||
d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
|
||||
q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
|
||||
q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
|
||||
}
|
||||
|
||||
q7s32 = vaddq_s32(q7s32, q8s32);
|
||||
q9s32 = vaddq_s32(q9s32, q10s32);
|
||||
q10s32 = vaddq_s32(q7s32, q9s32);
|
||||
|
||||
q1s64 = vpaddlq_s32(q10s32);
|
||||
d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
|
||||
|
||||
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d0s64), 0);
|
||||
return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
|
||||
}
|
||||
|
||||
unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride,
|
||||
const unsigned char *ref_ptr,
|
||||
int ref_stride) {
|
||||
int16x4_t d22s16, d24s16, d26s16, d28s16;
|
||||
int64x1_t d0s64;
|
||||
uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8;
|
||||
int32x4_t q7s32, q8s32, q9s32, q10s32;
|
||||
uint16x8_t q11u16, q12u16, q13u16, q14u16;
|
||||
int64x2_t q1s64;
|
||||
|
||||
d0u8 = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
d4u8 = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
d1u8 = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
d5u8 = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
d2u8 = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
d6u8 = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
d3u8 = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
d7u8 = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
|
||||
q11u16 = vsubl_u8(d0u8, d4u8);
|
||||
q12u16 = vsubl_u8(d1u8, d5u8);
|
||||
q13u16 = vsubl_u8(d2u8, d6u8);
|
||||
q14u16 = vsubl_u8(d3u8, d7u8);
|
||||
|
||||
d22s16 = vget_low_s16(vreinterpretq_s16_u16(q11u16));
|
||||
d24s16 = vget_low_s16(vreinterpretq_s16_u16(q12u16));
|
||||
d26s16 = vget_low_s16(vreinterpretq_s16_u16(q13u16));
|
||||
d28s16 = vget_low_s16(vreinterpretq_s16_u16(q14u16));
|
||||
|
||||
q7s32 = vmull_s16(d22s16, d22s16);
|
||||
q8s32 = vmull_s16(d24s16, d24s16);
|
||||
q9s32 = vmull_s16(d26s16, d26s16);
|
||||
q10s32 = vmull_s16(d28s16, d28s16);
|
||||
|
||||
q7s32 = vaddq_s32(q7s32, q8s32);
|
||||
q9s32 = vaddq_s32(q9s32, q10s32);
|
||||
q9s32 = vaddq_s32(q7s32, q9s32);
|
||||
|
||||
q1s64 = vpaddlq_s32(q9s32);
|
||||
d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
|
||||
|
||||
return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
|
||||
}
|
||||
+438
@@ -0,0 +1,438 @@
|
||||
;
|
||||
; Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
;**************Variables Vs Registers*****************************************
|
||||
; r0 => src
|
||||
; r1 => dst
|
||||
; r2 => src_stride
|
||||
; r3 => dst_stride
|
||||
; r4 => filter_x0
|
||||
; r8 => ht
|
||||
; r10 => wd
|
||||
|
||||
EXPORT |vpx_convolve8_avg_horiz_filter_type1_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve8_avg_horiz_filter_type1_neon| PROC
|
||||
|
||||
stmfd sp!, {r4 - r12, r14} ;stack stores the values of
|
||||
; the arguments
|
||||
vpush {d8 - d15} ; stack offset by 64
|
||||
mov r4, r1
|
||||
mov r1, r2
|
||||
mov r2, r4
|
||||
|
||||
start_loop_count
|
||||
ldr r4, [sp, #104] ;loads pi1_coeff
|
||||
ldr r8, [sp, #108] ;loads x0_q4
|
||||
add r4, r4, r8, lsl #4 ;r4 = filter[x0_q4]
|
||||
ldr r8, [sp, #128] ;loads ht
|
||||
ldr r10, [sp, #124] ;loads wd
|
||||
vld2.8 {d0, d1}, [r4] ;coeff = vld1_s8(pi1_coeff)
|
||||
mov r11, #1
|
||||
subs r14, r8, #0 ;checks for ht == 0
|
||||
vabs.s8 d2, d0 ;vabs_s8(coeff)
|
||||
vdup.8 d24, d2[0] ;coeffabs_0 = vdup_lane_u8(coeffabs,
|
||||
; 0)
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
vdup.8 d25, d2[1] ;coeffabs_1 = vdup_lane_u8(coeffabs,
|
||||
; 1)
|
||||
add r4, r12, r2 ;pu1_src_tmp2_8 = pu1_src + src_strd
|
||||
vdup.8 d26, d2[2] ;coeffabs_2 = vdup_lane_u8(coeffabs,
|
||||
; 2)
|
||||
rsb r9, r10, r2, lsl #1 ;2*src_strd - wd
|
||||
vdup.8 d27, d2[3] ;coeffabs_3 = vdup_lane_u8(coeffabs,
|
||||
; 3)
|
||||
rsb r8, r10, r3, lsl #1 ;2*dst_strd - wd
|
||||
vdup.8 d28, d2[4] ;coeffabs_4 = vdup_lane_u8(coeffabs,
|
||||
; 4)
|
||||
vdup.8 d29, d2[5] ;coeffabs_5 = vdup_lane_u8(coeffabs,
|
||||
; 5)
|
||||
vdup.8 d30, d2[6] ;coeffabs_6 = vdup_lane_u8(coeffabs,
|
||||
; 6)
|
||||
vdup.8 d31, d2[7] ;coeffabs_7 = vdup_lane_u8(coeffabs,
|
||||
; 7)
|
||||
mov r7, r1
|
||||
cmp r10, #4
|
||||
ble outer_loop_4
|
||||
|
||||
cmp r10, #24
|
||||
moveq r10, #16
|
||||
addeq r8, #8
|
||||
addeq r9, #8
|
||||
cmp r10, #16
|
||||
bge outer_loop_16
|
||||
|
||||
cmp r10, #12
|
||||
addeq r8, #4
|
||||
addeq r9, #4
|
||||
b outer_loop_8
|
||||
|
||||
outer_loop8_residual
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
mov r1, r7
|
||||
mov r14, #32
|
||||
add r1, #16
|
||||
add r12, #16
|
||||
mov r10, #8
|
||||
add r8, #8
|
||||
add r9, #8
|
||||
|
||||
outer_loop_8
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
subs r5, r10, #0 ;checks wd
|
||||
ble end_inner_loop_8
|
||||
|
||||
inner_loop_8
|
||||
mov r7, #0xc000
|
||||
vld1.u32 {d0}, [r12], r11 ;vector load pu1_src
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {d1}, [r12], r11
|
||||
vdup.16 q5, r7
|
||||
vld1.u32 {d2}, [r12], r11
|
||||
vld1.u32 {d3}, [r12], r11
|
||||
mov r7, #0x4000
|
||||
vld1.u32 {d4}, [r12], r11
|
||||
vmlsl.u8 q4, d1, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {d5}, [r12], r11
|
||||
vmlal.u8 q4, d3, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {d6}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {d7}, [r12], r11
|
||||
vmlal.u8 q4, d2, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {d12}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vmlal.u8 q4, d4, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vld1.u32 {d13}, [r4], r11
|
||||
vmlal.u8 q4, d5, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vld1.u32 {d14}, [r4], r11
|
||||
vmlsl.u8 q4, d6, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vld1.u32 {d15}, [r4], r11
|
||||
vmlsl.u8 q4, d7, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
vld1.u32 {d16}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vdup.16 q11, r7
|
||||
vmlal.u8 q5, d15, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {d17}, [r4], r11
|
||||
vmlal.u8 q5, d14, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vhadd.s16 q4, q4, q11
|
||||
vld1.u32 {d18}, [r4], r11
|
||||
vmlal.u8 q5, d16, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vld1.u32 {d19}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vmlal.u8 q5, d17, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vld1.u8 {d6}, [r1]
|
||||
vqrshrun.s16 d20, q4, #6 ;right shift and saturating narrow
|
||||
; result 1
|
||||
vmlsl.u8 q5, d18, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q5, d19, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
vld1.u8 {d7}, [r6]
|
||||
vrhadd.u8 d20, d20, d6
|
||||
vmlsl.u8 q5, d12, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vmlsl.u8 q5, d13, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vst1.8 {d20}, [r1]! ;store the result pu1_dst
|
||||
vhadd.s16 q5, q5, q11
|
||||
subs r5, r5, #8 ;decrement the wd loop
|
||||
vqrshrun.s16 d8, q5, #6 ;right shift and saturating narrow
|
||||
; result 2
|
||||
vrhadd.u8 d8, d8, d7
|
||||
vst1.8 {d8}, [r6]! ;store the result pu1_dst
|
||||
cmp r5, #4
|
||||
bgt inner_loop_8
|
||||
|
||||
end_inner_loop_8
|
||||
subs r14, r14, #2 ;decrement the ht loop
|
||||
add r12, r12, r9 ;increment the src pointer by
|
||||
; 2*src_strd-wd
|
||||
add r1, r1, r8 ;increment the dst pointer by
|
||||
; 2*dst_strd-wd
|
||||
bgt outer_loop_8
|
||||
|
||||
ldr r10, [sp, #120] ;loads wd
|
||||
cmp r10, #12
|
||||
beq outer_loop4_residual
|
||||
|
||||
end_loops
|
||||
b end_func
|
||||
|
||||
outer_loop_16
|
||||
str r0, [sp, #-4]!
|
||||
str r7, [sp, #-4]!
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
and r0, r12, #31
|
||||
mov r7, #0xc000
|
||||
sub r5, r10, #0 ;checks wd
|
||||
pld [r4, r2, lsl #1]
|
||||
pld [r12, r2, lsl #1]
|
||||
vld1.u32 {q0}, [r12], r11 ;vector load pu1_src
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {q1}, [r12], r11
|
||||
vld1.u32 {q2}, [r12], r11
|
||||
vld1.u32 {q3}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {q6}, [r12], r11
|
||||
vmlsl.u8 q4, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q7}, [r12], r11
|
||||
vmlal.u8 q4, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {q8}, [r12], r11
|
||||
vmlal.u8 q4, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {q9}, [r12], r11
|
||||
vmlal.u8 q4, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlal.u8 q4, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vdup.16 q10, r7
|
||||
vmlsl.u8 q4, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q4, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
|
||||
inner_loop_16
|
||||
vmlsl.u8 q10, d1, d24
|
||||
vdup.16 q5, r7
|
||||
vmlsl.u8 q10, d3, d25
|
||||
mov r7, #0x4000
|
||||
vdup.16 q11, r7
|
||||
vmlal.u8 q10, d5, d26
|
||||
vld1.u32 {q0}, [r4], r11 ;vector load pu1_src
|
||||
vhadd.s16 q4, q4, q11
|
||||
vld1.u32 {q1}, [r4], r11
|
||||
vmlal.u8 q10, d7, d27
|
||||
add r12, #8
|
||||
subs r5, r5, #16
|
||||
vmlal.u8 q10, d13, d28
|
||||
vld1.u32 {q2}, [r4], r11
|
||||
vmlal.u8 q10, d15, d29
|
||||
vld1.u32 {q3}, [r4], r11
|
||||
vqrshrun.s16 d8, q4, #6 ;right shift and saturating narrow
|
||||
; result 1
|
||||
vmlsl.u8 q10, d17, d30
|
||||
vld1.u32 {q6}, [r4], r11
|
||||
vmlsl.u8 q10, d19, d31
|
||||
vld1.u32 {q7}, [r4], r11
|
||||
add r7, r1, #8
|
||||
vmlsl.u8 q5, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vmlsl.u8 q5, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q8}, [r4], r11
|
||||
vhadd.s16 q10, q10, q11
|
||||
vld1.u32 {q9}, [r4], r11
|
||||
vld1.u8 {d0}, [r1]
|
||||
vmlal.u8 q5, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u8 {d2}, [r7]
|
||||
vmlal.u8 q5, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
add r4, #8
|
||||
mov r7, #0xc000
|
||||
vmlal.u8 q5, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlal.u8 q5, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vqrshrun.s16 d9, q10, #6
|
||||
vdup.16 q11, r7
|
||||
vmlsl.u8 q5, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q5, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
mov r7, #0x4000
|
||||
vrhadd.u8 d8, d8, d0
|
||||
vrhadd.u8 d9, d9, d2
|
||||
vmlsl.u8 q11, d1, d24
|
||||
vmlsl.u8 q11, d3, d25
|
||||
vdup.16 q10, r7
|
||||
vmlal.u8 q11, d5, d26
|
||||
pld [r12, r2, lsl #2]
|
||||
pld [r4, r2, lsl #2]
|
||||
addeq r12, r12, r9 ;increment the src pointer by
|
||||
; 2*src_strd-wd
|
||||
addeq r4, r12, r2 ;pu1_src + src_strd
|
||||
vmlal.u8 q11, d7, d27
|
||||
vmlal.u8 q11, d13, d28
|
||||
vst1.8 {q4}, [r1]! ;store the result pu1_dst
|
||||
subeq r14, r14, #2
|
||||
vhadd.s16 q5, q5, q10
|
||||
vmlal.u8 q11, d15, d29
|
||||
addeq r1, r1, r8
|
||||
vmlsl.u8 q11, d17, d30
|
||||
cmp r14, #0
|
||||
vmlsl.u8 q11, d19, d31
|
||||
vqrshrun.s16 d10, q5, #6 ;right shift and saturating narrow
|
||||
; result 2
|
||||
beq epilog_16
|
||||
|
||||
vld1.u32 {q0}, [r12], r11 ;vector load pu1_src
|
||||
mov r7, #0xc000
|
||||
cmp r5, #0
|
||||
vld1.u32 {q1}, [r12], r11
|
||||
vhadd.s16 q11, q11, q10
|
||||
vld1.u32 {q2}, [r12], r11
|
||||
vdup.16 q4, r7
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vdup.16 q10, r7
|
||||
vld1.u32 {q3}, [r12], r11
|
||||
add r7, r6, #8
|
||||
moveq r5, r10
|
||||
vld1.u8 {d0}, [r6]
|
||||
vmlsl.u8 q4, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d2}, [r7]
|
||||
vqrshrun.s16 d11, q11, #6
|
||||
vmlal.u8 q4, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {q6}, [r12], r11
|
||||
vrhadd.u8 d10, d10, d0
|
||||
vld1.u32 {q7}, [r12], r11
|
||||
vrhadd.u8 d11, d11, d2
|
||||
vld1.u32 {q8}, [r12], r11
|
||||
vmlal.u8 q4, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {q9}, [r12], r11
|
||||
vmlal.u8 q4, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlal.u8 q4, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
mov r7, #0xc000
|
||||
vmlsl.u8 q4, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vst1.8 {q5}, [r6]! ;store the result pu1_dst
|
||||
vmlsl.u8 q4, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
addeq r6, r1, r3 ;pu1_dst + dst_strd
|
||||
b inner_loop_16
|
||||
|
||||
epilog_16
|
||||
mov r7, #0x4000
|
||||
ldr r0, [sp], #4
|
||||
ldr r10, [sp, #120]
|
||||
vdup.16 q10, r7
|
||||
vhadd.s16 q11, q11, q10
|
||||
vqrshrun.s16 d11, q11, #6
|
||||
add r7, r6, #8
|
||||
vld1.u8 {d20}, [r6]
|
||||
vld1.u8 {d21}, [r7]
|
||||
vrhadd.u8 d10, d10, d20
|
||||
vrhadd.u8 d11, d11, d21
|
||||
vst1.8 {q5}, [r6]! ;store the result pu1_dst
|
||||
ldr r7, [sp], #4
|
||||
cmp r10, #24
|
||||
beq outer_loop8_residual
|
||||
|
||||
end_loops1
|
||||
b end_func
|
||||
|
||||
outer_loop4_residual
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
mov r1, r7
|
||||
add r1, #8
|
||||
mov r10, #4
|
||||
add r12, #8
|
||||
mov r14, #16
|
||||
add r8, #4
|
||||
add r9, #4
|
||||
|
||||
outer_loop_4
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
subs r5, r10, #0 ;checks wd
|
||||
ble end_inner_loop_4
|
||||
|
||||
inner_loop_4
|
||||
vld1.u32 {d0}, [r12], r11 ;vector load pu1_src
|
||||
vld1.u32 {d1}, [r12], r11
|
||||
vld1.u32 {d2}, [r12], r11
|
||||
vld1.u32 {d3}, [r12], r11
|
||||
vld1.u32 {d4}, [r12], r11
|
||||
vld1.u32 {d5}, [r12], r11
|
||||
vld1.u32 {d6}, [r12], r11
|
||||
vld1.u32 {d7}, [r12], r11
|
||||
sub r12, r12, #4
|
||||
vld1.u32 {d12}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vld1.u32 {d13}, [r4], r11
|
||||
vzip.32 d0, d12 ;vector zip the i iteration and ii
|
||||
; interation in single register
|
||||
vld1.u32 {d14}, [r4], r11
|
||||
vzip.32 d1, d13
|
||||
vld1.u32 {d15}, [r4], r11
|
||||
vzip.32 d2, d14
|
||||
vld1.u32 {d16}, [r4], r11
|
||||
vzip.32 d3, d15
|
||||
vld1.u32 {d17}, [r4], r11
|
||||
vzip.32 d4, d16
|
||||
vld1.u32 {d18}, [r4], r11
|
||||
vzip.32 d5, d17
|
||||
vld1.u32 {d19}, [r4], r11
|
||||
mov r7, #0xc000
|
||||
vdup.16 q4, r7
|
||||
sub r4, r4, #4
|
||||
vzip.32 d6, d18
|
||||
vzip.32 d7, d19
|
||||
vmlsl.u8 q4, d1, d25 ;arithmetic operations for ii
|
||||
; iteration in the same time
|
||||
vmlsl.u8 q4, d0, d24
|
||||
vmlal.u8 q4, d2, d26
|
||||
vmlal.u8 q4, d3, d27
|
||||
vmlal.u8 q4, d4, d28
|
||||
vmlal.u8 q4, d5, d29
|
||||
vmlsl.u8 q4, d6, d30
|
||||
vmlsl.u8 q4, d7, d31
|
||||
mov r7, #0x4000
|
||||
vdup.16 q10, r7
|
||||
vhadd.s16 q4, q4, q10
|
||||
vqrshrun.s16 d8, q4, #6
|
||||
vld1.u32 {d10[0]}, [r1]
|
||||
vld1.u32 {d10[1]}, [r6]
|
||||
vrhadd.u8 d8, d8, d10
|
||||
vst1.32 {d8[0]},[r1]! ;store the i iteration result which
|
||||
; is in upper part of the register
|
||||
vst1.32 {d8[1]},[r6]! ;store the ii iteration result which
|
||||
; is in lower part of the register
|
||||
subs r5, r5, #4 ;decrement the wd by 4
|
||||
bgt inner_loop_4
|
||||
|
||||
end_inner_loop_4
|
||||
subs r14, r14, #2 ;decrement the ht by 4
|
||||
add r12, r12, r9 ;increment the input pointer
|
||||
; 2*src_strd-wd
|
||||
add r1, r1, r8 ;increment the output pointer
|
||||
; 2*dst_strd-wd
|
||||
bgt outer_loop_4
|
||||
|
||||
end_func
|
||||
vpop {d8 - d15}
|
||||
ldmfd sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
|
||||
ENDP
|
||||
|
||||
END
|
||||
+439
@@ -0,0 +1,439 @@
|
||||
;
|
||||
; Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
;**************Variables Vs Registers***********************************
|
||||
; r0 => src
|
||||
; r1 => dst
|
||||
; r2 => src_stride
|
||||
; r3 => dst_stride
|
||||
; r4 => filter_x0
|
||||
; r8 => ht
|
||||
; r10 => wd
|
||||
|
||||
EXPORT |vpx_convolve8_avg_horiz_filter_type2_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve8_avg_horiz_filter_type2_neon| PROC
|
||||
|
||||
stmfd sp!, {r4 - r12, r14} ;stack stores the values of
|
||||
; the arguments
|
||||
vpush {d8 - d15} ; stack offset by 64
|
||||
mov r4, r1
|
||||
mov r1, r2
|
||||
mov r2, r4
|
||||
|
||||
start_loop_count
|
||||
ldr r4, [sp, #104] ;loads pi1_coeff
|
||||
ldr r8, [sp, #108] ;loads x0_q4
|
||||
add r4, r4, r8, lsl #4 ;r4 = filter[x0_q4]
|
||||
ldr r8, [sp, #128] ;loads ht
|
||||
ldr r10, [sp, #124] ;loads wd
|
||||
vld2.8 {d0, d1}, [r4] ;coeff = vld1_s8(pi1_coeff)
|
||||
mov r11, #1
|
||||
subs r14, r8, #0 ;checks for ht == 0
|
||||
vabs.s8 d2, d0 ;vabs_s8(coeff)
|
||||
vdup.8 d24, d2[0] ;coeffabs_0 = vdup_lane_u8(coeffabs,
|
||||
; 0)
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
vdup.8 d25, d2[1] ;coeffabs_1 = vdup_lane_u8(coeffabs,
|
||||
; 1)
|
||||
add r4, r12, r2 ;pu1_src_tmp2_8 = pu1_src + src_strd
|
||||
vdup.8 d26, d2[2] ;coeffabs_2 = vdup_lane_u8(coeffabs,
|
||||
; 2)
|
||||
rsb r9, r10, r2, lsl #1 ;2*src_strd - wd
|
||||
vdup.8 d27, d2[3] ;coeffabs_3 = vdup_lane_u8(coeffabs,
|
||||
; 3)
|
||||
rsb r8, r10, r3, lsl #1 ;2*dst_strd - wd
|
||||
vdup.8 d28, d2[4] ;coeffabs_4 = vdup_lane_u8(coeffabs,
|
||||
; 4)
|
||||
vdup.8 d29, d2[5] ;coeffabs_5 = vdup_lane_u8(coeffabs,
|
||||
; 5)
|
||||
vdup.8 d30, d2[6] ;coeffabs_6 = vdup_lane_u8(coeffabs,
|
||||
; 6)
|
||||
vdup.8 d31, d2[7] ;coeffabs_7 = vdup_lane_u8(coeffabs,
|
||||
; 7)
|
||||
mov r7, r1
|
||||
cmp r10, #4
|
||||
ble outer_loop_4
|
||||
|
||||
cmp r10, #24
|
||||
moveq r10, #16
|
||||
addeq r8, #8
|
||||
addeq r9, #8
|
||||
cmp r10, #16
|
||||
bge outer_loop_16
|
||||
|
||||
cmp r10, #12
|
||||
addeq r8, #4
|
||||
addeq r9, #4
|
||||
b outer_loop_8
|
||||
|
||||
outer_loop8_residual
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
mov r1, r7
|
||||
mov r14, #32
|
||||
add r1, #16
|
||||
add r12, #16
|
||||
mov r10, #8
|
||||
add r8, #8
|
||||
add r9, #8
|
||||
|
||||
outer_loop_8
|
||||
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
subs r5, r10, #0 ;checks wd
|
||||
ble end_inner_loop_8
|
||||
|
||||
inner_loop_8
|
||||
mov r7, #0xc000
|
||||
vld1.u32 {d0}, [r12], r11 ;vector load pu1_src
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {d1}, [r12], r11
|
||||
vdup.16 q5, r7
|
||||
vld1.u32 {d2}, [r12], r11
|
||||
vld1.u32 {d3}, [r12], r11
|
||||
mov r7, #0x4000
|
||||
vld1.u32 {d4}, [r12], r11
|
||||
vmlal.u8 q4, d1, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {d5}, [r12], r11
|
||||
vmlal.u8 q4, d3, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {d6}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {d7}, [r12], r11
|
||||
vmlsl.u8 q4, d2, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {d12}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vmlal.u8 q4, d4, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vld1.u32 {d13}, [r4], r11
|
||||
vmlsl.u8 q4, d5, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vld1.u32 {d14}, [r4], r11
|
||||
vmlal.u8 q4, d6, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vld1.u32 {d15}, [r4], r11
|
||||
vmlsl.u8 q4, d7, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
vld1.u32 {d16}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vdup.16 q11, r7
|
||||
vmlal.u8 q5, d15, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {d17}, [r4], r11
|
||||
vmlsl.u8 q5, d14, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vhadd.s16 q4, q4, q11
|
||||
vld1.u32 {d18}, [r4], r11
|
||||
vmlal.u8 q5, d16, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vld1.u32 {d19}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vmlsl.u8 q5, d17, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vld1.u8 {d6}, [r1]
|
||||
vqrshrun.s16 d20, q4, #6 ;right shift and saturating narrow
|
||||
; result 1
|
||||
vmlal.u8 q5, d18, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q5, d19, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
vld1.u8 {d7}, [r6]
|
||||
vrhadd.u8 d20, d20, d6
|
||||
vmlsl.u8 q5, d12, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vmlal.u8 q5, d13, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vst1.8 {d20}, [r1]! ;store the result pu1_dst
|
||||
vhadd.s16 q5, q5, q11
|
||||
subs r5, r5, #8 ;decrement the wd loop
|
||||
vqrshrun.s16 d8, q5, #6 ;right shift and saturating narrow
|
||||
; result 2
|
||||
vrhadd.u8 d8, d8, d7
|
||||
vst1.8 {d8}, [r6]! ;store the result pu1_dst
|
||||
cmp r5, #4
|
||||
bgt inner_loop_8
|
||||
|
||||
end_inner_loop_8
|
||||
subs r14, r14, #2 ;decrement the ht loop
|
||||
add r12, r12, r9 ;increment the src pointer by
|
||||
; 2*src_strd-wd
|
||||
add r1, r1, r8 ;increment the dst pointer by
|
||||
; 2*dst_strd-wd
|
||||
bgt outer_loop_8
|
||||
|
||||
ldr r10, [sp, #120] ;loads wd
|
||||
cmp r10, #12
|
||||
beq outer_loop4_residual
|
||||
|
||||
end_loops
|
||||
b end_func
|
||||
|
||||
outer_loop_16
|
||||
str r0, [sp, #-4]!
|
||||
str r7, [sp, #-4]!
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
and r0, r12, #31
|
||||
mov r7, #0xc000
|
||||
sub r5, r10, #0 ;checks wd
|
||||
pld [r4, r2, lsl #1]
|
||||
pld [r12, r2, lsl #1]
|
||||
vld1.u32 {q0}, [r12], r11 ;vector load pu1_src
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {q1}, [r12], r11
|
||||
vld1.u32 {q2}, [r12], r11
|
||||
vld1.u32 {q3}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {q6}, [r12], r11
|
||||
vmlal.u8 q4, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q7}, [r12], r11
|
||||
vmlsl.u8 q4, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {q8}, [r12], r11
|
||||
vmlal.u8 q4, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {q9}, [r12], r11
|
||||
vmlal.u8 q4, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlsl.u8 q4, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vdup.16 q10, r7
|
||||
vmlal.u8 q4, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q4, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
|
||||
inner_loop_16
|
||||
vmlsl.u8 q10, d1, d24
|
||||
vdup.16 q5, r7
|
||||
vmlal.u8 q10, d3, d25
|
||||
mov r7, #0x4000
|
||||
vdup.16 q11, r7
|
||||
vmlsl.u8 q10, d5, d26
|
||||
vld1.u32 {q0}, [r4], r11 ;vector load pu1_src
|
||||
vhadd.s16 q4, q4, q11
|
||||
vld1.u32 {q1}, [r4], r11
|
||||
vmlal.u8 q10, d7, d27
|
||||
add r12, #8
|
||||
subs r5, r5, #16
|
||||
vmlal.u8 q10, d13, d28
|
||||
vld1.u32 {q2}, [r4], r11
|
||||
vmlsl.u8 q10, d15, d29
|
||||
vld1.u32 {q3}, [r4], r11
|
||||
vqrshrun.s16 d8, q4, #6 ;right shift and saturating narrow
|
||||
; result 1
|
||||
vmlal.u8 q10, d17, d30
|
||||
vld1.u32 {q6}, [r4], r11
|
||||
vmlsl.u8 q10, d19, d31
|
||||
vld1.u32 {q7}, [r4], r11
|
||||
add r7, r1, #8
|
||||
vmlsl.u8 q5, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vmlal.u8 q5, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q8}, [r4], r11
|
||||
vhadd.s16 q10, q10, q11
|
||||
vld1.u32 {q9}, [r4], r11
|
||||
vld1.u8 {d0}, [r1]
|
||||
vmlsl.u8 q5, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u8 {d2}, [r7]
|
||||
vmlal.u8 q5, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
add r4, #8
|
||||
mov r7, #0xc000
|
||||
vmlal.u8 q5, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlsl.u8 q5, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vqrshrun.s16 d9, q10, #6
|
||||
vdup.16 q11, r7
|
||||
vmlal.u8 q5, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q5, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
mov r7, #0x4000
|
||||
vrhadd.u8 d8, d8, d0
|
||||
vrhadd.u8 d9, d9, d2
|
||||
vmlsl.u8 q11, d1, d24
|
||||
vmlal.u8 q11, d3, d25
|
||||
vdup.16 q10, r7
|
||||
vmlsl.u8 q11, d5, d26
|
||||
pld [r12, r2, lsl #2]
|
||||
pld [r4, r2, lsl #2]
|
||||
addeq r12, r12, r9 ;increment the src pointer by
|
||||
; 2*src_strd-wd
|
||||
addeq r4, r12, r2 ;pu1_src + src_strd
|
||||
vmlal.u8 q11, d7, d27
|
||||
vmlal.u8 q11, d13, d28
|
||||
vst1.8 {q4}, [r1]! ;store the result pu1_dst
|
||||
subeq r14, r14, #2
|
||||
vhadd.s16 q5, q5, q10
|
||||
vmlsl.u8 q11, d15, d29
|
||||
addeq r1, r1, r8
|
||||
vmlal.u8 q11, d17, d30
|
||||
cmp r14, #0
|
||||
vmlsl.u8 q11, d19, d31
|
||||
vqrshrun.s16 d10, q5, #6 ;right shift and saturating narrow
|
||||
; result 2
|
||||
beq epilog_16
|
||||
|
||||
vld1.u32 {q0}, [r12], r11 ;vector load pu1_src
|
||||
mov r7, #0xc000
|
||||
cmp r5, #0
|
||||
vld1.u32 {q1}, [r12], r11
|
||||
vhadd.s16 q11, q11, q10
|
||||
vld1.u32 {q2}, [r12], r11
|
||||
vdup.16 q4, r7
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vdup.16 q10, r7
|
||||
vld1.u32 {q3}, [r12], r11
|
||||
add r7, r6, #8
|
||||
moveq r5, r10
|
||||
vld1.u8 {d0}, [r6]
|
||||
vmlal.u8 q4, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d2}, [r7]
|
||||
vqrshrun.s16 d11, q11, #6
|
||||
vmlsl.u8 q4, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {q6}, [r12], r11
|
||||
vrhadd.u8 d10, d10, d0
|
||||
vld1.u32 {q7}, [r12], r11
|
||||
vrhadd.u8 d11, d11, d2
|
||||
vld1.u32 {q8}, [r12], r11
|
||||
vmlal.u8 q4, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {q9}, [r12], r11
|
||||
vmlal.u8 q4, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlsl.u8 q4, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
mov r7, #0xc000
|
||||
vmlal.u8 q4, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vst1.8 {q5}, [r6]! ;store the result pu1_dst
|
||||
vmlsl.u8 q4, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
addeq r6, r1, r3 ;pu1_dst + dst_strd
|
||||
b inner_loop_16
|
||||
|
||||
epilog_16
|
||||
mov r7, #0x4000
|
||||
ldr r0, [sp], #4
|
||||
ldr r10, [sp, #120]
|
||||
vdup.16 q10, r7
|
||||
vhadd.s16 q11, q11, q10
|
||||
vqrshrun.s16 d11, q11, #6
|
||||
add r7, r6, #8
|
||||
vld1.u8 {d20}, [r6]
|
||||
vld1.u8 {d21}, [r7]
|
||||
vrhadd.u8 d10, d10, d20
|
||||
vrhadd.u8 d11, d11, d21
|
||||
vst1.8 {q5}, [r6]! ;store the result pu1_dst
|
||||
ldr r7, [sp], #4
|
||||
cmp r10, #24
|
||||
beq outer_loop8_residual
|
||||
|
||||
end_loops1
|
||||
b end_func
|
||||
|
||||
outer_loop4_residual
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
mov r1, r7
|
||||
add r1, #8
|
||||
mov r10, #4
|
||||
add r12, #8
|
||||
mov r14, #16
|
||||
add r8, #4
|
||||
add r9, #4
|
||||
|
||||
outer_loop_4
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
subs r5, r10, #0 ;checks wd
|
||||
ble end_inner_loop_4
|
||||
|
||||
inner_loop_4
|
||||
vld1.u32 {d0}, [r12], r11 ;vector load pu1_src
|
||||
vld1.u32 {d1}, [r12], r11
|
||||
vld1.u32 {d2}, [r12], r11
|
||||
vld1.u32 {d3}, [r12], r11
|
||||
vld1.u32 {d4}, [r12], r11
|
||||
vld1.u32 {d5}, [r12], r11
|
||||
vld1.u32 {d6}, [r12], r11
|
||||
vld1.u32 {d7}, [r12], r11
|
||||
sub r12, r12, #4
|
||||
vld1.u32 {d12}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vld1.u32 {d13}, [r4], r11
|
||||
vzip.32 d0, d12 ;vector zip the i iteration and ii
|
||||
; interation in single register
|
||||
vld1.u32 {d14}, [r4], r11
|
||||
vzip.32 d1, d13
|
||||
vld1.u32 {d15}, [r4], r11
|
||||
vzip.32 d2, d14
|
||||
vld1.u32 {d16}, [r4], r11
|
||||
vzip.32 d3, d15
|
||||
vld1.u32 {d17}, [r4], r11
|
||||
vzip.32 d4, d16
|
||||
vld1.u32 {d18}, [r4], r11
|
||||
vzip.32 d5, d17
|
||||
vld1.u32 {d19}, [r4], r11
|
||||
mov r7, #0xc000
|
||||
vdup.16 q4, r7
|
||||
sub r4, r4, #4
|
||||
vzip.32 d6, d18
|
||||
vzip.32 d7, d19
|
||||
vmlal.u8 q4, d1, d25 ;arithmetic operations for ii
|
||||
; iteration in the same time
|
||||
vmlsl.u8 q4, d0, d24
|
||||
vmlsl.u8 q4, d2, d26
|
||||
vmlal.u8 q4, d3, d27
|
||||
vmlal.u8 q4, d4, d28
|
||||
vmlsl.u8 q4, d5, d29
|
||||
vmlal.u8 q4, d6, d30
|
||||
vmlsl.u8 q4, d7, d31
|
||||
mov r7, #0x4000
|
||||
vdup.16 q10, r7
|
||||
vhadd.s16 q4, q4, q10
|
||||
vqrshrun.s16 d8, q4, #6
|
||||
vld1.u32 {d10[0]}, [r1]
|
||||
vld1.u32 {d10[1]}, [r6]
|
||||
vrhadd.u8 d8, d8, d10
|
||||
vst1.32 {d8[0]},[r1]! ;store the i iteration result which
|
||||
; is in upper part of the register
|
||||
vst1.32 {d8[1]},[r6]! ;store the ii iteration result which
|
||||
; is in lower part of the register
|
||||
subs r5, r5, #4 ;decrement the wd by 4
|
||||
bgt inner_loop_4
|
||||
|
||||
end_inner_loop_4
|
||||
subs r14, r14, #2 ;decrement the ht by 4
|
||||
add r12, r12, r9 ;increment the input pointer
|
||||
; 2*src_strd-wd
|
||||
add r1, r1, r8 ;increment the output pointer
|
||||
; 2*dst_strd-wd
|
||||
bgt outer_loop_4
|
||||
|
||||
end_func
|
||||
vpop {d8 - d15}
|
||||
ldmfd sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
|
||||
ENDP
|
||||
|
||||
END
|
||||
+486
@@ -0,0 +1,486 @@
|
||||
;
|
||||
; Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
;**************Variables Vs Registers***********************************
|
||||
; r0 => src
|
||||
; r1 => dst
|
||||
; r2 => src_stride
|
||||
; r6 => dst_stride
|
||||
; r12 => filter_y0
|
||||
; r5 => ht
|
||||
; r3 => wd
|
||||
|
||||
EXPORT |vpx_convolve8_avg_vert_filter_type1_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve8_avg_vert_filter_type1_neon| PROC
|
||||
|
||||
stmfd sp!, {r4 - r12, r14} ;stack stores the values of
|
||||
; the arguments
|
||||
vpush {d8 - d15} ; stack offset by 64
|
||||
mov r4, r1
|
||||
mov r1, r2
|
||||
mov r2, r4
|
||||
vmov.i16 q15, #0x4000
|
||||
mov r11, #0xc000
|
||||
ldr r12, [sp, #104] ;load filter
|
||||
ldr r6, [sp, #116] ;load y0_q4
|
||||
add r12, r12, r6, lsl #4 ;r12 = filter[y0_q4]
|
||||
mov r6, r3
|
||||
ldr r5, [sp, #124] ;load wd
|
||||
vld2.8 {d0, d1}, [r12] ;coeff = vld1_s8(pi1_coeff)
|
||||
sub r12, r2, r2, lsl #2 ;src_ctrd & pi1_coeff
|
||||
vabs.s8 d0, d0 ;vabs_s8(coeff)
|
||||
add r0, r0, r12 ;r0->pu1_src r12->pi1_coeff
|
||||
ldr r3, [sp, #128] ;load ht
|
||||
subs r7, r3, #0 ;r3->ht
|
||||
vdup.u8 d22, d0[0] ;coeffabs_0 = vdup_lane_u8(coeffabs,
|
||||
; 0);
|
||||
cmp r5, #8
|
||||
vdup.u8 d23, d0[1] ;coeffabs_1 = vdup_lane_u8(coeffabs,
|
||||
; 1);
|
||||
vdup.u8 d24, d0[2] ;coeffabs_2 = vdup_lane_u8(coeffabs,
|
||||
; 2);
|
||||
vdup.u8 d25, d0[3] ;coeffabs_3 = vdup_lane_u8(coeffabs,
|
||||
; 3);
|
||||
vdup.u8 d26, d0[4] ;coeffabs_4 = vdup_lane_u8(coeffabs,
|
||||
; 4);
|
||||
vdup.u8 d27, d0[5] ;coeffabs_5 = vdup_lane_u8(coeffabs,
|
||||
; 5);
|
||||
vdup.u8 d28, d0[6] ;coeffabs_6 = vdup_lane_u8(coeffabs,
|
||||
; 6);
|
||||
vdup.u8 d29, d0[7] ;coeffabs_7 = vdup_lane_u8(coeffabs,
|
||||
; 7);
|
||||
blt core_loop_wd_4 ;core loop wd 4 jump
|
||||
str r0, [sp, #-4]!
|
||||
str r1, [sp, #-4]!
|
||||
bic r4, r5, #7 ;r5 ->wd
|
||||
rsb r9, r4, r6, lsl #2 ;r6->dst_strd r5 ->wd
|
||||
rsb r8, r4, r2, lsl #2 ;r2->src_strd
|
||||
mov r3, r5, lsr #3 ;divide by 8
|
||||
mul r7, r3 ;multiply height by width
|
||||
sub r7, #4 ;subtract by one for epilog
|
||||
|
||||
prolog
|
||||
and r10, r0, #31
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vdup.16 q4, r11
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
subs r4, r4, #8
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vdup.16 q5, r11
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
addle r0, r0, r8
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
bicle r4, r5, #7 ;r5 ->wd
|
||||
vmlal.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
pld [r3]
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
pld [r3, r2]
|
||||
pld [r3, r2, lsl #1]
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
add r3, r3, r2
|
||||
vmlal.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
pld [r3, r2, lsl #1]
|
||||
vmlsl.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
vld1.u8 {d20}, [r1]
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d3, d23
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vrhadd.u8 d8, d8, d20
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d4, d24
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d5, d25
|
||||
vmlal.u8 q6, d6, d26
|
||||
add r14, r1, r6
|
||||
vmlal.u8 q6, d7, d27
|
||||
vmlsl.u8 q6, d16, d28
|
||||
vst1.8 {d8}, [r1]! ;vst1_u8(pu1_dst,sto_res);
|
||||
vmlsl.u8 q6, d17, d29
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
addle r1, r1, r9
|
||||
vmlsl.u8 q7, d4, d23
|
||||
subs r7, r7, #4
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vmlal.u8 q7, d5, d24
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d6, d25
|
||||
vrhadd.u8 d10, d10, d20
|
||||
vhadd.s16 q6, q6, q15
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q7, d7, d26
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d16, d27
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d17, d28
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
blt epilog_end ;jumps to epilog_end
|
||||
|
||||
beq epilog ;jumps to epilog
|
||||
|
||||
main_loop_8
|
||||
subs r4, r4, #8
|
||||
vmlsl.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d20}, [r14]
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
addle r0, r0, r8
|
||||
bicle r4, r5, #7 ;r5 ->wd
|
||||
vmlal.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vrhadd.u8 d12, d12, d20
|
||||
vhadd.s16 q7, q7, q15
|
||||
vdup.16 q5, r11
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vmlsl.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vmlsl.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
vrhadd.u8 d14, d14, d20
|
||||
vmlal.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
vst1.8 {d14}, [r14], r6
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
add r14, r1, #0
|
||||
vmlal.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
add r1, r1, #8
|
||||
vmlsl.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
addle r1, r1, r9
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vmlsl.u8 q6, d3, d23
|
||||
add r10, r3, r2, lsl #3 ; 10*strd - 8+2
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vrhadd.u8 d8, d8, d20
|
||||
add r10, r10, r2 ; 11*strd
|
||||
vmlal.u8 q6, d4, d24
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d5, d25
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d6, d26
|
||||
vst1.8 {d8}, [r14], r6 ;vst1_u8(pu1_dst,sto_res);
|
||||
pld [r10] ;11+ 0
|
||||
vmlal.u8 q6, d7, d27
|
||||
pld [r10, r2] ;11+ 1*strd
|
||||
pld [r10, r2, lsl #1] ;11+ 2*strd
|
||||
vmlsl.u8 q6, d16, d28
|
||||
add r10, r10, r2 ;12*strd
|
||||
vmlsl.u8 q6, d17, d29
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
|
||||
pld [r10, r2, lsl #1] ;11+ 3*strd
|
||||
vmlsl.u8 q7, d4, d23
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vrhadd.u8 d10, d10, d20
|
||||
subs r7, r7, #4
|
||||
vmlal.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vhadd.s16 q6, q6, q15
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q7, d7, d26
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d16, d27
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d17, d28
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
bgt main_loop_8 ;jumps to main_loop_8
|
||||
|
||||
epilog
|
||||
vld1.u8 {d20}, [r14]
|
||||
vmlsl.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
vmlal.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vrhadd.u8 d12, d12, d20
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vhadd.s16 q7, q7, q15
|
||||
vdup.16 q5, r11
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vmlal.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vmlsl.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
vrhadd.u8 d14, d14, d20
|
||||
vmlal.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
vmlal.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
vmlsl.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
vst1.8 {d14}, [r14], r6
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
vld1.u8 {d20}, [r1]
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d3, d23
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vrhadd.u8 d8, d8, d20
|
||||
vmlal.u8 q6, d4, d24
|
||||
vmlal.u8 q6, d5, d25
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d6, d26
|
||||
vmlal.u8 q6, d7, d27
|
||||
add r14, r1, r6
|
||||
vmlsl.u8 q6, d16, d28
|
||||
vst1.8 {d8}, [r1]! ;vst1_u8(pu1_dst,sto_res);
|
||||
vmlsl.u8 q6, d17, d29
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d4, d23
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vrhadd.u8 d10, d10, d20
|
||||
vmlal.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vhadd.s16 q6, q6, q15
|
||||
vmlal.u8 q7, d7, d26
|
||||
vmlal.u8 q7, d16, d27
|
||||
vmlsl.u8 q7, d17, d28
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
|
||||
epilog_end
|
||||
vld1.u8 {d20}, [r14]
|
||||
vrhadd.u8 d12, d12, d20
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vhadd.s16 q7, q7, q15
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
vld1.u8 {d20}, [r14]
|
||||
vrhadd.u8 d14, d14, d20
|
||||
vst1.8 {d14}, [r14], r6
|
||||
|
||||
end_loops
|
||||
tst r5, #7
|
||||
ldr r1, [sp], #4
|
||||
ldr r0, [sp], #4
|
||||
vpopeq {d8 - d15}
|
||||
ldmfdeq sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
mov r5, #4
|
||||
add r0, r0, #8
|
||||
add r1, r1, #8
|
||||
mov r7, #16
|
||||
|
||||
core_loop_wd_4
|
||||
rsb r9, r5, r6, lsl #2 ;r6->dst_strd r5 ->wd
|
||||
rsb r8, r5, r2, lsl #2 ;r2->src_strd
|
||||
vmov.i8 d4, #0
|
||||
|
||||
outer_loop_wd_4
|
||||
subs r12, r5, #0
|
||||
ble end_inner_loop_wd_4 ;outer loop jump
|
||||
|
||||
inner_loop_wd_4
|
||||
add r3, r0, r2
|
||||
vld1.u32 {d4[1]},[r3], r2 ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 1);
|
||||
subs r12, r12, #4
|
||||
vdup.u32 d5, d4[1] ;src_tmp2 = vdup_lane_u32(src_tmp1,
|
||||
; 1);
|
||||
vld1.u32 {d5[1]},[r3], r2 ;src_tmp2 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp2, 1);
|
||||
vld1.u32 {d4[0]},[r0] ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 0);
|
||||
vdup.16 q0, r11
|
||||
vmlsl.u8 q0, d5, d23 ;mul_res1 =
|
||||
; vmull_u8(vreinterpret_u8_u32(src_tmp2), coeffabs_1);
|
||||
vdup.u32 d6, d5[1] ;src_tmp3 = vdup_lane_u32(src_tmp2,
|
||||
; 1);
|
||||
add r0, r0, #4
|
||||
vld1.u32 {d6[1]},[r3], r2 ;src_tmp3 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp3, 1);
|
||||
vmlsl.u8 q0, d4, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp1), coeffabs_0);
|
||||
vdup.u32 d7, d6[1] ;src_tmp4 = vdup_lane_u32(src_tmp3,
|
||||
; 1);
|
||||
vld1.u32 {d7[1]},[r3], r2 ;src_tmp4 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp4, 1);
|
||||
vmlal.u8 q0, d6, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp3), coeffabs_2);
|
||||
vdup.16 q4, r11
|
||||
vmlsl.u8 q4, d7, d23
|
||||
vdup.u32 d4, d7[1] ;src_tmp1 = vdup_lane_u32(src_tmp4,
|
||||
; 1);
|
||||
vmull.u8 q1, d7, d25 ;mul_res2 =
|
||||
; vmull_u8(vreinterpret_u8_u32(src_tmp4), coeffabs_3);
|
||||
vld1.u32 {d4[1]},[r3], r2 ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 1);
|
||||
vmlsl.u8 q4, d6, d22
|
||||
vmlal.u8 q0, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp1), coeffabs_4);
|
||||
vdup.u32 d5, d4[1] ;src_tmp2 = vdup_lane_u32(src_tmp1,
|
||||
; 1);
|
||||
vmlal.u8 q4, d4, d24
|
||||
vld1.u32 {d5[1]},[r3], r2 ;src_tmp2 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp2, 1);
|
||||
vmlal.u8 q1, d5, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; vreinterpret_u8_u32(src_tmp2), coeffabs_5);
|
||||
vdup.u32 d6, d5[1] ;src_tmp3 = vdup_lane_u32(src_tmp2,
|
||||
; 1);
|
||||
vmlal.u8 q4, d5, d25
|
||||
vld1.u32 {d6[1]},[r3], r2 ;src_tmp3 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp3, 1);
|
||||
vmlsl.u8 q0, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp3), coeffabs_6);
|
||||
vdup.u32 d7, d6[1] ;src_tmp4 = vdup_lane_u32(src_tmp3,
|
||||
; 1);
|
||||
vmlal.u8 q4, d6, d26
|
||||
vld1.u32 {d7[1]},[r3], r2 ;src_tmp4 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp4, 1);
|
||||
vmlsl.u8 q1, d7, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; vreinterpret_u8_u32(src_tmp4), coeffabs_7);
|
||||
vdup.u32 d4, d7[1]
|
||||
vadd.i16 q0, q0, q1 ;mul_res1 = vaddq_u16(mul_res1,
|
||||
; mul_res2);
|
||||
vmlal.u8 q4, d7, d27
|
||||
vld1.u32 {d4[1]},[r3], r2
|
||||
vmlsl.u8 q4, d4, d28
|
||||
vdup.u32 d5, d4[1]
|
||||
vhadd.s16 q0, q0, q15
|
||||
vqrshrun.s16 d0, q0, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u32 {d5[1]},[r3]
|
||||
add r3, r1, r6
|
||||
vld1.u32 {d20[0]}, [r1]
|
||||
vld1.u32 {d20[1]}, [r3]
|
||||
vrhadd.u8 d0, d0, d20
|
||||
vst1.32 {d0[0]},[r1] ;vst1_lane_u32((uint32_t *)pu1_dst,
|
||||
; vreinterpret_u32_u8(sto_res), 0);
|
||||
vmlsl.u8 q4, d5, d29
|
||||
vst1.32 {d0[1]},[r3], r6 ;vst1_lane_u32((uint32_t
|
||||
; *)pu1_dst_tmp, vreinterpret_u32_u8(sto_res), 1);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vqrshrun.s16 d8, q4, #6
|
||||
mov r4, r3
|
||||
vld1.u32 {d20[0]}, [r4], r6
|
||||
vld1.u32 {d20[1]}, [r4]
|
||||
vrhadd.u8 d8, d8, d20
|
||||
vst1.32 {d8[0]},[r3], r6
|
||||
add r1, r1, #4
|
||||
vst1.32 {d8[1]},[r3]
|
||||
bgt inner_loop_wd_4
|
||||
|
||||
end_inner_loop_wd_4
|
||||
subs r7, r7, #4
|
||||
add r1, r1, r9
|
||||
add r0, r0, r8
|
||||
bgt outer_loop_wd_4
|
||||
|
||||
vpop {d8 - d15}
|
||||
ldmfd sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
|
||||
ENDP
|
||||
|
||||
END
|
||||
+487
@@ -0,0 +1,487 @@
|
||||
;
|
||||
; Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
;**************Variables Vs Registers***********************************
|
||||
; r0 => src
|
||||
; r1 => dst
|
||||
; r2 => src_stride
|
||||
; r6 => dst_stride
|
||||
; r12 => filter_y0
|
||||
; r5 => ht
|
||||
; r3 => wd
|
||||
|
||||
EXPORT |vpx_convolve8_avg_vert_filter_type2_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve8_avg_vert_filter_type2_neon| PROC
|
||||
|
||||
stmfd sp!, {r4 - r12, r14} ;stack stores the values of
|
||||
; the arguments
|
||||
vpush {d8 - d15} ; stack offset by 64
|
||||
mov r4, r1
|
||||
mov r1, r2
|
||||
mov r2, r4
|
||||
vmov.i16 q15, #0x4000
|
||||
mov r11, #0xc000
|
||||
ldr r12, [sp, #104] ;load filter
|
||||
ldr r6, [sp, #116] ;load y0_q4
|
||||
add r12, r12, r6, lsl #4 ;r12 = filter[y0_q4]
|
||||
mov r6, r3
|
||||
ldr r5, [sp, #124] ;load wd
|
||||
vld2.8 {d0, d1}, [r12] ;coeff = vld1_s8(pi1_coeff)
|
||||
sub r12, r2, r2, lsl #2 ;src_ctrd & pi1_coeff
|
||||
vabs.s8 d0, d0 ;vabs_s8(coeff)
|
||||
add r0, r0, r12 ;r0->pu1_src r12->pi1_coeff
|
||||
ldr r3, [sp, #128] ;load ht
|
||||
subs r7, r3, #0 ;r3->ht
|
||||
vdup.u8 d22, d0[0] ;coeffabs_0 = vdup_lane_u8(coeffabs,
|
||||
; 0);
|
||||
cmp r5, #8
|
||||
vdup.u8 d23, d0[1] ;coeffabs_1 = vdup_lane_u8(coeffabs,
|
||||
; 1);
|
||||
vdup.u8 d24, d0[2] ;coeffabs_2 = vdup_lane_u8(coeffabs,
|
||||
; 2);
|
||||
vdup.u8 d25, d0[3] ;coeffabs_3 = vdup_lane_u8(coeffabs,
|
||||
; 3);
|
||||
vdup.u8 d26, d0[4] ;coeffabs_4 = vdup_lane_u8(coeffabs,
|
||||
; 4);
|
||||
vdup.u8 d27, d0[5] ;coeffabs_5 = vdup_lane_u8(coeffabs,
|
||||
; 5);
|
||||
vdup.u8 d28, d0[6] ;coeffabs_6 = vdup_lane_u8(coeffabs,
|
||||
; 6);
|
||||
vdup.u8 d29, d0[7] ;coeffabs_7 = vdup_lane_u8(coeffabs,
|
||||
; 7);
|
||||
blt core_loop_wd_4 ;core loop wd 4 jump
|
||||
|
||||
str r0, [sp, #-4]!
|
||||
str r1, [sp, #-4]!
|
||||
bic r4, r5, #7 ;r5 ->wd
|
||||
rsb r9, r4, r6, lsl #2 ;r6->dst_strd r5 ->wd
|
||||
rsb r8, r4, r2, lsl #2 ;r2->src_strd
|
||||
mov r3, r5, lsr #3 ;divide by 8
|
||||
mul r7, r3 ;multiply height by width
|
||||
sub r7, #4 ;subtract by one for epilog
|
||||
|
||||
prolog
|
||||
and r10, r0, #31
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vdup.16 q4, r11
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
subs r4, r4, #8
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vdup.16 q5, r11
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
addle r0, r0, r8
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
bicle r4, r5, #7 ;r5 ->wd
|
||||
vmlsl.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
pld [r3]
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
pld [r3, r2]
|
||||
pld [r3, r2, lsl #1]
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
add r3, r3, r2
|
||||
vmlsl.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
pld [r3, r2, lsl #1]
|
||||
vmlal.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
vld1.u8 {d20}, [r1]
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d3, d23
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vrhadd.u8 d8, d8, d20
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d4, d24
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d5, d25
|
||||
vmlal.u8 q6, d6, d26
|
||||
add r14, r1, r6
|
||||
vmlsl.u8 q6, d7, d27
|
||||
vmlal.u8 q6, d16, d28
|
||||
vst1.8 {d8}, [r1]! ;vst1_u8(pu1_dst,sto_res);
|
||||
vmlsl.u8 q6, d17, d29
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
addle r1, r1, r9
|
||||
vmlal.u8 q7, d4, d23
|
||||
subs r7, r7, #4
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vmlsl.u8 q7, d5, d24
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d6, d25
|
||||
vrhadd.u8 d10, d10, d20
|
||||
vhadd.s16 q6, q6, q15
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q7, d7, d26
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d16, d27
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d17, d28
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
blt epilog_end ;jumps to epilog_end
|
||||
|
||||
beq epilog ;jumps to epilog
|
||||
|
||||
main_loop_8
|
||||
subs r4, r4, #8
|
||||
vmlal.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d20}, [r14]
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
addle r0, r0, r8
|
||||
bicle r4, r5, #7 ;r5 ->wd
|
||||
vmlsl.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vrhadd.u8 d12, d12, d20
|
||||
vhadd.s16 q7, q7, q15
|
||||
vdup.16 q5, r11
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vmlal.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vmlal.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
vrhadd.u8 d14, d14, d20
|
||||
vmlsl.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
vst1.8 {d14}, [r14], r6
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
add r14, r1, #0
|
||||
vmlsl.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
add r1, r1, #8
|
||||
vmlal.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
addle r1, r1, r9
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vmlal.u8 q6, d3, d23
|
||||
add r10, r3, r2, lsl #3 ; 10*strd - 8+2
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vrhadd.u8 d8, d8, d20
|
||||
add r10, r10, r2 ; 11*strd
|
||||
vmlsl.u8 q6, d4, d24
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d5, d25
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d6, d26
|
||||
vst1.8 {d8}, [r14], r6 ;vst1_u8(pu1_dst,sto_res);
|
||||
pld [r10] ;11+ 0
|
||||
vmlsl.u8 q6, d7, d27
|
||||
pld [r10, r2] ;11+ 1*strd
|
||||
pld [r10, r2, lsl #1] ;11+ 2*strd
|
||||
vmlal.u8 q6, d16, d28
|
||||
add r10, r10, r2 ;12*strd
|
||||
vmlsl.u8 q6, d17, d29
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
pld [r10, r2, lsl #1] ;11+ 3*strd
|
||||
vmlal.u8 q7, d4, d23
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vrhadd.u8 d10, d10, d20
|
||||
subs r7, r7, #4
|
||||
vmlsl.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vhadd.s16 q6, q6, q15
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q7, d7, d26
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d16, d27
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d17, d28
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
bgt main_loop_8 ;jumps to main_loop_8
|
||||
|
||||
epilog
|
||||
vld1.u8 {d20}, [r14]
|
||||
vmlal.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
vmlsl.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vrhadd.u8 d12, d12, d20
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vhadd.s16 q7, q7, q15
|
||||
vdup.16 q5, r11
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vmlsl.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vmlal.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
vrhadd.u8 d14, d14, d20
|
||||
vmlsl.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
vmlsl.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
vmlal.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
vst1.8 {d14}, [r14], r6
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
vld1.u8 {d20}, [r1]
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d3, d23
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vrhadd.u8 d8, d8, d20
|
||||
vmlsl.u8 q6, d4, d24
|
||||
vmlal.u8 q6, d5, d25
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d6, d26
|
||||
vmlsl.u8 q6, d7, d27
|
||||
add r14, r1, r6
|
||||
vmlal.u8 q6, d16, d28
|
||||
vst1.8 {d8}, [r1]! ;vst1_u8(pu1_dst,sto_res);
|
||||
vmlsl.u8 q6, d17, d29
|
||||
vld1.u8 {d20}, [r14]
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d4, d23
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vrhadd.u8 d10, d10, d20
|
||||
vmlsl.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vhadd.s16 q6, q6, q15
|
||||
vmlal.u8 q7, d7, d26
|
||||
vmlsl.u8 q7, d16, d27
|
||||
vmlal.u8 q7, d17, d28
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
|
||||
epilog_end
|
||||
vld1.u8 {d20}, [r14]
|
||||
vrhadd.u8 d12, d12, d20
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vhadd.s16 q7, q7, q15
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
vld1.u8 {d20}, [r14]
|
||||
vrhadd.u8 d14, d14, d20
|
||||
vst1.8 {d14}, [r14], r6
|
||||
|
||||
end_loops
|
||||
tst r5, #7
|
||||
ldr r1, [sp], #4
|
||||
ldr r0, [sp], #4
|
||||
vpopeq {d8 - d15}
|
||||
ldmfdeq sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
|
||||
mov r5, #4
|
||||
add r0, r0, #8
|
||||
add r1, r1, #8
|
||||
mov r7, #16
|
||||
|
||||
core_loop_wd_4
|
||||
rsb r9, r5, r6, lsl #2 ;r6->dst_strd r5 ->wd
|
||||
rsb r8, r5, r2, lsl #2 ;r2->src_strd
|
||||
vmov.i8 d4, #0
|
||||
|
||||
outer_loop_wd_4
|
||||
subs r12, r5, #0
|
||||
ble end_inner_loop_wd_4 ;outer loop jump
|
||||
|
||||
inner_loop_wd_4
|
||||
add r3, r0, r2
|
||||
vld1.u32 {d4[1]},[r3], r2 ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 1);
|
||||
subs r12, r12, #4
|
||||
vdup.u32 d5, d4[1] ;src_tmp2 = vdup_lane_u32(src_tmp1,
|
||||
; 1);
|
||||
vld1.u32 {d5[1]},[r3], r2 ;src_tmp2 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp2, 1);
|
||||
vld1.u32 {d4[0]},[r0] ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 0);
|
||||
vdup.16 q0, r11
|
||||
vmlal.u8 q0, d5, d23 ;mul_res1 =
|
||||
; vmull_u8(vreinterpret_u8_u32(src_tmp2), coeffabs_1);
|
||||
vdup.u32 d6, d5[1] ;src_tmp3 = vdup_lane_u32(src_tmp2,
|
||||
; 1);
|
||||
add r0, r0, #4
|
||||
vld1.u32 {d6[1]},[r3], r2 ;src_tmp3 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp3, 1);
|
||||
vmlsl.u8 q0, d4, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp1), coeffabs_0);
|
||||
vdup.u32 d7, d6[1] ;src_tmp4 = vdup_lane_u32(src_tmp3,
|
||||
; 1);
|
||||
vld1.u32 {d7[1]},[r3], r2 ;src_tmp4 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp4, 1);
|
||||
vmlsl.u8 q0, d6, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp3), coeffabs_2);
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q4, d7, d23
|
||||
vdup.u32 d4, d7[1] ;src_tmp1 = vdup_lane_u32(src_tmp4,
|
||||
; 1);
|
||||
vmull.u8 q1, d7, d25 ;mul_res2 =
|
||||
; vmull_u8(vreinterpret_u8_u32(src_tmp4), coeffabs_3);
|
||||
vld1.u32 {d4[1]},[r3], r2 ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 1);
|
||||
vmlsl.u8 q4, d6, d22
|
||||
vmlal.u8 q0, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp1), coeffabs_4);
|
||||
vdup.u32 d5, d4[1] ;src_tmp2 = vdup_lane_u32(src_tmp1,
|
||||
; 1);
|
||||
vmlsl.u8 q4, d4, d24
|
||||
vld1.u32 {d5[1]},[r3], r2 ;src_tmp2 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp2, 1);
|
||||
vmlsl.u8 q1, d5, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; vreinterpret_u8_u32(src_tmp2), coeffabs_5);
|
||||
vdup.u32 d6, d5[1] ;src_tmp3 = vdup_lane_u32(src_tmp2,
|
||||
; 1);
|
||||
vmlal.u8 q4, d5, d25
|
||||
vld1.u32 {d6[1]},[r3], r2 ;src_tmp3 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp3, 1);
|
||||
vmlal.u8 q0, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp3), coeffabs_6);
|
||||
vdup.u32 d7, d6[1] ;src_tmp4 = vdup_lane_u32(src_tmp3,
|
||||
; 1);
|
||||
vmlal.u8 q4, d6, d26
|
||||
vld1.u32 {d7[1]},[r3], r2 ;src_tmp4 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp4, 1);
|
||||
vmlsl.u8 q1, d7, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; vreinterpret_u8_u32(src_tmp4), coeffabs_7);
|
||||
vdup.u32 d4, d7[1]
|
||||
vadd.i16 q0, q0, q1 ;mul_res1 = vaddq_u16(mul_res1,
|
||||
; mul_res2);
|
||||
vmlsl.u8 q4, d7, d27
|
||||
vld1.u32 {d4[1]},[r3], r2
|
||||
vmlal.u8 q4, d4, d28
|
||||
vdup.u32 d5, d4[1]
|
||||
vhadd.s16 q0, q0, q15
|
||||
vqrshrun.s16 d0, q0, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u32 {d5[1]},[r3]
|
||||
add r3, r1, r6
|
||||
vld1.u32 {d20[0]}, [r1]
|
||||
vld1.u32 {d20[1]}, [r3]
|
||||
vrhadd.u8 d0, d0, d20
|
||||
vst1.32 {d0[0]},[r1] ;vst1_lane_u32((uint32_t *)pu1_dst,
|
||||
; vreinterpret_u32_u8(sto_res), 0);
|
||||
vmlsl.u8 q4, d5, d29
|
||||
vst1.32 {d0[1]},[r3], r6 ;vst1_lane_u32((uint32_t
|
||||
; *)pu1_dst_tmp, vreinterpret_u32_u8(sto_res), 1);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vqrshrun.s16 d8, q4, #6
|
||||
mov r4, r3
|
||||
vld1.u32 {d20[0]}, [r4], r6
|
||||
vld1.u32 {d20[1]}, [r4]
|
||||
vrhadd.u8 d8, d8, d20
|
||||
vst1.32 {d8[0]},[r3], r6
|
||||
add r1, r1, #4
|
||||
vst1.32 {d8[1]},[r3]
|
||||
bgt inner_loop_wd_4
|
||||
|
||||
end_inner_loop_wd_4
|
||||
subs r7, r7, #4
|
||||
add r1, r1, r9
|
||||
add r0, r0, r8
|
||||
bgt outer_loop_wd_4
|
||||
|
||||
vpop {d8 - d15}
|
||||
ldmfd sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
|
||||
ENDP
|
||||
|
||||
END
|
||||
+415
@@ -0,0 +1,415 @@
|
||||
;
|
||||
; Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
;**************Variables Vs Registers***********************************
|
||||
; r0 => src
|
||||
; r1 => dst
|
||||
; r2 => src_stride
|
||||
; r3 => dst_stride
|
||||
; r4 => filter_x0
|
||||
; r8 => ht
|
||||
; r10 => wd
|
||||
|
||||
EXPORT |vpx_convolve8_horiz_filter_type1_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve8_horiz_filter_type1_neon| PROC
|
||||
|
||||
stmfd sp!, {r4 - r12, r14} ;stack stores the values of
|
||||
; the arguments
|
||||
vpush {d8 - d15} ; stack offset by 64
|
||||
mov r4, r1
|
||||
mov r1, r2
|
||||
mov r2, r4
|
||||
start_loop_count
|
||||
ldr r4, [sp, #104] ;loads pi1_coeff
|
||||
ldr r8, [sp, #108] ;loads x0_q4
|
||||
add r4, r4, r8, lsl #4 ;r4 = filter[x0_q4]
|
||||
ldr r8, [sp, #128] ;loads ht
|
||||
ldr r10, [sp, #124] ;loads wd
|
||||
vld2.8 {d0, d1}, [r4] ;coeff = vld1_s8(pi1_coeff)
|
||||
mov r11, #1
|
||||
subs r14, r8, #0 ;checks for ht == 0
|
||||
vabs.s8 d2, d0 ;vabs_s8(coeff)
|
||||
vdup.8 d24, d2[0] ;coeffabs_0 = vdup_lane_u8(coeffabs,
|
||||
; 0)
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
vdup.8 d25, d2[1] ;coeffabs_1 = vdup_lane_u8(coeffabs,
|
||||
; 1)
|
||||
add r4, r12, r2 ;pu1_src_tmp2_8 = pu1_src + src_strd
|
||||
vdup.8 d26, d2[2] ;coeffabs_2 = vdup_lane_u8(coeffabs,
|
||||
; 2)
|
||||
rsb r9, r10, r2, lsl #1 ;2*src_strd - wd
|
||||
vdup.8 d27, d2[3] ;coeffabs_3 = vdup_lane_u8(coeffabs,
|
||||
; 3)
|
||||
rsb r8, r10, r3, lsl #1 ;2*dst_strd - wd
|
||||
vdup.8 d28, d2[4] ;coeffabs_4 = vdup_lane_u8(coeffabs,
|
||||
; 4)
|
||||
vdup.8 d29, d2[5] ;coeffabs_5 = vdup_lane_u8(coeffabs,
|
||||
; 5)
|
||||
vdup.8 d30, d2[6] ;coeffabs_6 = vdup_lane_u8(coeffabs,
|
||||
; 6)
|
||||
vdup.8 d31, d2[7] ;coeffabs_7 = vdup_lane_u8(coeffabs,
|
||||
; 7)
|
||||
mov r7, r1
|
||||
cmp r10, #4
|
||||
ble outer_loop_4
|
||||
|
||||
cmp r10, #24
|
||||
moveq r10, #16
|
||||
addeq r8, #8
|
||||
addeq r9, #8
|
||||
cmp r10, #16
|
||||
bge outer_loop_16
|
||||
|
||||
cmp r10, #12
|
||||
addeq r8, #4
|
||||
addeq r9, #4
|
||||
b outer_loop_8
|
||||
|
||||
outer_loop8_residual
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
mov r1, r7
|
||||
mov r14, #32
|
||||
add r1, #16
|
||||
add r12, #16
|
||||
mov r10, #8
|
||||
add r8, #8
|
||||
add r9, #8
|
||||
|
||||
outer_loop_8
|
||||
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
subs r5, r10, #0 ;checks wd
|
||||
ble end_inner_loop_8
|
||||
|
||||
inner_loop_8
|
||||
mov r7, #0xc000
|
||||
vld1.u32 {d0}, [r12], r11 ;vector load pu1_src
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {d1}, [r12], r11
|
||||
vdup.16 q5, r7
|
||||
vld1.u32 {d2}, [r12], r11
|
||||
vld1.u32 {d3}, [r12], r11
|
||||
mov r7, #0x4000
|
||||
vld1.u32 {d4}, [r12], r11
|
||||
vmlsl.u8 q4, d1, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {d5}, [r12], r11
|
||||
vmlal.u8 q4, d3, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {d6}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {d7}, [r12], r11
|
||||
vmlal.u8 q4, d2, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {d12}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vmlal.u8 q4, d4, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vld1.u32 {d13}, [r4], r11
|
||||
vmlal.u8 q4, d5, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vld1.u32 {d14}, [r4], r11
|
||||
vmlsl.u8 q4, d6, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vld1.u32 {d15}, [r4], r11
|
||||
vmlsl.u8 q4, d7, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
vld1.u32 {d16}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vdup.16 q11, r7
|
||||
vmlal.u8 q5, d15, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {d17}, [r4], r11
|
||||
vmlal.u8 q5, d14, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vhadd.s16 q4, q4, q11
|
||||
vld1.u32 {d18}, [r4], r11
|
||||
vmlal.u8 q5, d16, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vld1.u32 {d19}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vmlal.u8 q5, d17, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vmlsl.u8 q5, d18, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q5, d19, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
vqrshrun.s16 d20, q4, #6 ;right shift and saturating narrow
|
||||
; result 1
|
||||
vmlsl.u8 q5, d12, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vmlsl.u8 q5, d13, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vst1.8 {d20}, [r1]! ;store the result pu1_dst
|
||||
vhadd.s16 q5, q5, q11
|
||||
subs r5, r5, #8 ;decrement the wd loop
|
||||
vqrshrun.s16 d8, q5, #6 ;right shift and saturating narrow
|
||||
; result 2
|
||||
vst1.8 {d8}, [r6]! ;store the result pu1_dst
|
||||
cmp r5, #4
|
||||
bgt inner_loop_8
|
||||
|
||||
end_inner_loop_8
|
||||
subs r14, r14, #2 ;decrement the ht loop
|
||||
add r12, r12, r9 ;increment the src pointer by
|
||||
; 2*src_strd-wd
|
||||
add r1, r1, r8 ;increment the dst pointer by
|
||||
; 2*dst_strd-wd
|
||||
bgt outer_loop_8
|
||||
|
||||
ldr r10, [sp, #120] ;loads wd
|
||||
cmp r10, #12
|
||||
beq outer_loop4_residual
|
||||
|
||||
end_loops
|
||||
b end_func
|
||||
|
||||
outer_loop_16
|
||||
str r0, [sp, #-4]!
|
||||
str r7, [sp, #-4]!
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
and r0, r12, #31
|
||||
mov r7, #0xc000
|
||||
sub r5, r10, #0 ;checks wd
|
||||
pld [r4, r2, lsl #1]
|
||||
pld [r12, r2, lsl #1]
|
||||
vld1.u32 {q0}, [r12], r11 ;vector load pu1_src
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {q1}, [r12], r11
|
||||
vld1.u32 {q2}, [r12], r11
|
||||
vld1.u32 {q3}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {q6}, [r12], r11
|
||||
vmlsl.u8 q4, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q7}, [r12], r11
|
||||
vmlal.u8 q4, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {q8}, [r12], r11
|
||||
vmlal.u8 q4, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {q9}, [r12], r11
|
||||
vmlal.u8 q4, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlal.u8 q4, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vdup.16 q10, r7
|
||||
vmlsl.u8 q4, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q4, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
|
||||
inner_loop_16
|
||||
vmlsl.u8 q10, d1, d24
|
||||
vdup.16 q5, r7
|
||||
vmlsl.u8 q10, d3, d25
|
||||
mov r7, #0x4000
|
||||
vdup.16 q11, r7
|
||||
vmlal.u8 q10, d5, d26
|
||||
vld1.u32 {q0}, [r4], r11 ;vector load pu1_src
|
||||
vhadd.s16 q4, q4, q11
|
||||
vld1.u32 {q1}, [r4], r11
|
||||
vmlal.u8 q10, d7, d27
|
||||
add r12, #8
|
||||
subs r5, r5, #16
|
||||
vmlal.u8 q10, d13, d28
|
||||
vld1.u32 {q2}, [r4], r11
|
||||
vmlal.u8 q10, d15, d29
|
||||
vld1.u32 {q3}, [r4], r11
|
||||
vqrshrun.s16 d8, q4, #6 ;right shift and saturating narrow
|
||||
; result 1
|
||||
vmlsl.u8 q10, d17, d30
|
||||
vld1.u32 {q6}, [r4], r11
|
||||
vmlsl.u8 q10, d19, d31
|
||||
vld1.u32 {q7}, [r4], r11
|
||||
vmlsl.u8 q5, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vmlsl.u8 q5, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q8}, [r4], r11
|
||||
vhadd.s16 q10, q10, q11
|
||||
vld1.u32 {q9}, [r4], r11
|
||||
vmlal.u8 q5, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vmlal.u8 q5, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
add r4, #8
|
||||
mov r7, #0xc000
|
||||
vmlal.u8 q5, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlal.u8 q5, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vqrshrun.s16 d9, q10, #6
|
||||
vdup.16 q11, r7
|
||||
vmlsl.u8 q5, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q5, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
mov r7, #0x4000
|
||||
vmlsl.u8 q11, d1, d24
|
||||
vst1.8 {q4}, [r1]! ;store the result pu1_dst
|
||||
vmlsl.u8 q11, d3, d25
|
||||
vdup.16 q10, r7
|
||||
vmlal.u8 q11, d5, d26
|
||||
pld [r12, r2, lsl #2]
|
||||
pld [r4, r2, lsl #2]
|
||||
addeq r12, r12, r9 ;increment the src pointer by
|
||||
; 2*src_strd-wd
|
||||
addeq r4, r12, r2 ;pu1_src + src_strd
|
||||
vmlal.u8 q11, d7, d27
|
||||
addeq r1, r1, r8
|
||||
subeq r14, r14, #2
|
||||
vmlal.u8 q11, d13, d28
|
||||
vhadd.s16 q5, q5, q10
|
||||
vmlal.u8 q11, d15, d29
|
||||
vmlsl.u8 q11, d17, d30
|
||||
cmp r14, #0
|
||||
vmlsl.u8 q11, d19, d31
|
||||
vqrshrun.s16 d10, q5, #6 ;right shift and saturating narrow
|
||||
; result 2
|
||||
beq epilog_16
|
||||
|
||||
vld1.u32 {q0}, [r12], r11 ;vector load pu1_src
|
||||
mov r7, #0xc000
|
||||
cmp r5, #0
|
||||
vld1.u32 {q1}, [r12], r11
|
||||
vhadd.s16 q11, q11, q10
|
||||
vld1.u32 {q2}, [r12], r11
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {q3}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {q6}, [r12], r11
|
||||
vld1.u32 {q7}, [r12], r11
|
||||
vmlsl.u8 q4, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q8}, [r12], r11
|
||||
vmlal.u8 q4, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {q9}, [r12], r11
|
||||
vqrshrun.s16 d11, q11, #6
|
||||
vmlal.u8 q4, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
moveq r5, r10
|
||||
vmlal.u8 q4, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vdup.16 q10, r7
|
||||
vmlal.u8 q4, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vst1.8 {q5}, [r6]! ;store the result pu1_dst
|
||||
vmlsl.u8 q4, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q4, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
addeq r6, r1, r3 ;pu1_dst + dst_strd
|
||||
b inner_loop_16
|
||||
|
||||
epilog_16
|
||||
mov r7, #0x4000
|
||||
ldr r0, [sp], #4
|
||||
ldr r10, [sp, #120]
|
||||
vdup.16 q10, r7
|
||||
vhadd.s16 q11, q11, q10
|
||||
vqrshrun.s16 d11, q11, #6
|
||||
vst1.8 {q5}, [r6]! ;store the result pu1_dst
|
||||
ldr r7, [sp], #4
|
||||
cmp r10, #24
|
||||
beq outer_loop8_residual
|
||||
|
||||
end_loops1
|
||||
b end_func
|
||||
|
||||
outer_loop4_residual
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
mov r1, r7
|
||||
add r1, #8
|
||||
mov r10, #4
|
||||
add r12, #8
|
||||
mov r14, #16
|
||||
add r8, #4
|
||||
add r9, #4
|
||||
|
||||
outer_loop_4
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
subs r5, r10, #0 ;checks wd
|
||||
ble end_inner_loop_4
|
||||
|
||||
inner_loop_4
|
||||
vld1.u32 {d0}, [r12], r11 ;vector load pu1_src
|
||||
vld1.u32 {d1}, [r12], r11
|
||||
vld1.u32 {d2}, [r12], r11
|
||||
vld1.u32 {d3}, [r12], r11
|
||||
vld1.u32 {d4}, [r12], r11
|
||||
vld1.u32 {d5}, [r12], r11
|
||||
vld1.u32 {d6}, [r12], r11
|
||||
vld1.u32 {d7}, [r12], r11
|
||||
sub r12, r12, #4
|
||||
vld1.u32 {d12}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vld1.u32 {d13}, [r4], r11
|
||||
vzip.32 d0, d12 ;vector zip the i iteration and ii
|
||||
; interation in single register
|
||||
vld1.u32 {d14}, [r4], r11
|
||||
vzip.32 d1, d13
|
||||
vld1.u32 {d15}, [r4], r11
|
||||
vzip.32 d2, d14
|
||||
vld1.u32 {d16}, [r4], r11
|
||||
vzip.32 d3, d15
|
||||
vld1.u32 {d17}, [r4], r11
|
||||
vzip.32 d4, d16
|
||||
vld1.u32 {d18}, [r4], r11
|
||||
vzip.32 d5, d17
|
||||
vld1.u32 {d19}, [r4], r11
|
||||
mov r7, #0xc000
|
||||
vdup.16 q4, r7
|
||||
sub r4, r4, #4
|
||||
vzip.32 d6, d18
|
||||
vzip.32 d7, d19
|
||||
vmlsl.u8 q4, d1, d25 ;arithmetic operations for ii
|
||||
; iteration in the same time
|
||||
vmlsl.u8 q4, d0, d24
|
||||
vmlal.u8 q4, d2, d26
|
||||
vmlal.u8 q4, d3, d27
|
||||
vmlal.u8 q4, d4, d28
|
||||
vmlal.u8 q4, d5, d29
|
||||
vmlsl.u8 q4, d6, d30
|
||||
vmlsl.u8 q4, d7, d31
|
||||
mov r7, #0x4000
|
||||
vdup.16 q10, r7
|
||||
vhadd.s16 q4, q4, q10
|
||||
vqrshrun.s16 d8, q4, #6
|
||||
vst1.32 {d8[0]},[r1]! ;store the i iteration result which
|
||||
; is in upper part of the register
|
||||
vst1.32 {d8[1]},[r6]! ;store the ii iteration result which
|
||||
; is in lower part of the register
|
||||
subs r5, r5, #4 ;decrement the wd by 4
|
||||
bgt inner_loop_4
|
||||
|
||||
end_inner_loop_4
|
||||
subs r14, r14, #2 ;decrement the ht by 4
|
||||
add r12, r12, r9 ;increment the input pointer
|
||||
; 2*src_strd-wd
|
||||
add r1, r1, r8 ;increment the output pointer
|
||||
; 2*dst_strd-wd
|
||||
bgt outer_loop_4
|
||||
|
||||
end_func
|
||||
vpop {d8 - d15}
|
||||
ldmfd sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
|
||||
ENDP
|
||||
|
||||
END
|
||||
+415
@@ -0,0 +1,415 @@
|
||||
;
|
||||
; Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
;**************Variables Vs Registers***********************************
|
||||
; r0 => src
|
||||
; r1 => dst
|
||||
; r2 => src_stride
|
||||
; r3 => dst_stride
|
||||
; r4 => filter_x0
|
||||
; r8 => ht
|
||||
; r10 => wd
|
||||
|
||||
EXPORT |vpx_convolve8_horiz_filter_type2_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve8_horiz_filter_type2_neon| PROC
|
||||
|
||||
stmfd sp!, {r4 - r12, r14} ;stack stores the values of
|
||||
; the arguments
|
||||
vpush {d8 - d15} ; stack offset by 64
|
||||
mov r4, r1
|
||||
mov r1, r2
|
||||
mov r2, r4
|
||||
|
||||
start_loop_count
|
||||
ldr r4, [sp, #104] ;loads pi1_coeff
|
||||
ldr r8, [sp, #108] ;loads x0_q4
|
||||
add r4, r4, r8, lsl #4 ;r4 = filter[x0_q4]
|
||||
ldr r8, [sp, #128] ;loads ht
|
||||
ldr r10, [sp, #124] ;loads wd
|
||||
vld2.8 {d0, d1}, [r4] ;coeff = vld1_s8(pi1_coeff)
|
||||
mov r11, #1
|
||||
subs r14, r8, #0 ;checks for ht == 0
|
||||
vabs.s8 d2, d0 ;vabs_s8(coeff)
|
||||
vdup.8 d24, d2[0] ;coeffabs_0 = vdup_lane_u8(coeffabs,
|
||||
; 0)
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
vdup.8 d25, d2[1] ;coeffabs_1 = vdup_lane_u8(coeffabs,
|
||||
; 1)
|
||||
add r4, r12, r2 ;pu1_src_tmp2_8 = pu1_src + src_strd
|
||||
vdup.8 d26, d2[2] ;coeffabs_2 = vdup_lane_u8(coeffabs,
|
||||
; 2)
|
||||
rsb r9, r10, r2, lsl #1 ;2*src_strd - wd
|
||||
vdup.8 d27, d2[3] ;coeffabs_3 = vdup_lane_u8(coeffabs,
|
||||
; 3)
|
||||
rsb r8, r10, r3, lsl #1 ;2*dst_strd - wd
|
||||
vdup.8 d28, d2[4] ;coeffabs_4 = vdup_lane_u8(coeffabs,
|
||||
; 4)
|
||||
vdup.8 d29, d2[5] ;coeffabs_5 = vdup_lane_u8(coeffabs,
|
||||
; 5)
|
||||
vdup.8 d30, d2[6] ;coeffabs_6 = vdup_lane_u8(coeffabs,
|
||||
; 6)
|
||||
vdup.8 d31, d2[7] ;coeffabs_7 = vdup_lane_u8(coeffabs,
|
||||
; 7)
|
||||
mov r7, r1
|
||||
cmp r10, #4
|
||||
ble outer_loop_4
|
||||
|
||||
cmp r10, #24
|
||||
moveq r10, #16
|
||||
addeq r8, #8
|
||||
addeq r9, #8
|
||||
cmp r10, #16
|
||||
bge outer_loop_16
|
||||
|
||||
cmp r10, #12
|
||||
addeq r8, #4
|
||||
addeq r9, #4
|
||||
b outer_loop_8
|
||||
|
||||
outer_loop8_residual
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
mov r1, r7
|
||||
mov r14, #32
|
||||
add r1, #16
|
||||
add r12, #16
|
||||
mov r10, #8
|
||||
add r8, #8
|
||||
add r9, #8
|
||||
|
||||
outer_loop_8
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
subs r5, r10, #0 ;checks wd
|
||||
ble end_inner_loop_8
|
||||
|
||||
inner_loop_8
|
||||
mov r7, #0xc000
|
||||
vld1.u32 {d0}, [r12], r11 ;vector load pu1_src
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {d1}, [r12], r11
|
||||
vdup.16 q5, r7
|
||||
vld1.u32 {d2}, [r12], r11
|
||||
vld1.u32 {d3}, [r12], r11
|
||||
mov r7, #0x4000
|
||||
vld1.u32 {d4}, [r12], r11
|
||||
vmlal.u8 q4, d1, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {d5}, [r12], r11
|
||||
vmlal.u8 q4, d3, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {d6}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {d7}, [r12], r11
|
||||
vmlsl.u8 q4, d2, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {d12}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vmlal.u8 q4, d4, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vld1.u32 {d13}, [r4], r11
|
||||
vmlsl.u8 q4, d5, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vld1.u32 {d14}, [r4], r11
|
||||
vmlal.u8 q4, d6, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vld1.u32 {d15}, [r4], r11
|
||||
vmlsl.u8 q4, d7, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
vld1.u32 {d16}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vdup.16 q11, r7
|
||||
vmlal.u8 q5, d15, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {d17}, [r4], r11
|
||||
vmlsl.u8 q5, d14, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vhadd.s16 q4, q4, q11
|
||||
vld1.u32 {d18}, [r4], r11
|
||||
vmlal.u8 q5, d16, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vld1.u32 {d19}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vmlsl.u8 q5, d17, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vmlal.u8 q5, d18, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q5, d19, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
vqrshrun.s16 d20, q4, #6 ;right shift and saturating narrow
|
||||
; result 1
|
||||
vmlsl.u8 q5, d12, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vmlal.u8 q5, d13, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vst1.8 {d20}, [r1]! ;store the result pu1_dst
|
||||
vhadd.s16 q5, q5, q11
|
||||
subs r5, r5, #8 ;decrement the wd loop
|
||||
vqrshrun.s16 d8, q5, #6 ;right shift and saturating narrow
|
||||
; result 2
|
||||
vst1.8 {d8}, [r6]! ;store the result pu1_dst
|
||||
cmp r5, #4
|
||||
bgt inner_loop_8
|
||||
|
||||
end_inner_loop_8
|
||||
subs r14, r14, #2 ;decrement the ht loop
|
||||
add r12, r12, r9 ;increment the src pointer by
|
||||
; 2*src_strd-wd
|
||||
add r1, r1, r8 ;increment the dst pointer by
|
||||
; 2*dst_strd-wd
|
||||
bgt outer_loop_8
|
||||
|
||||
ldr r10, [sp, #120] ;loads wd
|
||||
cmp r10, #12
|
||||
beq outer_loop4_residual
|
||||
|
||||
end_loops
|
||||
b end_func
|
||||
|
||||
outer_loop_16
|
||||
str r0, [sp, #-4]!
|
||||
str r7, [sp, #-4]!
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
and r0, r12, #31
|
||||
mov r7, #0xc000
|
||||
sub r5, r10, #0 ;checks wd
|
||||
pld [r4, r2, lsl #1]
|
||||
pld [r12, r2, lsl #1]
|
||||
vld1.u32 {q0}, [r12], r11 ;vector load pu1_src
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {q1}, [r12], r11
|
||||
vld1.u32 {q2}, [r12], r11
|
||||
vld1.u32 {q3}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {q6}, [r12], r11
|
||||
vmlal.u8 q4, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q7}, [r12], r11
|
||||
vmlsl.u8 q4, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {q8}, [r12], r11
|
||||
vmlal.u8 q4, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
vld1.u32 {q9}, [r12], r11
|
||||
vmlal.u8 q4, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlsl.u8 q4, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vdup.16 q10, r7
|
||||
vmlal.u8 q4, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q4, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
|
||||
inner_loop_16
|
||||
vmlsl.u8 q10, d1, d24
|
||||
vdup.16 q5, r7
|
||||
vmlal.u8 q10, d3, d25
|
||||
mov r7, #0x4000
|
||||
vdup.16 q11, r7
|
||||
vmlsl.u8 q10, d5, d26
|
||||
vld1.u32 {q0}, [r4], r11 ;vector load pu1_src
|
||||
vhadd.s16 q4, q4, q11
|
||||
vld1.u32 {q1}, [r4], r11
|
||||
vmlal.u8 q10, d7, d27
|
||||
add r12, #8
|
||||
subs r5, r5, #16
|
||||
vmlal.u8 q10, d13, d28
|
||||
vld1.u32 {q2}, [r4], r11
|
||||
vmlsl.u8 q10, d15, d29
|
||||
vld1.u32 {q3}, [r4], r11
|
||||
vqrshrun.s16 d8, q4, #6 ;right shift and saturating narrow
|
||||
; result 1
|
||||
vmlal.u8 q10, d17, d30
|
||||
vld1.u32 {q6}, [r4], r11
|
||||
vmlsl.u8 q10, d19, d31
|
||||
vld1.u32 {q7}, [r4], r11
|
||||
vmlsl.u8 q5, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vmlal.u8 q5, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q8}, [r4], r11
|
||||
vhadd.s16 q10, q10, q11
|
||||
vld1.u32 {q9}, [r4], r11
|
||||
vmlsl.u8 q5, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vmlal.u8 q5, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
add r4, #8
|
||||
mov r7, #0xc000
|
||||
vmlal.u8 q5, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vmlsl.u8 q5, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vqrshrun.s16 d9, q10, #6
|
||||
vdup.16 q11, r7
|
||||
vmlal.u8 q5, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q5, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
mov r7, #0x4000
|
||||
vmlsl.u8 q11, d1, d24
|
||||
vst1.8 {q4}, [r1]! ;store the result pu1_dst
|
||||
vmlal.u8 q11, d3, d25
|
||||
vdup.16 q10, r7
|
||||
vmlsl.u8 q11, d5, d26
|
||||
pld [r12, r2, lsl #2]
|
||||
pld [r4, r2, lsl #2]
|
||||
addeq r12, r12, r9 ;increment the src pointer by
|
||||
; 2*src_strd-wd
|
||||
addeq r4, r12, r2 ;pu1_src + src_strd
|
||||
vmlal.u8 q11, d7, d27
|
||||
addeq r1, r1, r8
|
||||
subeq r14, r14, #2
|
||||
vmlal.u8 q11, d13, d28
|
||||
vhadd.s16 q5, q5, q10
|
||||
vmlsl.u8 q11, d15, d29
|
||||
vmlal.u8 q11, d17, d30
|
||||
cmp r14, #0
|
||||
vmlsl.u8 q11, d19, d31
|
||||
vqrshrun.s16 d10, q5, #6 ;right shift and saturating narrow
|
||||
; result 2
|
||||
beq epilog_16
|
||||
|
||||
vld1.u32 {q0}, [r12], r11 ;vector load pu1_src
|
||||
mov r7, #0xc000
|
||||
cmp r5, #0
|
||||
vld1.u32 {q1}, [r12], r11
|
||||
vhadd.s16 q11, q11, q10
|
||||
vld1.u32 {q2}, [r12], r11
|
||||
vdup.16 q4, r7
|
||||
vld1.u32 {q3}, [r12], r11
|
||||
vmlsl.u8 q4, d0, d24 ;mul_res = vmlsl_u8(src[0_0],
|
||||
; coeffabs_0);
|
||||
vld1.u32 {q6}, [r12], r11
|
||||
vld1.u32 {q7}, [r12], r11
|
||||
vmlal.u8 q4, d2, d25 ;mul_res = vmlal_u8(src[0_1],
|
||||
; coeffabs_1);
|
||||
vld1.u32 {q8}, [r12], r11
|
||||
vmlsl.u8 q4, d4, d26 ;mul_res = vmlsl_u8(src[0_2],
|
||||
; coeffabs_2);
|
||||
vld1.u32 {q9}, [r12], r11
|
||||
vqrshrun.s16 d11, q11, #6
|
||||
vmlal.u8 q4, d6, d27 ;mul_res = vmull_u8(src[0_3],
|
||||
; coeffabs_3);
|
||||
moveq r5, r10
|
||||
vmlal.u8 q4, d12, d28 ;mul_res = vmlal_u8(src[0_4],
|
||||
; coeffabs_4);
|
||||
vdup.16 q10, r7
|
||||
vmlsl.u8 q4, d14, d29 ;mul_res = vmlsl_u8(src[0_5],
|
||||
; coeffabs_5);
|
||||
vst1.8 {q5}, [r6]! ;store the result pu1_dst
|
||||
vmlal.u8 q4, d16, d30 ;mul_res = vmlal_u8(src[0_6],
|
||||
; coeffabs_6);
|
||||
vmlsl.u8 q4, d18, d31 ;mul_res = vmlsl_u8(src[0_7],
|
||||
; coeffabs_7);
|
||||
addeq r6, r1, r3 ;pu1_dst + dst_strd
|
||||
b inner_loop_16
|
||||
|
||||
epilog_16
|
||||
mov r7, #0x4000
|
||||
ldr r0, [sp], #4
|
||||
ldr r10, [sp, #120]
|
||||
vdup.16 q10, r7
|
||||
vhadd.s16 q11, q11, q10
|
||||
vqrshrun.s16 d11, q11, #6
|
||||
vst1.8 {q5}, [r6]! ;store the result pu1_dst
|
||||
ldr r7, [sp], #4
|
||||
cmp r10, #24
|
||||
beq outer_loop8_residual
|
||||
|
||||
end_loops1
|
||||
b end_func
|
||||
|
||||
outer_loop4_residual
|
||||
sub r12, r0, #3 ;pu1_src - 3
|
||||
mov r1, r7
|
||||
add r1, #8
|
||||
mov r10, #4
|
||||
add r12, #8
|
||||
mov r14, #16
|
||||
add r8, #4
|
||||
add r9, #4
|
||||
|
||||
outer_loop_4
|
||||
add r6, r1, r3 ;pu1_dst + dst_strd
|
||||
add r4, r12, r2 ;pu1_src + src_strd
|
||||
subs r5, r10, #0 ;checks wd
|
||||
ble end_inner_loop_4
|
||||
|
||||
inner_loop_4
|
||||
vld1.u32 {d0}, [r12], r11 ;vector load pu1_src
|
||||
vld1.u32 {d1}, [r12], r11
|
||||
vld1.u32 {d2}, [r12], r11
|
||||
vld1.u32 {d3}, [r12], r11
|
||||
vld1.u32 {d4}, [r12], r11
|
||||
vld1.u32 {d5}, [r12], r11
|
||||
vld1.u32 {d6}, [r12], r11
|
||||
vld1.u32 {d7}, [r12], r11
|
||||
sub r12, r12, #4
|
||||
vld1.u32 {d12}, [r4], r11 ;vector load pu1_src + src_strd
|
||||
vld1.u32 {d13}, [r4], r11
|
||||
vzip.32 d0, d12 ;vector zip the i iteration and ii
|
||||
; interation in single register
|
||||
vld1.u32 {d14}, [r4], r11
|
||||
vzip.32 d1, d13
|
||||
vld1.u32 {d15}, [r4], r11
|
||||
vzip.32 d2, d14
|
||||
vld1.u32 {d16}, [r4], r11
|
||||
vzip.32 d3, d15
|
||||
vld1.u32 {d17}, [r4], r11
|
||||
vzip.32 d4, d16
|
||||
vld1.u32 {d18}, [r4], r11
|
||||
vzip.32 d5, d17
|
||||
vld1.u32 {d19}, [r4], r11
|
||||
mov r7, #0xc000
|
||||
vdup.16 q4, r7
|
||||
sub r4, r4, #4
|
||||
vzip.32 d6, d18
|
||||
vzip.32 d7, d19
|
||||
vmlal.u8 q4, d1, d25 ;arithmetic operations for ii
|
||||
; iteration in the same time
|
||||
vmlsl.u8 q4, d0, d24
|
||||
vmlsl.u8 q4, d2, d26
|
||||
vmlal.u8 q4, d3, d27
|
||||
vmlal.u8 q4, d4, d28
|
||||
vmlsl.u8 q4, d5, d29
|
||||
vmlal.u8 q4, d6, d30
|
||||
vmlsl.u8 q4, d7, d31
|
||||
mov r7, #0x4000
|
||||
vdup.16 q10, r7
|
||||
vhadd.s16 q4, q4, q10
|
||||
vqrshrun.s16 d8, q4, #6
|
||||
vst1.32 {d8[0]},[r1]! ;store the i iteration result which
|
||||
; is in upper part of the register
|
||||
vst1.32 {d8[1]},[r6]! ;store the ii iteration result which
|
||||
; is in lower part of the register
|
||||
subs r5, r5, #4 ;decrement the wd by 4
|
||||
bgt inner_loop_4
|
||||
|
||||
end_inner_loop_4
|
||||
subs r14, r14, #2 ;decrement the ht by 4
|
||||
add r12, r12, r9 ;increment the input pointer
|
||||
; 2*src_strd-wd
|
||||
add r1, r1, r8 ;increment the output pointer
|
||||
; 2*dst_strd-wd
|
||||
bgt outer_loop_4
|
||||
|
||||
end_func
|
||||
vpop {d8 - d15}
|
||||
ldmfd sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
|
||||
ENDP
|
||||
|
||||
END
|
||||
@@ -0,0 +1,959 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/arm/vpx_convolve8_neon.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
|
||||
// Note:
|
||||
// 1. src is not always 32-bit aligned, so don't call vld1_lane_u32(src).
|
||||
// 2. After refactoring the shared code in kernel loops with inline functions,
|
||||
// the decoder speed dropped a lot when using gcc compiler. Therefore there is
|
||||
// no refactoring for those parts by now.
|
||||
// 3. For horizontal convolve, there is an alternative optimization that
|
||||
// convolves a single row in each loop. For each row, 8 sample banks with 4 or 8
|
||||
// samples in each are read from memory: src, (src+1), (src+2), (src+3),
|
||||
// (src+4), (src+5), (src+6), (src+7), or prepared by vector extract
|
||||
// instructions. This optimization is much faster in speed unit test, but slowed
|
||||
// down the whole decoder by 5%.
|
||||
|
||||
static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p,
|
||||
const uint8x8_t s0, const uint8x8_t s1,
|
||||
const uint8x8_t s2, const uint8x8_t s3,
|
||||
const uint8x8_t s4, const uint8x8_t s5,
|
||||
const uint8x8_t s6, const uint8x8_t s7) {
|
||||
vst1_u8(s, s0);
|
||||
s += p;
|
||||
vst1_u8(s, s1);
|
||||
s += p;
|
||||
vst1_u8(s, s2);
|
||||
s += p;
|
||||
vst1_u8(s, s3);
|
||||
s += p;
|
||||
vst1_u8(s, s4);
|
||||
s += p;
|
||||
vst1_u8(s, s5);
|
||||
s += p;
|
||||
vst1_u8(s, s6);
|
||||
s += p;
|
||||
vst1_u8(s, s7);
|
||||
}
|
||||
|
||||
void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
const int16x8_t filters = vld1q_s16(filter[x0_q4]);
|
||||
uint8x8_t t0, t1, t2, t3;
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
assert(x_step_q4 == 16);
|
||||
|
||||
(void)x_step_q4;
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
src -= 3;
|
||||
|
||||
if (h == 4) {
|
||||
uint8x8_t d01, d23;
|
||||
int16x4_t filter3, filter4, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, d0,
|
||||
d1, d2, d3;
|
||||
int16x8_t tt0, tt1, tt2, tt3;
|
||||
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
filter3 = vdup_lane_s16(vget_low_s16(filters), 3);
|
||||
filter4 = vdup_lane_s16(vget_high_s16(filters), 0);
|
||||
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
|
||||
transpose_u8_8x4(&t0, &t1, &t2, &t3);
|
||||
tt0 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
tt1 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
tt2 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
tt3 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s0 = vget_low_s16(tt0);
|
||||
s1 = vget_low_s16(tt1);
|
||||
s2 = vget_low_s16(tt2);
|
||||
s3 = vget_low_s16(tt3);
|
||||
s4 = vget_high_s16(tt0);
|
||||
s5 = vget_high_s16(tt1);
|
||||
s6 = vget_high_s16(tt2);
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
src += 7;
|
||||
|
||||
do {
|
||||
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
|
||||
transpose_u8_8x4(&t0, &t1, &t2, &t3);
|
||||
tt0 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
tt1 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
tt2 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
tt3 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s7 = vget_low_s16(tt0);
|
||||
s8 = vget_low_s16(tt1);
|
||||
s9 = vget_low_s16(tt2);
|
||||
s10 = vget_low_s16(tt3);
|
||||
|
||||
d0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
d1 = convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
d2 = convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
d3 = convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
|
||||
d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), 7);
|
||||
d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), 7);
|
||||
transpose_u8_4x4(&d01, &d23);
|
||||
|
||||
vst1_lane_u32((uint32_t *)(dst + 0 * dst_stride),
|
||||
vreinterpret_u32_u8(d01), 0);
|
||||
vst1_lane_u32((uint32_t *)(dst + 1 * dst_stride),
|
||||
vreinterpret_u32_u8(d23), 0);
|
||||
vst1_lane_u32((uint32_t *)(dst + 2 * dst_stride),
|
||||
vreinterpret_u32_u8(d01), 1);
|
||||
vst1_lane_u32((uint32_t *)(dst + 3 * dst_stride),
|
||||
vreinterpret_u32_u8(d23), 1);
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
src += 4;
|
||||
dst += 4;
|
||||
w -= 4;
|
||||
} while (w > 0);
|
||||
} else {
|
||||
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
|
||||
int width;
|
||||
const uint8_t *s;
|
||||
uint8x8_t t4, t5, t6, t7;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
|
||||
if (w == 4) {
|
||||
do {
|
||||
load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
|
||||
s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
|
||||
s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
|
||||
|
||||
load_u8_8x8(src + 7, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6,
|
||||
&t7);
|
||||
src += 8 * src_stride;
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(dst + 4 * dst_stride);
|
||||
__builtin_prefetch(dst + 5 * dst_stride);
|
||||
__builtin_prefetch(dst + 6 * dst_stride);
|
||||
__builtin_prefetch(dst + 7 * dst_stride);
|
||||
transpose_u8_4x8(&t0, &t1, &t2, &t3, t4, t5, t6, t7);
|
||||
s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
__builtin_prefetch(src + 7 * src_stride);
|
||||
t0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
t1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
t2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
t3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
|
||||
transpose_u8_8x4(&t0, &t1, &t2, &t3);
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t0), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t1), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t2), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t3), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t0), 1);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t1), 1);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t2), 1);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t3), 1);
|
||||
dst += dst_stride;
|
||||
h -= 8;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
uint8_t *d;
|
||||
int16x8_t s11, s12, s13, s14;
|
||||
|
||||
do {
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
__builtin_prefetch(src + 7 * src_stride);
|
||||
load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
|
||||
s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
|
||||
s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
|
||||
|
||||
width = w;
|
||||
s = src + 7;
|
||||
d = dst;
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(dst + 4 * dst_stride);
|
||||
__builtin_prefetch(dst + 5 * dst_stride);
|
||||
__builtin_prefetch(dst + 6 * dst_stride);
|
||||
__builtin_prefetch(dst + 7 * dst_stride);
|
||||
|
||||
do {
|
||||
load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
|
||||
s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
|
||||
s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
|
||||
s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
|
||||
|
||||
t0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
t1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
t2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
t3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
t4 = convolve8_8(s4, s5, s6, s7, s8, s9, s10, s11, filters, filter3,
|
||||
filter4);
|
||||
t5 = convolve8_8(s5, s6, s7, s8, s9, s10, s11, s12, filters, filter3,
|
||||
filter4);
|
||||
t6 = convolve8_8(s6, s7, s8, s9, s10, s11, s12, s13, filters, filter3,
|
||||
filter4);
|
||||
t7 = convolve8_8(s7, s8, s9, s10, s11, s12, s13, s14, filters,
|
||||
filter3, filter4);
|
||||
|
||||
transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
store_u8_8x8(d, dst_stride, t0, t1, t2, t3, t4, t5, t6, t7);
|
||||
|
||||
s0 = s8;
|
||||
s1 = s9;
|
||||
s2 = s10;
|
||||
s3 = s11;
|
||||
s4 = s12;
|
||||
s5 = s13;
|
||||
s6 = s14;
|
||||
s += 8;
|
||||
d += 8;
|
||||
width -= 8;
|
||||
} while (width > 0);
|
||||
src += 8 * src_stride;
|
||||
dst += 8 * dst_stride;
|
||||
h -= 8;
|
||||
} while (h > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h) {
|
||||
const int16x8_t filters = vld1q_s16(filter[x0_q4]);
|
||||
uint8x8_t t0, t1, t2, t3;
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
assert(x_step_q4 == 16);
|
||||
|
||||
(void)x_step_q4;
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
src -= 3;
|
||||
|
||||
if (h == 4) {
|
||||
uint8x8_t d01, d23;
|
||||
int16x4_t filter3, filter4, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, d0,
|
||||
d1, d2, d3;
|
||||
int16x8_t tt0, tt1, tt2, tt3;
|
||||
uint32x4_t d0123 = vdupq_n_u32(0);
|
||||
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
filter3 = vdup_lane_s16(vget_low_s16(filters), 3);
|
||||
filter4 = vdup_lane_s16(vget_high_s16(filters), 0);
|
||||
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
|
||||
transpose_u8_8x4(&t0, &t1, &t2, &t3);
|
||||
tt0 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
tt1 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
tt2 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
tt3 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s0 = vget_low_s16(tt0);
|
||||
s1 = vget_low_s16(tt1);
|
||||
s2 = vget_low_s16(tt2);
|
||||
s3 = vget_low_s16(tt3);
|
||||
s4 = vget_high_s16(tt0);
|
||||
s5 = vget_high_s16(tt1);
|
||||
s6 = vget_high_s16(tt2);
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
src += 7;
|
||||
|
||||
do {
|
||||
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
|
||||
transpose_u8_8x4(&t0, &t1, &t2, &t3);
|
||||
tt0 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
tt1 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
tt2 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
tt3 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s7 = vget_low_s16(tt0);
|
||||
s8 = vget_low_s16(tt1);
|
||||
s9 = vget_low_s16(tt2);
|
||||
s10 = vget_low_s16(tt3);
|
||||
|
||||
d0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
d1 = convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
d2 = convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
d3 = convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
|
||||
d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), 7);
|
||||
d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), 7);
|
||||
transpose_u8_4x4(&d01, &d23);
|
||||
|
||||
d0123 = vld1q_lane_u32((uint32_t *)(dst + 0 * dst_stride), d0123, 0);
|
||||
d0123 = vld1q_lane_u32((uint32_t *)(dst + 1 * dst_stride), d0123, 2);
|
||||
d0123 = vld1q_lane_u32((uint32_t *)(dst + 2 * dst_stride), d0123, 1);
|
||||
d0123 = vld1q_lane_u32((uint32_t *)(dst + 3 * dst_stride), d0123, 3);
|
||||
d0123 = vreinterpretq_u32_u8(
|
||||
vrhaddq_u8(vreinterpretq_u8_u32(d0123), vcombine_u8(d01, d23)));
|
||||
|
||||
vst1q_lane_u32((uint32_t *)(dst + 0 * dst_stride), d0123, 0);
|
||||
vst1q_lane_u32((uint32_t *)(dst + 1 * dst_stride), d0123, 2);
|
||||
vst1q_lane_u32((uint32_t *)(dst + 2 * dst_stride), d0123, 1);
|
||||
vst1q_lane_u32((uint32_t *)(dst + 3 * dst_stride), d0123, 3);
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
src += 4;
|
||||
dst += 4;
|
||||
w -= 4;
|
||||
} while (w > 0);
|
||||
} else {
|
||||
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
|
||||
int width;
|
||||
const uint8_t *s;
|
||||
uint8x8_t t4, t5, t6, t7;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
|
||||
if (w == 4) {
|
||||
uint32x4_t d0415 = vdupq_n_u32(0);
|
||||
uint32x4_t d2637 = vdupq_n_u32(0);
|
||||
do {
|
||||
load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
|
||||
s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
|
||||
s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
|
||||
|
||||
load_u8_8x8(src + 7, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6,
|
||||
&t7);
|
||||
src += 8 * src_stride;
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(dst + 4 * dst_stride);
|
||||
__builtin_prefetch(dst + 5 * dst_stride);
|
||||
__builtin_prefetch(dst + 6 * dst_stride);
|
||||
__builtin_prefetch(dst + 7 * dst_stride);
|
||||
transpose_u8_4x8(&t0, &t1, &t2, &t3, t4, t5, t6, t7);
|
||||
s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
__builtin_prefetch(src + 7 * src_stride);
|
||||
t0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
t1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
t2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
t3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
|
||||
transpose_u8_8x4(&t0, &t1, &t2, &t3);
|
||||
|
||||
d0415 = vld1q_lane_u32((uint32_t *)(dst + 0 * dst_stride), d0415, 0);
|
||||
d0415 = vld1q_lane_u32((uint32_t *)(dst + 1 * dst_stride), d0415, 2);
|
||||
d2637 = vld1q_lane_u32((uint32_t *)(dst + 2 * dst_stride), d2637, 0);
|
||||
d2637 = vld1q_lane_u32((uint32_t *)(dst + 3 * dst_stride), d2637, 2);
|
||||
d0415 = vld1q_lane_u32((uint32_t *)(dst + 4 * dst_stride), d0415, 1);
|
||||
d0415 = vld1q_lane_u32((uint32_t *)(dst + 5 * dst_stride), d0415, 3);
|
||||
d2637 = vld1q_lane_u32((uint32_t *)(dst + 6 * dst_stride), d2637, 1);
|
||||
d2637 = vld1q_lane_u32((uint32_t *)(dst + 7 * dst_stride), d2637, 3);
|
||||
d0415 = vreinterpretq_u32_u8(
|
||||
vrhaddq_u8(vreinterpretq_u8_u32(d0415), vcombine_u8(t0, t1)));
|
||||
d2637 = vreinterpretq_u32_u8(
|
||||
vrhaddq_u8(vreinterpretq_u8_u32(d2637), vcombine_u8(t2, t3)));
|
||||
|
||||
vst1q_lane_u32((uint32_t *)dst, d0415, 0);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d0415, 2);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d2637, 0);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d2637, 2);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d0415, 1);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d0415, 3);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d2637, 1);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d2637, 3);
|
||||
dst += dst_stride;
|
||||
h -= 8;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
uint8_t *d;
|
||||
int16x8_t s11, s12, s13, s14;
|
||||
uint8x16_t d01, d23, d45, d67;
|
||||
|
||||
do {
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
__builtin_prefetch(src + 7 * src_stride);
|
||||
load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
|
||||
s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
|
||||
s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
|
||||
|
||||
width = w;
|
||||
s = src + 7;
|
||||
d = dst;
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(dst + 4 * dst_stride);
|
||||
__builtin_prefetch(dst + 5 * dst_stride);
|
||||
__builtin_prefetch(dst + 6 * dst_stride);
|
||||
__builtin_prefetch(dst + 7 * dst_stride);
|
||||
|
||||
do {
|
||||
load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
|
||||
s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
|
||||
s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
|
||||
s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
|
||||
s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
|
||||
s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
|
||||
s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
|
||||
s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
|
||||
|
||||
t0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
t1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
t2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
t3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
t4 = convolve8_8(s4, s5, s6, s7, s8, s9, s10, s11, filters, filter3,
|
||||
filter4);
|
||||
t5 = convolve8_8(s5, s6, s7, s8, s9, s10, s11, s12, filters, filter3,
|
||||
filter4);
|
||||
t6 = convolve8_8(s6, s7, s8, s9, s10, s11, s12, s13, filters, filter3,
|
||||
filter4);
|
||||
t7 = convolve8_8(s7, s8, s9, s10, s11, s12, s13, s14, filters,
|
||||
filter3, filter4);
|
||||
|
||||
transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
|
||||
|
||||
d01 = vcombine_u8(vld1_u8(d + 0 * dst_stride),
|
||||
vld1_u8(d + 1 * dst_stride));
|
||||
d23 = vcombine_u8(vld1_u8(d + 2 * dst_stride),
|
||||
vld1_u8(d + 3 * dst_stride));
|
||||
d45 = vcombine_u8(vld1_u8(d + 4 * dst_stride),
|
||||
vld1_u8(d + 5 * dst_stride));
|
||||
d67 = vcombine_u8(vld1_u8(d + 6 * dst_stride),
|
||||
vld1_u8(d + 7 * dst_stride));
|
||||
d01 = vrhaddq_u8(d01, vcombine_u8(t0, t1));
|
||||
d23 = vrhaddq_u8(d23, vcombine_u8(t2, t3));
|
||||
d45 = vrhaddq_u8(d45, vcombine_u8(t4, t5));
|
||||
d67 = vrhaddq_u8(d67, vcombine_u8(t6, t7));
|
||||
|
||||
store_u8_8x8(d, dst_stride, vget_low_u8(d01), vget_high_u8(d01),
|
||||
vget_low_u8(d23), vget_high_u8(d23), vget_low_u8(d45),
|
||||
vget_high_u8(d45), vget_low_u8(d67), vget_high_u8(d67));
|
||||
|
||||
s0 = s8;
|
||||
s1 = s9;
|
||||
s2 = s10;
|
||||
s3 = s11;
|
||||
s4 = s12;
|
||||
s5 = s13;
|
||||
s6 = s14;
|
||||
s += 8;
|
||||
d += 8;
|
||||
width -= 8;
|
||||
} while (width > 0);
|
||||
src += 8 * src_stride;
|
||||
dst += 8 * dst_stride;
|
||||
h -= 8;
|
||||
} while (h > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
const int16x8_t filters = vld1q_s16(filter[y0_q4]);
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
assert(y_step_q4 == 16);
|
||||
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
src -= 3 * src_stride;
|
||||
|
||||
if (w == 4) {
|
||||
const int16x4_t filter3 = vdup_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x4_t filter4 = vdup_lane_s16(vget_high_s16(filters), 0);
|
||||
uint8x8_t d01, d23;
|
||||
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, d0, d1, d2, d3;
|
||||
|
||||
s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
|
||||
do {
|
||||
s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
d0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
d1 = convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
d2 = convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
d3 = convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
|
||||
d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), 7);
|
||||
d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), 7);
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
|
||||
dst += dst_stride;
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
|
||||
int height;
|
||||
const uint8_t *s;
|
||||
uint8_t *d;
|
||||
uint8x8_t t0, t1, t2, t3;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
|
||||
do {
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
s = src;
|
||||
s0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s1 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s2 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s3 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s4 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s6 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
d = dst;
|
||||
height = h;
|
||||
|
||||
do {
|
||||
s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s8 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s9 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s10 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
|
||||
__builtin_prefetch(d + 0 * dst_stride);
|
||||
__builtin_prefetch(d + 1 * dst_stride);
|
||||
__builtin_prefetch(d + 2 * dst_stride);
|
||||
__builtin_prefetch(d + 3 * dst_stride);
|
||||
__builtin_prefetch(s + 0 * src_stride);
|
||||
__builtin_prefetch(s + 1 * src_stride);
|
||||
__builtin_prefetch(s + 2 * src_stride);
|
||||
__builtin_prefetch(s + 3 * src_stride);
|
||||
t0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
t1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
t2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
t3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
|
||||
vst1_u8(d, t0);
|
||||
d += dst_stride;
|
||||
vst1_u8(d, t1);
|
||||
d += dst_stride;
|
||||
vst1_u8(d, t2);
|
||||
d += dst_stride;
|
||||
vst1_u8(d, t3);
|
||||
d += dst_stride;
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
height -= 4;
|
||||
} while (height > 0);
|
||||
src += 8;
|
||||
dst += 8;
|
||||
w -= 8;
|
||||
} while (w > 0);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
const int16x8_t filters = vld1q_s16(filter[y0_q4]);
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
assert(y_step_q4 == 16);
|
||||
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
src -= 3 * src_stride;
|
||||
|
||||
if (w == 4) {
|
||||
const int16x4_t filter3 = vdup_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x4_t filter4 = vdup_lane_s16(vget_high_s16(filters), 0);
|
||||
uint8x8_t d01, d23;
|
||||
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, d0, d1, d2, d3;
|
||||
uint32x4_t d0123 = vdupq_n_u32(0);
|
||||
|
||||
s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
|
||||
do {
|
||||
s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
|
||||
src += src_stride;
|
||||
|
||||
__builtin_prefetch(dst + 0 * dst_stride);
|
||||
__builtin_prefetch(dst + 1 * dst_stride);
|
||||
__builtin_prefetch(dst + 2 * dst_stride);
|
||||
__builtin_prefetch(dst + 3 * dst_stride);
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
d0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
d1 = convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
d2 = convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
d3 = convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
|
||||
d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), 7);
|
||||
d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), 7);
|
||||
|
||||
d0123 = vld1q_lane_u32((uint32_t *)(dst + 0 * dst_stride), d0123, 0);
|
||||
d0123 = vld1q_lane_u32((uint32_t *)(dst + 1 * dst_stride), d0123, 1);
|
||||
d0123 = vld1q_lane_u32((uint32_t *)(dst + 2 * dst_stride), d0123, 2);
|
||||
d0123 = vld1q_lane_u32((uint32_t *)(dst + 3 * dst_stride), d0123, 3);
|
||||
d0123 = vreinterpretq_u32_u8(
|
||||
vrhaddq_u8(vreinterpretq_u8_u32(d0123), vcombine_u8(d01, d23)));
|
||||
|
||||
vst1q_lane_u32((uint32_t *)dst, d0123, 0);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d0123, 1);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d0123, 2);
|
||||
dst += dst_stride;
|
||||
vst1q_lane_u32((uint32_t *)dst, d0123, 3);
|
||||
dst += dst_stride;
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
|
||||
int height;
|
||||
const uint8_t *s;
|
||||
uint8_t *d;
|
||||
uint8x8_t t0, t1, t2, t3;
|
||||
uint8x16_t d01, d23, dd01, dd23;
|
||||
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
|
||||
do {
|
||||
__builtin_prefetch(src + 0 * src_stride);
|
||||
__builtin_prefetch(src + 1 * src_stride);
|
||||
__builtin_prefetch(src + 2 * src_stride);
|
||||
__builtin_prefetch(src + 3 * src_stride);
|
||||
__builtin_prefetch(src + 4 * src_stride);
|
||||
__builtin_prefetch(src + 5 * src_stride);
|
||||
__builtin_prefetch(src + 6 * src_stride);
|
||||
s = src;
|
||||
s0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s1 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s2 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s3 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s4 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s6 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
d = dst;
|
||||
height = h;
|
||||
|
||||
do {
|
||||
s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s8 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s9 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
s10 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
|
||||
s += src_stride;
|
||||
|
||||
__builtin_prefetch(d + 0 * dst_stride);
|
||||
__builtin_prefetch(d + 1 * dst_stride);
|
||||
__builtin_prefetch(d + 2 * dst_stride);
|
||||
__builtin_prefetch(d + 3 * dst_stride);
|
||||
__builtin_prefetch(s + 0 * src_stride);
|
||||
__builtin_prefetch(s + 1 * src_stride);
|
||||
__builtin_prefetch(s + 2 * src_stride);
|
||||
__builtin_prefetch(s + 3 * src_stride);
|
||||
t0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters, filter3,
|
||||
filter4);
|
||||
t1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters, filter3,
|
||||
filter4);
|
||||
t2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters, filter3,
|
||||
filter4);
|
||||
t3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters, filter3,
|
||||
filter4);
|
||||
|
||||
d01 = vcombine_u8(t0, t1);
|
||||
d23 = vcombine_u8(t2, t3);
|
||||
dd01 = vcombine_u8(vld1_u8(d + 0 * dst_stride),
|
||||
vld1_u8(d + 1 * dst_stride));
|
||||
dd23 = vcombine_u8(vld1_u8(d + 2 * dst_stride),
|
||||
vld1_u8(d + 3 * dst_stride));
|
||||
dd01 = vrhaddq_u8(dd01, d01);
|
||||
dd23 = vrhaddq_u8(dd23, d23);
|
||||
|
||||
vst1_u8(d, vget_low_u8(dd01));
|
||||
d += dst_stride;
|
||||
vst1_u8(d, vget_high_u8(dd01));
|
||||
d += dst_stride;
|
||||
vst1_u8(d, vget_low_u8(dd23));
|
||||
d += dst_stride;
|
||||
vst1_u8(d, vget_high_u8(dd23));
|
||||
d += dst_stride;
|
||||
|
||||
s0 = s4;
|
||||
s1 = s5;
|
||||
s2 = s6;
|
||||
s3 = s7;
|
||||
s4 = s8;
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
height -= 4;
|
||||
} while (height > 0);
|
||||
src += 8;
|
||||
dst += 8;
|
||||
w -= 8;
|
||||
} while (w > 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_H_
|
||||
#define VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_H_
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
|
||||
static INLINE void load_u8_8x4(const uint8_t *s, const ptrdiff_t p,
|
||||
uint8x8_t *const s0, uint8x8_t *const s1,
|
||||
uint8x8_t *const s2, uint8x8_t *const s3) {
|
||||
*s0 = vld1_u8(s);
|
||||
s += p;
|
||||
*s1 = vld1_u8(s);
|
||||
s += p;
|
||||
*s2 = vld1_u8(s);
|
||||
s += p;
|
||||
*s3 = vld1_u8(s);
|
||||
}
|
||||
|
||||
static INLINE void load_u8_8x8(const uint8_t *s, const ptrdiff_t p,
|
||||
uint8x8_t *const s0, uint8x8_t *const s1,
|
||||
uint8x8_t *const s2, uint8x8_t *const s3,
|
||||
uint8x8_t *const s4, uint8x8_t *const s5,
|
||||
uint8x8_t *const s6, uint8x8_t *const s7) {
|
||||
*s0 = vld1_u8(s);
|
||||
s += p;
|
||||
*s1 = vld1_u8(s);
|
||||
s += p;
|
||||
*s2 = vld1_u8(s);
|
||||
s += p;
|
||||
*s3 = vld1_u8(s);
|
||||
s += p;
|
||||
*s4 = vld1_u8(s);
|
||||
s += p;
|
||||
*s5 = vld1_u8(s);
|
||||
s += p;
|
||||
*s6 = vld1_u8(s);
|
||||
s += p;
|
||||
*s7 = vld1_u8(s);
|
||||
}
|
||||
|
||||
static INLINE void load_u8_16x8(const uint8_t *s, const ptrdiff_t p,
|
||||
uint8x16_t *const s0, uint8x16_t *const s1,
|
||||
uint8x16_t *const s2, uint8x16_t *const s3,
|
||||
uint8x16_t *const s4, uint8x16_t *const s5,
|
||||
uint8x16_t *const s6, uint8x16_t *const s7) {
|
||||
*s0 = vld1q_u8(s);
|
||||
s += p;
|
||||
*s1 = vld1q_u8(s);
|
||||
s += p;
|
||||
*s2 = vld1q_u8(s);
|
||||
s += p;
|
||||
*s3 = vld1q_u8(s);
|
||||
s += p;
|
||||
*s4 = vld1q_u8(s);
|
||||
s += p;
|
||||
*s5 = vld1q_u8(s);
|
||||
s += p;
|
||||
*s6 = vld1q_u8(s);
|
||||
s += p;
|
||||
*s7 = vld1q_u8(s);
|
||||
}
|
||||
|
||||
static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1,
|
||||
const int16x4_t s2, const int16x4_t s3,
|
||||
const int16x4_t s4, const int16x4_t s5,
|
||||
const int16x4_t s6, const int16x4_t s7,
|
||||
const int16x8_t filters,
|
||||
const int16x4_t filter3,
|
||||
const int16x4_t filter4) {
|
||||
const int16x4_t filters_lo = vget_low_s16(filters);
|
||||
const int16x4_t filters_hi = vget_high_s16(filters);
|
||||
int16x4_t sum;
|
||||
|
||||
sum = vmul_lane_s16(s0, filters_lo, 0);
|
||||
sum = vmla_lane_s16(sum, s1, filters_lo, 1);
|
||||
sum = vmla_lane_s16(sum, s2, filters_lo, 2);
|
||||
sum = vmla_lane_s16(sum, s5, filters_hi, 1);
|
||||
sum = vmla_lane_s16(sum, s6, filters_hi, 2);
|
||||
sum = vmla_lane_s16(sum, s7, filters_hi, 3);
|
||||
sum = vqadd_s16(sum, vmul_s16(s3, filter3));
|
||||
sum = vqadd_s16(sum, vmul_s16(s4, filter4));
|
||||
return sum;
|
||||
}
|
||||
|
||||
static INLINE uint8x8_t convolve8_8(const int16x8_t s0, const int16x8_t s1,
|
||||
const int16x8_t s2, const int16x8_t s3,
|
||||
const int16x8_t s4, const int16x8_t s5,
|
||||
const int16x8_t s6, const int16x8_t s7,
|
||||
const int16x8_t filters,
|
||||
const int16x8_t filter3,
|
||||
const int16x8_t filter4) {
|
||||
const int16x4_t filters_lo = vget_low_s16(filters);
|
||||
const int16x4_t filters_hi = vget_high_s16(filters);
|
||||
int16x8_t sum;
|
||||
|
||||
sum = vmulq_lane_s16(s0, filters_lo, 0);
|
||||
sum = vmlaq_lane_s16(sum, s1, filters_lo, 1);
|
||||
sum = vmlaq_lane_s16(sum, s2, filters_lo, 2);
|
||||
sum = vmlaq_lane_s16(sum, s5, filters_hi, 1);
|
||||
sum = vmlaq_lane_s16(sum, s6, filters_hi, 2);
|
||||
sum = vmlaq_lane_s16(sum, s7, filters_hi, 3);
|
||||
sum = vqaddq_s16(sum, vmulq_s16(s3, filter3));
|
||||
sum = vqaddq_s16(sum, vmulq_s16(s4, filter4));
|
||||
return vqrshrun_n_s16(sum, 7);
|
||||
}
|
||||
|
||||
static INLINE uint8x8_t scale_filter_8(const uint8x8_t *const s,
|
||||
const int16x8_t filters) {
|
||||
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
|
||||
int16x8_t ss[8];
|
||||
|
||||
ss[0] = vreinterpretq_s16_u16(vmovl_u8(s[0]));
|
||||
ss[1] = vreinterpretq_s16_u16(vmovl_u8(s[1]));
|
||||
ss[2] = vreinterpretq_s16_u16(vmovl_u8(s[2]));
|
||||
ss[3] = vreinterpretq_s16_u16(vmovl_u8(s[3]));
|
||||
ss[4] = vreinterpretq_s16_u16(vmovl_u8(s[4]));
|
||||
ss[5] = vreinterpretq_s16_u16(vmovl_u8(s[5]));
|
||||
ss[6] = vreinterpretq_s16_u16(vmovl_u8(s[6]));
|
||||
ss[7] = vreinterpretq_s16_u16(vmovl_u8(s[7]));
|
||||
|
||||
return convolve8_8(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7],
|
||||
filters, filter3, filter4);
|
||||
}
|
||||
|
||||
#endif // VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vp9/common/vp9_filter.h"
|
||||
#include "vpx_dsp/arm/vpx_convolve8_neon_asm.h"
|
||||
|
||||
/* Type1 and Type2 functions are called depending on the position of the
|
||||
* negative and positive coefficients in the filter. In type1, the filter kernel
|
||||
* used is sub_pel_filters_8lp, in which only the first two and the last two
|
||||
* coefficients are negative. In type2, the negative coefficients are 0, 2, 5 &
|
||||
* 7.
|
||||
*/
|
||||
|
||||
#define DEFINE_FILTER(dir) \
|
||||
void vpx_convolve8_##dir##_neon( \
|
||||
const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \
|
||||
ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w, int h) { \
|
||||
if (filter == vp9_filter_kernels[1]) { \
|
||||
vpx_convolve8_##dir##_filter_type1_neon( \
|
||||
src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, y0_q4, \
|
||||
y_step_q4, w, h); \
|
||||
} else { \
|
||||
vpx_convolve8_##dir##_filter_type2_neon( \
|
||||
src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, y0_q4, \
|
||||
y_step_q4, w, h); \
|
||||
} \
|
||||
}
|
||||
|
||||
DEFINE_FILTER(horiz);
|
||||
DEFINE_FILTER(avg_horiz);
|
||||
DEFINE_FILTER(vert);
|
||||
DEFINE_FILTER(avg_vert);
|
||||
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_ASM_H_
|
||||
#define VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_ASM_H_
|
||||
|
||||
#define DECLARE_FILTER(dir, type) \
|
||||
void vpx_convolve8_##dir##_filter_##type##_neon( \
|
||||
const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \
|
||||
ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w, int h);
|
||||
|
||||
DECLARE_FILTER(horiz, type1);
|
||||
DECLARE_FILTER(avg_horiz, type1);
|
||||
DECLARE_FILTER(horiz, type2);
|
||||
DECLARE_FILTER(avg_horiz, type2);
|
||||
DECLARE_FILTER(vert, type1);
|
||||
DECLARE_FILTER(avg_vert, type1);
|
||||
DECLARE_FILTER(vert, type2);
|
||||
DECLARE_FILTER(avg_vert, type2);
|
||||
|
||||
#endif // VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_ASM_H_
|
||||
+457
@@ -0,0 +1,457 @@
|
||||
;
|
||||
; Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
;**************Variables Vs Registers***********************************
|
||||
; r0 => src
|
||||
; r1 => dst
|
||||
; r2 => src_stride
|
||||
; r6 => dst_stride
|
||||
; r12 => filter_y0
|
||||
; r5 => ht
|
||||
; r3 => wd
|
||||
|
||||
EXPORT |vpx_convolve8_vert_filter_type1_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve8_vert_filter_type1_neon| PROC
|
||||
|
||||
stmfd sp!, {r4 - r12, r14} ;stack stores the values of
|
||||
; the arguments
|
||||
vpush {d8 - d15} ; stack offset by 64
|
||||
mov r4, r1
|
||||
mov r1, r2
|
||||
mov r2, r4
|
||||
vmov.i16 q15, #0x4000
|
||||
mov r11, #0xc000
|
||||
ldr r12, [sp, #104] ;load filter
|
||||
ldr r6, [sp, #116] ;load y0_q4
|
||||
add r12, r12, r6, lsl #4 ;r12 = filter[y0_q4]
|
||||
mov r6, r3
|
||||
ldr r5, [sp, #124] ;load wd
|
||||
vld2.8 {d0, d1}, [r12] ;coeff = vld1_s8(pi1_coeff)
|
||||
sub r12, r2, r2, lsl #2 ;src_ctrd & pi1_coeff
|
||||
vabs.s8 d0, d0 ;vabs_s8(coeff)
|
||||
add r0, r0, r12 ;r0->pu1_src r12->pi1_coeff
|
||||
ldr r3, [sp, #128] ;load ht
|
||||
subs r7, r3, #0 ;r3->ht
|
||||
vdup.u8 d22, d0[0] ;coeffabs_0 = vdup_lane_u8(coeffabs,
|
||||
; 0);
|
||||
cmp r5, #8
|
||||
vdup.u8 d23, d0[1] ;coeffabs_1 = vdup_lane_u8(coeffabs,
|
||||
; 1);
|
||||
vdup.u8 d24, d0[2] ;coeffabs_2 = vdup_lane_u8(coeffabs,
|
||||
; 2);
|
||||
vdup.u8 d25, d0[3] ;coeffabs_3 = vdup_lane_u8(coeffabs,
|
||||
; 3);
|
||||
vdup.u8 d26, d0[4] ;coeffabs_4 = vdup_lane_u8(coeffabs,
|
||||
; 4);
|
||||
vdup.u8 d27, d0[5] ;coeffabs_5 = vdup_lane_u8(coeffabs,
|
||||
; 5);
|
||||
vdup.u8 d28, d0[6] ;coeffabs_6 = vdup_lane_u8(coeffabs,
|
||||
; 6);
|
||||
vdup.u8 d29, d0[7] ;coeffabs_7 = vdup_lane_u8(coeffabs,
|
||||
; 7);
|
||||
blt core_loop_wd_4 ;core loop wd 4 jump
|
||||
|
||||
str r0, [sp, #-4]!
|
||||
str r1, [sp, #-4]!
|
||||
bic r4, r5, #7 ;r5 ->wd
|
||||
rsb r9, r4, r6, lsl #2 ;r6->dst_strd r5 ->wd
|
||||
rsb r8, r4, r2, lsl #2 ;r2->src_strd
|
||||
mov r3, r5, lsr #3 ;divide by 8
|
||||
mul r7, r3 ;multiply height by width
|
||||
sub r7, #4 ;subtract by one for epilog
|
||||
|
||||
prolog
|
||||
and r10, r0, #31
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vdup.16 q4, r11
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
subs r4, r4, #8
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vdup.16 q5, r11
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
addle r0, r0, r8
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
bicle r4, r5, #7 ;r5 ->wd
|
||||
vmlal.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
pld [r3]
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
pld [r3, r2]
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
pld [r3, r2, lsl #1]
|
||||
vmlal.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
add r3, r3, r2
|
||||
vmlsl.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
pld [r3, r2, lsl #1]
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d3, d23
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d4, d24
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d5, d25
|
||||
vmlal.u8 q6, d6, d26
|
||||
vmlal.u8 q6, d7, d27
|
||||
vmlsl.u8 q6, d16, d28
|
||||
vmlsl.u8 q6, d17, d29
|
||||
add r14, r1, r6
|
||||
vst1.8 {d8}, [r1]! ;vst1_u8(pu1_dst,sto_res);
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
addle r1, r1, r9
|
||||
vmlsl.u8 q7, d4, d23
|
||||
subs r7, r7, #4
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vmlal.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vhadd.s16 q6, q6, q15
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q7, d7, d26
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d16, d27
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d17, d28
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
blt epilog_end ;jumps to epilog_end
|
||||
|
||||
beq epilog ;jumps to epilog
|
||||
|
||||
main_loop_8
|
||||
subs r4, r4, #8
|
||||
vmlsl.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
addle r0, r0, r8
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
bicle r4, r5, #7 ;r5 ->wd
|
||||
vmlal.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vhadd.s16 q7, q7, q15
|
||||
vdup.16 q5, r11
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vmlsl.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vmlsl.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
vmlal.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
vst1.8 {d14}, [r14], r6
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
add r14, r1, #0
|
||||
vmlal.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
add r1, r1, #8
|
||||
vmlsl.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
addle r1, r1, r9
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vmlsl.u8 q6, d3, d23
|
||||
add r10, r3, r2, lsl #3 ; 10*strd - 8+2
|
||||
vmlsl.u8 q6, d2, d22
|
||||
add r10, r10, r2 ; 11*strd
|
||||
vmlal.u8 q6, d4, d24
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d5, d25
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d6, d26
|
||||
vst1.8 {d8}, [r14], r6 ;vst1_u8(pu1_dst,sto_res);
|
||||
pld [r10] ;11+ 0
|
||||
vmlal.u8 q6, d7, d27
|
||||
pld [r10, r2] ;11+ 1*strd
|
||||
vmlsl.u8 q6, d16, d28
|
||||
pld [r10, r2, lsl #1] ;11+ 2*strd
|
||||
vmlsl.u8 q6, d17, d29
|
||||
add r10, r10, r2 ;12*strd
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
pld [r10, r2, lsl #1] ;11+ 3*strd
|
||||
vmlsl.u8 q7, d4, d23
|
||||
vmlsl.u8 q7, d3, d22
|
||||
subs r7, r7, #4
|
||||
vmlal.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vhadd.s16 q6, q6, q15
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q7, d7, d26
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d16, d27
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d17, d28
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
bgt main_loop_8 ;jumps to main_loop_8
|
||||
|
||||
epilog
|
||||
vmlsl.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
vmlal.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vhadd.s16 q7, q7, q15
|
||||
vdup.16 q5, r11
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vmlal.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vmlsl.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
vmlal.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
vmlal.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
vmlsl.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
vst1.8 {d14}, [r14], r6
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d3, d23
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vmlal.u8 q6, d4, d24
|
||||
vmlal.u8 q6, d5, d25
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d6, d26
|
||||
vmlal.u8 q6, d7, d27
|
||||
vmlsl.u8 q6, d16, d28
|
||||
vmlsl.u8 q6, d17, d29
|
||||
add r14, r1, r6
|
||||
vst1.8 {d8}, [r1]! ;vst1_u8(pu1_dst,sto_res);
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d4, d23
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vmlal.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vhadd.s16 q6, q6, q15
|
||||
vmlal.u8 q7, d7, d26
|
||||
vmlal.u8 q7, d16, d27
|
||||
vmlsl.u8 q7, d17, d28
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
|
||||
epilog_end
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vhadd.s16 q7, q7, q15
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
vst1.8 {d14}, [r14], r6
|
||||
|
||||
end_loops
|
||||
tst r5, #7
|
||||
ldr r1, [sp], #4
|
||||
ldr r0, [sp], #4
|
||||
vpopeq {d8 - d15}
|
||||
ldmfdeq sp!, {r4 - r12, r15} ;reload the registers from
|
||||
; sp
|
||||
mov r5, #4
|
||||
add r0, r0, #8
|
||||
add r1, r1, #8
|
||||
mov r7, #16
|
||||
|
||||
core_loop_wd_4
|
||||
rsb r9, r5, r6, lsl #2 ;r6->dst_strd r5 ->wd
|
||||
rsb r8, r5, r2, lsl #2 ;r2->src_strd
|
||||
vmov.i8 d4, #0
|
||||
|
||||
outer_loop_wd_4
|
||||
subs r12, r5, #0
|
||||
ble end_inner_loop_wd_4 ;outer loop jump
|
||||
|
||||
inner_loop_wd_4
|
||||
add r3, r0, r2
|
||||
vld1.u32 {d4[1]},[r3], r2 ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 1);
|
||||
subs r12, r12, #4
|
||||
vdup.u32 d5, d4[1] ;src_tmp2 = vdup_lane_u32(src_tmp1,
|
||||
; 1);
|
||||
vld1.u32 {d5[1]},[r3], r2 ;src_tmp2 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp2, 1);
|
||||
vld1.u32 {d4[0]},[r0] ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 0);
|
||||
vdup.16 q0, r11
|
||||
vmlsl.u8 q0, d5, d23 ;mul_res1 =
|
||||
; vmull_u8(vreinterpret_u8_u32(src_tmp2), coeffabs_1);
|
||||
|
||||
vdup.u32 d6, d5[1] ;src_tmp3 = vdup_lane_u32(src_tmp2,
|
||||
; 1);
|
||||
add r0, r0, #4
|
||||
vld1.u32 {d6[1]},[r3], r2 ;src_tmp3 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp3, 1);
|
||||
vmlsl.u8 q0, d4, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp1), coeffabs_0);
|
||||
vdup.u32 d7, d6[1] ;src_tmp4 = vdup_lane_u32(src_tmp3,
|
||||
; 1);
|
||||
vld1.u32 {d7[1]},[r3], r2 ;src_tmp4 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp4, 1);
|
||||
vmlal.u8 q0, d6, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp3), coeffabs_2);
|
||||
vdup.16 q4, r11
|
||||
vmlsl.u8 q4, d7, d23
|
||||
vdup.u32 d4, d7[1] ;src_tmp1 = vdup_lane_u32(src_tmp4,
|
||||
; 1);
|
||||
vmull.u8 q1, d7, d25 ;mul_res2 =
|
||||
; vmull_u8(vreinterpret_u8_u32(src_tmp4), coeffabs_3);
|
||||
vld1.u32 {d4[1]},[r3], r2 ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 1);
|
||||
vmlsl.u8 q4, d6, d22
|
||||
vmlal.u8 q0, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp1), coeffabs_4);
|
||||
vdup.u32 d5, d4[1] ;src_tmp2 = vdup_lane_u32(src_tmp1,
|
||||
; 1);
|
||||
vmlal.u8 q4, d4, d24
|
||||
vld1.u32 {d5[1]},[r3], r2 ;src_tmp2 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp2, 1);
|
||||
vmlal.u8 q1, d5, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; vreinterpret_u8_u32(src_tmp2), coeffabs_5);
|
||||
vdup.u32 d6, d5[1] ;src_tmp3 = vdup_lane_u32(src_tmp2,
|
||||
; 1);
|
||||
vmlal.u8 q4, d5, d25
|
||||
vld1.u32 {d6[1]},[r3], r2 ;src_tmp3 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp3, 1);
|
||||
vmlsl.u8 q0, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp3), coeffabs_6);
|
||||
vdup.u32 d7, d6[1] ;src_tmp4 = vdup_lane_u32(src_tmp3,
|
||||
; 1);
|
||||
vmlal.u8 q4, d6, d26
|
||||
vld1.u32 {d7[1]},[r3], r2 ;src_tmp4 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp4, 1);
|
||||
vmlsl.u8 q1, d7, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; vreinterpret_u8_u32(src_tmp4), coeffabs_7);
|
||||
vdup.u32 d4, d7[1]
|
||||
vadd.i16 q0, q0, q1 ;mul_res1 = vaddq_u16(mul_res1,
|
||||
; mul_res2);
|
||||
vmlal.u8 q4, d7, d27
|
||||
vld1.u32 {d4[1]},[r3], r2
|
||||
vmlsl.u8 q4, d4, d28
|
||||
vdup.u32 d5, d4[1]
|
||||
vhadd.s16 q0, q0, q15
|
||||
vqrshrun.s16 d0, q0, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u32 {d5[1]},[r3]
|
||||
add r3, r1, r6
|
||||
vst1.32 {d0[0]},[r1] ;vst1_lane_u32((uint32_t *)pu1_dst,
|
||||
; vreinterpret_u32_u8(sto_res), 0);
|
||||
vmlsl.u8 q4, d5, d29
|
||||
vst1.32 {d0[1]},[r3], r6 ;vst1_lane_u32((uint32_t
|
||||
; *)pu1_dst_tmp, vreinterpret_u32_u8(sto_res), 1);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vqrshrun.s16 d8, q4, #6
|
||||
vst1.32 {d8[0]},[r3], r6
|
||||
add r1, r1, #4
|
||||
vst1.32 {d8[1]},[r3]
|
||||
bgt inner_loop_wd_4
|
||||
|
||||
end_inner_loop_wd_4
|
||||
subs r7, r7, #4
|
||||
add r1, r1, r9
|
||||
add r0, r0, r8
|
||||
bgt outer_loop_wd_4
|
||||
|
||||
vpop {d8 - d15}
|
||||
ldmfd sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
|
||||
ENDP
|
||||
|
||||
END
|
||||
+455
@@ -0,0 +1,455 @@
|
||||
;
|
||||
; Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
;**************Variables Vs Registers***********************************
|
||||
; r0 => src
|
||||
; r1 => dst
|
||||
; r2 => src_stride
|
||||
; r6 => dst_stride
|
||||
; r12 => filter_y0
|
||||
; r5 => ht
|
||||
; r3 => wd
|
||||
|
||||
EXPORT |vpx_convolve8_vert_filter_type2_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve8_vert_filter_type2_neon| PROC
|
||||
|
||||
stmfd sp!, {r4 - r12, r14} ;stack stores the values of
|
||||
; the arguments
|
||||
vpush {d8 - d15} ; stack offset by 64
|
||||
mov r4, r1
|
||||
mov r1, r2
|
||||
mov r2, r4
|
||||
vmov.i16 q15, #0x4000
|
||||
mov r11, #0xc000
|
||||
ldr r12, [sp, #104] ;load filter
|
||||
ldr r6, [sp, #116] ;load y0_q4
|
||||
add r12, r12, r6, lsl #4 ;r12 = filter[y0_q4]
|
||||
mov r6, r3
|
||||
ldr r5, [sp, #124] ;load wd
|
||||
vld2.8 {d0, d1}, [r12] ;coeff = vld1_s8(pi1_coeff)
|
||||
sub r12, r2, r2, lsl #2 ;src_ctrd & pi1_coeff
|
||||
vabs.s8 d0, d0 ;vabs_s8(coeff)
|
||||
add r0, r0, r12 ;r0->pu1_src r12->pi1_coeff
|
||||
ldr r3, [sp, #128] ;load ht
|
||||
subs r7, r3, #0 ;r3->ht
|
||||
vdup.u8 d22, d0[0] ;coeffabs_0 = vdup_lane_u8(coeffabs,
|
||||
; 0);
|
||||
cmp r5, #8
|
||||
vdup.u8 d23, d0[1] ;coeffabs_1 = vdup_lane_u8(coeffabs,
|
||||
; 1);
|
||||
vdup.u8 d24, d0[2] ;coeffabs_2 = vdup_lane_u8(coeffabs,
|
||||
; 2);
|
||||
vdup.u8 d25, d0[3] ;coeffabs_3 = vdup_lane_u8(coeffabs,
|
||||
; 3);
|
||||
vdup.u8 d26, d0[4] ;coeffabs_4 = vdup_lane_u8(coeffabs,
|
||||
; 4);
|
||||
vdup.u8 d27, d0[5] ;coeffabs_5 = vdup_lane_u8(coeffabs,
|
||||
; 5);
|
||||
vdup.u8 d28, d0[6] ;coeffabs_6 = vdup_lane_u8(coeffabs,
|
||||
; 6);
|
||||
vdup.u8 d29, d0[7] ;coeffabs_7 = vdup_lane_u8(coeffabs,
|
||||
; 7);
|
||||
blt core_loop_wd_4 ;core loop wd 4 jump
|
||||
|
||||
str r0, [sp, #-4]!
|
||||
str r1, [sp, #-4]!
|
||||
bic r4, r5, #7 ;r5 ->wd
|
||||
rsb r9, r4, r6, lsl #2 ;r6->dst_strd r5 ->wd
|
||||
rsb r8, r4, r2, lsl #2 ;r2->src_strd
|
||||
mov r3, r5, lsr #3 ;divide by 8
|
||||
mul r7, r3 ;multiply height by width
|
||||
sub r7, #4 ;subtract by one for epilog
|
||||
|
||||
prolog
|
||||
and r10, r0, #31
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vdup.16 q4, r11
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
subs r4, r4, #8
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vdup.16 q5, r11
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
addle r0, r0, r8
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
bicle r4, r5, #7 ;r5 ->wd
|
||||
vmlsl.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
pld [r3]
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
pld [r3, r2]
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
pld [r3, r2, lsl #1]
|
||||
vmlsl.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
add r3, r3, r2
|
||||
vmlal.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
pld [r3, r2, lsl #1]
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d3, d23
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q6, d4, d24
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d5, d25
|
||||
vmlal.u8 q6, d6, d26
|
||||
vmlsl.u8 q6, d7, d27
|
||||
vmlal.u8 q6, d16, d28
|
||||
vmlsl.u8 q6, d17, d29
|
||||
add r14, r1, r6
|
||||
vst1.8 {d8}, [r1]! ;vst1_u8(pu1_dst,sto_res);
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
addle r1, r1, r9
|
||||
vmlal.u8 q7, d4, d23
|
||||
subs r7, r7, #4
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vmlsl.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vhadd.s16 q6, q6, q15
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q7, d7, d26
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d16, d27
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d17, d28
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
blt epilog_end ;jumps to epilog_end
|
||||
|
||||
beq epilog ;jumps to epilog
|
||||
|
||||
main_loop_8
|
||||
subs r4, r4, #8
|
||||
vmlal.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
; coeffabs_1);
|
||||
addle r0, r0, r8
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
bicle r4, r5, #7 ;r5 ->wd
|
||||
vmlsl.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vhadd.s16 q7, q7, q15
|
||||
vdup.16 q5, r11
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vmlal.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
add r3, r0, r2 ;pu1_src_tmp += src_strd;
|
||||
vmlal.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
vld1.u8 {d0}, [r0]! ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
vmlsl.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
vld1.u8 {d1}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
vst1.8 {d14}, [r14], r6
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
add r14, r1, #0
|
||||
vmlsl.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
add r1, r1, #8
|
||||
vmlal.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
addle r1, r1, r9
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vmlal.u8 q6, d3, d23
|
||||
add r10, r3, r2, lsl #3 ; 10*strd - 8+2
|
||||
vmlsl.u8 q6, d2, d22
|
||||
add r10, r10, r2 ; 11*strd
|
||||
vmlsl.u8 q6, d4, d24
|
||||
vld1.u8 {d2}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d5, d25
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d6, d26
|
||||
vst1.8 {d8}, [r14], r6 ;vst1_u8(pu1_dst,sto_res);
|
||||
pld [r10] ;11+ 0
|
||||
vmlsl.u8 q6, d7, d27
|
||||
pld [r10, r2] ;11+ 1*strd
|
||||
vmlal.u8 q6, d16, d28
|
||||
pld [r10, r2, lsl #1] ;11+ 2*strd
|
||||
vmlsl.u8 q6, d17, d29
|
||||
add r10, r10, r2 ;12*strd
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
pld [r10, r2, lsl #1] ;11+ 3*strd
|
||||
vmlal.u8 q7, d4, d23
|
||||
vmlsl.u8 q7, d3, d22
|
||||
subs r7, r7, #4
|
||||
vmlsl.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vld1.u8 {d3}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vhadd.s16 q6, q6, q15
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q7, d7, d26
|
||||
vld1.u8 {d4}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d16, d27
|
||||
vld1.u8 {d5}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d17, d28
|
||||
vld1.u8 {d6}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vld1.u8 {d7}, [r3], r2 ;src_tmp4 = vld1_u8(pu1_src_tmp);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
bgt main_loop_8 ;jumps to main_loop_8
|
||||
|
||||
epilog
|
||||
vmlal.u8 q4, d1, d23 ;mul_res1 = vmull_u8(src_tmp2,
|
||||
vmlsl.u8 q4, d0, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_0);
|
||||
vmlsl.u8 q4, d2, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_2);
|
||||
vmlal.u8 q4, d3, d25 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_3);
|
||||
vhadd.s16 q7, q7, q15
|
||||
vdup.16 q5, r11
|
||||
vmlal.u8 q4, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp1, coeffabs_4);
|
||||
vmlsl.u8 q4, d5, d27 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp2, coeffabs_5);
|
||||
vmlal.u8 q4, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; src_tmp3, coeffabs_6);
|
||||
vmlsl.u8 q4, d7, d29 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; src_tmp4, coeffabs_7);
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
vld1.u8 {d16}, [r3], r2 ;src_tmp1 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q5, d2, d23 ;mul_res2 = vmull_u8(src_tmp3,
|
||||
; coeffabs_1);
|
||||
vmlsl.u8 q5, d1, d22 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_0);
|
||||
vmlsl.u8 q5, d3, d24 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_2);
|
||||
vmlal.u8 q5, d4, d25 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_3);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vdup.16 q6, r11
|
||||
vmlal.u8 q5, d5, d26 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp2, coeffabs_4);
|
||||
vmlsl.u8 q5, d6, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp3, coeffabs_5);
|
||||
vmlal.u8 q5, d7, d28 ;mul_res2 = vmlal_u8(mul_res2,
|
||||
; src_tmp4, coeffabs_6);
|
||||
vmlsl.u8 q5, d16, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; src_tmp1, coeffabs_7);
|
||||
vst1.8 {d14}, [r14], r6
|
||||
vqrshrun.s16 d8, q4, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d17}, [r3], r2 ;src_tmp2 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q6, d3, d23
|
||||
vmlsl.u8 q6, d2, d22
|
||||
vmlsl.u8 q6, d4, d24
|
||||
vmlal.u8 q6, d5, d25
|
||||
vhadd.s16 q5, q5, q15
|
||||
vdup.16 q7, r11
|
||||
vmlal.u8 q6, d6, d26
|
||||
vmlsl.u8 q6, d7, d27
|
||||
vmlal.u8 q6, d16, d28
|
||||
vmlsl.u8 q6, d17, d29
|
||||
add r14, r1, r6
|
||||
vst1.8 {d8}, [r1]! ;vst1_u8(pu1_dst,sto_res);
|
||||
vqrshrun.s16 d10, q5, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u8 {d18}, [r3], r2 ;src_tmp3 = vld1_u8(pu1_src_tmp);
|
||||
vmlal.u8 q7, d4, d23
|
||||
vmlsl.u8 q7, d3, d22
|
||||
vmlsl.u8 q7, d5, d24
|
||||
vmlal.u8 q7, d6, d25
|
||||
vhadd.s16 q6, q6, q15
|
||||
vmlal.u8 q7, d7, d26
|
||||
vmlsl.u8 q7, d16, d27
|
||||
vmlal.u8 q7, d17, d28
|
||||
vmlsl.u8 q7, d18, d29
|
||||
vst1.8 {d10}, [r14], r6 ;vst1_u8(pu1_dst_tmp,sto_res);
|
||||
vqrshrun.s16 d12, q6, #6
|
||||
|
||||
epilog_end
|
||||
vst1.8 {d12}, [r14], r6
|
||||
vhadd.s16 q7, q7, q15
|
||||
vqrshrun.s16 d14, q7, #6
|
||||
vst1.8 {d14}, [r14], r6
|
||||
|
||||
end_loops
|
||||
tst r5, #7
|
||||
ldr r1, [sp], #4
|
||||
ldr r0, [sp], #4
|
||||
vpopeq {d8 - d15}
|
||||
ldmfdeq sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
mov r5, #4
|
||||
add r0, r0, #8
|
||||
add r1, r1, #8
|
||||
mov r7, #16
|
||||
|
||||
core_loop_wd_4
|
||||
rsb r9, r5, r6, lsl #2 ;r6->dst_strd r5 ->wd
|
||||
rsb r8, r5, r2, lsl #2 ;r2->src_strd
|
||||
vmov.i8 d4, #0
|
||||
|
||||
outer_loop_wd_4
|
||||
subs r12, r5, #0
|
||||
ble end_inner_loop_wd_4 ;outer loop jump
|
||||
|
||||
inner_loop_wd_4
|
||||
add r3, r0, r2
|
||||
vld1.u32 {d4[1]},[r3], r2 ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 1);
|
||||
subs r12, r12, #4
|
||||
vdup.u32 d5, d4[1] ;src_tmp2 = vdup_lane_u32(src_tmp1,
|
||||
; 1);
|
||||
vld1.u32 {d5[1]},[r3], r2 ;src_tmp2 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp2, 1);
|
||||
vld1.u32 {d4[0]},[r0] ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 0);
|
||||
vdup.16 q0, r11
|
||||
vmlal.u8 q0, d5, d23 ;mul_res1 =
|
||||
; vmull_u8(vreinterpret_u8_u32(src_tmp2), coeffabs_1);
|
||||
vdup.u32 d6, d5[1] ;src_tmp3 = vdup_lane_u32(src_tmp2,
|
||||
; 1);
|
||||
add r0, r0, #4
|
||||
vld1.u32 {d6[1]},[r3], r2 ;src_tmp3 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp3, 1);
|
||||
vmlsl.u8 q0, d4, d22 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp1), coeffabs_0);
|
||||
vdup.u32 d7, d6[1] ;src_tmp4 = vdup_lane_u32(src_tmp3,
|
||||
; 1);
|
||||
vld1.u32 {d7[1]},[r3], r2 ;src_tmp4 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp4, 1);
|
||||
vmlsl.u8 q0, d6, d24 ;mul_res1 = vmlsl_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp3), coeffabs_2);
|
||||
vdup.16 q4, r11
|
||||
vmlal.u8 q4, d7, d23
|
||||
vdup.u32 d4, d7[1] ;src_tmp1 = vdup_lane_u32(src_tmp4,
|
||||
; 1);
|
||||
vmull.u8 q1, d7, d25 ;mul_res2 =
|
||||
; vmull_u8(vreinterpret_u8_u32(src_tmp4), coeffabs_3);
|
||||
vld1.u32 {d4[1]},[r3], r2 ;src_tmp1 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp1, 1);
|
||||
vmlsl.u8 q4, d6, d22
|
||||
vmlal.u8 q0, d4, d26 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp1), coeffabs_4);
|
||||
vdup.u32 d5, d4[1] ;src_tmp2 = vdup_lane_u32(src_tmp1,
|
||||
; 1);
|
||||
vmlsl.u8 q4, d4, d24
|
||||
vld1.u32 {d5[1]},[r3], r2 ;src_tmp2 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp2, 1);
|
||||
vmlsl.u8 q1, d5, d27 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; vreinterpret_u8_u32(src_tmp2), coeffabs_5);
|
||||
vdup.u32 d6, d5[1] ;src_tmp3 = vdup_lane_u32(src_tmp2,
|
||||
; 1);
|
||||
vmlal.u8 q4, d5, d25
|
||||
vld1.u32 {d6[1]},[r3], r2 ;src_tmp3 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp3, 1);
|
||||
vmlal.u8 q0, d6, d28 ;mul_res1 = vmlal_u8(mul_res1,
|
||||
; vreinterpret_u8_u32(src_tmp3), coeffabs_6);
|
||||
vdup.u32 d7, d6[1] ;src_tmp4 = vdup_lane_u32(src_tmp3,
|
||||
; 1);
|
||||
vmlal.u8 q4, d6, d26
|
||||
vld1.u32 {d7[1]},[r3], r2 ;src_tmp4 = vld1_lane_u32((uint32_t
|
||||
; *)pu1_src_tmp, src_tmp4, 1);
|
||||
vmlsl.u8 q1, d7, d29 ;mul_res2 = vmlsl_u8(mul_res2,
|
||||
; vreinterpret_u8_u32(src_tmp4), coeffabs_7);
|
||||
vdup.u32 d4, d7[1]
|
||||
vadd.i16 q0, q0, q1 ;mul_res1 = vaddq_u16(mul_res1,
|
||||
; mul_res2);
|
||||
vmlsl.u8 q4, d7, d27
|
||||
vld1.u32 {d4[1]},[r3], r2
|
||||
vmlal.u8 q4, d4, d28
|
||||
vdup.u32 d5, d4[1]
|
||||
vhadd.s16 q0, q0, q15
|
||||
vqrshrun.s16 d0, q0, #6 ;sto_res = vqmovun_s16(sto_res_tmp);
|
||||
vld1.u32 {d5[1]},[r3]
|
||||
add r3, r1, r6
|
||||
vst1.32 {d0[0]},[r1] ;vst1_lane_u32((uint32_t *)pu1_dst,
|
||||
; vreinterpret_u32_u8(sto_res), 0);
|
||||
vmlsl.u8 q4, d5, d29
|
||||
vst1.32 {d0[1]},[r3], r6 ;vst1_lane_u32((uint32_t
|
||||
; *)pu1_dst_tmp, vreinterpret_u32_u8(sto_res), 1);
|
||||
vhadd.s16 q4, q4, q15
|
||||
vqrshrun.s16 d8, q4, #6
|
||||
vst1.32 {d8[0]},[r3], r6
|
||||
add r1, r1, #4
|
||||
vst1.32 {d8[1]},[r3]
|
||||
bgt inner_loop_wd_4
|
||||
|
||||
end_inner_loop_wd_4
|
||||
subs r7, r7, #4
|
||||
add r1, r1, r9
|
||||
add r0, r0, r8
|
||||
bgt outer_loop_wd_4
|
||||
|
||||
vpop {d8 - d15}
|
||||
ldmfd sp!, {r4 - r12, r15} ;reload the registers from sp
|
||||
|
||||
ENDP
|
||||
|
||||
END
|
||||
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
|
||||
void vpx_convolve_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4, int x_step_q4,
|
||||
int y0_q4, int y_step_q4, int w, int h) {
|
||||
(void)filter;
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
if (w < 8) { // avg4
|
||||
uint8x8_t s0, s1;
|
||||
uint8x8_t dd0 = vdup_n_u8(0);
|
||||
uint32x2x2_t s01;
|
||||
do {
|
||||
s0 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
s1 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
s01 = vzip_u32(vreinterpret_u32_u8(s0), vreinterpret_u32_u8(s1));
|
||||
dd0 = vreinterpret_u8_u32(
|
||||
vld1_lane_u32((const uint32_t *)dst, vreinterpret_u32_u8(dd0), 0));
|
||||
dd0 = vreinterpret_u8_u32(vld1_lane_u32(
|
||||
(const uint32_t *)(dst + dst_stride), vreinterpret_u32_u8(dd0), 1));
|
||||
dd0 = vrhadd_u8(vreinterpret_u8_u32(s01.val[0]), dd0);
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(dd0), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(dd0), 1);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w == 8) { // avg8
|
||||
uint8x8_t s0, s1, d0, d1;
|
||||
uint8x16_t s01, d01;
|
||||
do {
|
||||
s0 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
s1 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
d0 = vld1_u8(dst);
|
||||
d1 = vld1_u8(dst + dst_stride);
|
||||
|
||||
s01 = vcombine_u8(s0, s1);
|
||||
d01 = vcombine_u8(d0, d1);
|
||||
d01 = vrhaddq_u8(s01, d01);
|
||||
|
||||
vst1_u8(dst, vget_low_u8(d01));
|
||||
dst += dst_stride;
|
||||
vst1_u8(dst, vget_high_u8(d01));
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w < 32) { // avg16
|
||||
uint8x16_t s0, s1, d0, d1;
|
||||
do {
|
||||
s0 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
s1 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
d0 = vld1q_u8(dst);
|
||||
d1 = vld1q_u8(dst + dst_stride);
|
||||
|
||||
d0 = vrhaddq_u8(s0, d0);
|
||||
d1 = vrhaddq_u8(s1, d1);
|
||||
|
||||
vst1q_u8(dst, d0);
|
||||
dst += dst_stride;
|
||||
vst1q_u8(dst, d1);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w == 32) { // avg32
|
||||
uint8x16_t s0, s1, s2, s3, d0, d1, d2, d3;
|
||||
do {
|
||||
s0 = vld1q_u8(src);
|
||||
s1 = vld1q_u8(src + 16);
|
||||
src += src_stride;
|
||||
s2 = vld1q_u8(src);
|
||||
s3 = vld1q_u8(src + 16);
|
||||
src += src_stride;
|
||||
d0 = vld1q_u8(dst);
|
||||
d1 = vld1q_u8(dst + 16);
|
||||
d2 = vld1q_u8(dst + dst_stride);
|
||||
d3 = vld1q_u8(dst + dst_stride + 16);
|
||||
|
||||
d0 = vrhaddq_u8(s0, d0);
|
||||
d1 = vrhaddq_u8(s1, d1);
|
||||
d2 = vrhaddq_u8(s2, d2);
|
||||
d3 = vrhaddq_u8(s3, d3);
|
||||
|
||||
vst1q_u8(dst, d0);
|
||||
vst1q_u8(dst + 16, d1);
|
||||
dst += dst_stride;
|
||||
vst1q_u8(dst, d2);
|
||||
vst1q_u8(dst + 16, d3);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else { // avg64
|
||||
uint8x16_t s0, s1, s2, s3, d0, d1, d2, d3;
|
||||
do {
|
||||
s0 = vld1q_u8(src);
|
||||
s1 = vld1q_u8(src + 16);
|
||||
s2 = vld1q_u8(src + 32);
|
||||
s3 = vld1q_u8(src + 48);
|
||||
src += src_stride;
|
||||
d0 = vld1q_u8(dst);
|
||||
d1 = vld1q_u8(dst + 16);
|
||||
d2 = vld1q_u8(dst + 32);
|
||||
d3 = vld1q_u8(dst + 48);
|
||||
|
||||
d0 = vrhaddq_u8(s0, d0);
|
||||
d1 = vrhaddq_u8(s1, d1);
|
||||
d2 = vrhaddq_u8(s2, d2);
|
||||
d3 = vrhaddq_u8(s3, d3);
|
||||
|
||||
vst1q_u8(dst, d0);
|
||||
vst1q_u8(dst + 16, d1);
|
||||
vst1q_u8(dst + 32, d2);
|
||||
vst1q_u8(dst + 48, d3);
|
||||
dst += dst_stride;
|
||||
} while (--h);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
;
|
||||
; Copyright (c) 2013 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
|
||||
EXPORT |vpx_convolve_avg_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve_avg_neon| PROC
|
||||
push {r4-r6, lr}
|
||||
ldrd r4, r5, [sp, #36]
|
||||
mov r6, r2
|
||||
|
||||
cmp r4, #32
|
||||
bgt avg64
|
||||
beq avg32
|
||||
cmp r4, #8
|
||||
bgt avg16
|
||||
beq avg8
|
||||
b avg4
|
||||
|
||||
avg64
|
||||
sub lr, r1, #32
|
||||
sub r4, r3, #32
|
||||
avg64_h
|
||||
pld [r0, r1, lsl #1]
|
||||
vld1.8 {q0-q1}, [r0]!
|
||||
vld1.8 {q2-q3}, [r0], lr
|
||||
pld [r2, r3]
|
||||
vld1.8 {q8-q9}, [r6@128]!
|
||||
vld1.8 {q10-q11}, [r6@128], r4
|
||||
vrhadd.u8 q0, q0, q8
|
||||
vrhadd.u8 q1, q1, q9
|
||||
vrhadd.u8 q2, q2, q10
|
||||
vrhadd.u8 q3, q3, q11
|
||||
vst1.8 {q0-q1}, [r2@128]!
|
||||
vst1.8 {q2-q3}, [r2@128], r4
|
||||
subs r5, r5, #1
|
||||
bgt avg64_h
|
||||
pop {r4-r6, pc}
|
||||
|
||||
avg32
|
||||
vld1.8 {q0-q1}, [r0], r1
|
||||
vld1.8 {q2-q3}, [r0], r1
|
||||
vld1.8 {q8-q9}, [r6@128], r3
|
||||
vld1.8 {q10-q11}, [r6@128], r3
|
||||
pld [r0]
|
||||
vrhadd.u8 q0, q0, q8
|
||||
pld [r0, r1]
|
||||
vrhadd.u8 q1, q1, q9
|
||||
pld [r6]
|
||||
vrhadd.u8 q2, q2, q10
|
||||
pld [r6, r3]
|
||||
vrhadd.u8 q3, q3, q11
|
||||
vst1.8 {q0-q1}, [r2@128], r3
|
||||
vst1.8 {q2-q3}, [r2@128], r3
|
||||
subs r5, r5, #2
|
||||
bgt avg32
|
||||
pop {r4-r6, pc}
|
||||
|
||||
avg16
|
||||
vld1.8 {q0}, [r0], r1
|
||||
vld1.8 {q1}, [r0], r1
|
||||
vld1.8 {q2}, [r6@128], r3
|
||||
vld1.8 {q3}, [r6@128], r3
|
||||
pld [r0]
|
||||
pld [r0, r1]
|
||||
vrhadd.u8 q0, q0, q2
|
||||
pld [r6]
|
||||
pld [r6, r3]
|
||||
vrhadd.u8 q1, q1, q3
|
||||
vst1.8 {q0}, [r2@128], r3
|
||||
vst1.8 {q1}, [r2@128], r3
|
||||
subs r5, r5, #2
|
||||
bgt avg16
|
||||
pop {r4-r6, pc}
|
||||
|
||||
avg8
|
||||
vld1.8 {d0}, [r0], r1
|
||||
vld1.8 {d1}, [r0], r1
|
||||
vld1.8 {d2}, [r6@64], r3
|
||||
vld1.8 {d3}, [r6@64], r3
|
||||
pld [r0]
|
||||
pld [r0, r1]
|
||||
vrhadd.u8 q0, q0, q1
|
||||
pld [r6]
|
||||
pld [r6, r3]
|
||||
vst1.8 {d0}, [r2@64], r3
|
||||
vst1.8 {d1}, [r2@64], r3
|
||||
subs r5, r5, #2
|
||||
bgt avg8
|
||||
pop {r4-r6, pc}
|
||||
|
||||
avg4
|
||||
vld1.32 {d0[0]}, [r0], r1
|
||||
vld1.32 {d0[1]}, [r0], r1
|
||||
vld1.32 {d2[0]}, [r6@32], r3
|
||||
vld1.32 {d2[1]}, [r6@32], r3
|
||||
vrhadd.u8 d0, d0, d2
|
||||
vst1.32 {d0[0]}, [r2@32], r3
|
||||
vst1.32 {d0[1]}, [r2@32], r3
|
||||
subs r5, r5, #2
|
||||
bgt avg4
|
||||
pop {r4-r6, pc}
|
||||
ENDP
|
||||
|
||||
END
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
|
||||
void vpx_convolve_copy_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
(void)filter;
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
if (w < 8) { // copy4
|
||||
do {
|
||||
*(uint32_t *)dst = *(const uint32_t *)src;
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
*(uint32_t *)dst = *(const uint32_t *)src;
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w == 8) { // copy8
|
||||
uint8x8_t s0, s1;
|
||||
do {
|
||||
s0 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
s1 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
vst1_u8(dst, s0);
|
||||
dst += dst_stride;
|
||||
vst1_u8(dst, s1);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w < 32) { // copy16
|
||||
uint8x16_t s0, s1;
|
||||
do {
|
||||
s0 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
s1 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
vst1q_u8(dst, s0);
|
||||
dst += dst_stride;
|
||||
vst1q_u8(dst, s1);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else if (w == 32) { // copy32
|
||||
uint8x16_t s0, s1, s2, s3;
|
||||
do {
|
||||
s0 = vld1q_u8(src);
|
||||
s1 = vld1q_u8(src + 16);
|
||||
src += src_stride;
|
||||
s2 = vld1q_u8(src);
|
||||
s3 = vld1q_u8(src + 16);
|
||||
src += src_stride;
|
||||
|
||||
vst1q_u8(dst, s0);
|
||||
vst1q_u8(dst + 16, s1);
|
||||
dst += dst_stride;
|
||||
vst1q_u8(dst, s2);
|
||||
vst1q_u8(dst + 16, s3);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} else { // copy64
|
||||
uint8x16_t s0, s1, s2, s3;
|
||||
do {
|
||||
s0 = vld1q_u8(src);
|
||||
s1 = vld1q_u8(src + 16);
|
||||
s2 = vld1q_u8(src + 32);
|
||||
s3 = vld1q_u8(src + 48);
|
||||
src += src_stride;
|
||||
|
||||
vst1q_u8(dst, s0);
|
||||
vst1q_u8(dst + 16, s1);
|
||||
vst1q_u8(dst + 32, s2);
|
||||
vst1q_u8(dst + 48, s3);
|
||||
dst += dst_stride;
|
||||
} while (--h);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
;
|
||||
; Copyright (c) 2013 The WebM project authors. All Rights Reserved.
|
||||
;
|
||||
; Use of this source code is governed by a BSD-style license
|
||||
; that can be found in the LICENSE file in the root of the source
|
||||
; tree. An additional intellectual property rights grant can be found
|
||||
; in the file PATENTS. All contributing project authors may
|
||||
; be found in the AUTHORS file in the root of the source tree.
|
||||
;
|
||||
|
||||
EXPORT |vpx_convolve_copy_neon|
|
||||
ARM
|
||||
REQUIRE8
|
||||
PRESERVE8
|
||||
|
||||
AREA ||.text||, CODE, READONLY, ALIGN=2
|
||||
|
||||
|vpx_convolve_copy_neon| PROC
|
||||
push {r4-r5, lr}
|
||||
ldrd r4, r5, [sp, #32]
|
||||
|
||||
cmp r4, #32
|
||||
bgt copy64
|
||||
beq copy32
|
||||
cmp r4, #8
|
||||
bgt copy16
|
||||
beq copy8
|
||||
b copy4
|
||||
|
||||
copy64
|
||||
sub lr, r1, #32
|
||||
sub r3, r3, #32
|
||||
copy64_h
|
||||
pld [r0, r1, lsl #1]
|
||||
vld1.8 {q0-q1}, [r0]!
|
||||
vld1.8 {q2-q3}, [r0], lr
|
||||
vst1.8 {q0-q1}, [r2@128]!
|
||||
vst1.8 {q2-q3}, [r2@128], r3
|
||||
subs r5, r5, #1
|
||||
bgt copy64_h
|
||||
pop {r4-r5, pc}
|
||||
|
||||
copy32
|
||||
pld [r0, r1, lsl #1]
|
||||
vld1.8 {q0-q1}, [r0], r1
|
||||
pld [r0, r1, lsl #1]
|
||||
vld1.8 {q2-q3}, [r0], r1
|
||||
vst1.8 {q0-q1}, [r2@128], r3
|
||||
vst1.8 {q2-q3}, [r2@128], r3
|
||||
subs r5, r5, #2
|
||||
bgt copy32
|
||||
pop {r4-r5, pc}
|
||||
|
||||
copy16
|
||||
pld [r0, r1, lsl #1]
|
||||
vld1.8 {q0}, [r0], r1
|
||||
pld [r0, r1, lsl #1]
|
||||
vld1.8 {q1}, [r0], r1
|
||||
vst1.8 {q0}, [r2@128], r3
|
||||
vst1.8 {q1}, [r2@128], r3
|
||||
subs r5, r5, #2
|
||||
bgt copy16
|
||||
pop {r4-r5, pc}
|
||||
|
||||
copy8
|
||||
pld [r0, r1, lsl #1]
|
||||
vld1.8 {d0}, [r0], r1
|
||||
pld [r0, r1, lsl #1]
|
||||
vld1.8 {d2}, [r0], r1
|
||||
vst1.8 {d0}, [r2@64], r3
|
||||
vst1.8 {d2}, [r2@64], r3
|
||||
subs r5, r5, #2
|
||||
bgt copy8
|
||||
pop {r4-r5, pc}
|
||||
|
||||
copy4
|
||||
ldr r12, [r0], r1
|
||||
str r12, [r2], r3
|
||||
subs r5, r5, #1
|
||||
bgt copy4
|
||||
pop {r4-r5, pc}
|
||||
ENDP
|
||||
|
||||
END
|
||||
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (c) 2013 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
|
||||
void vpx_convolve8_neon(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
|
||||
ptrdiff_t dst_stride, const InterpKernel *filter,
|
||||
int x0_q4, int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h) {
|
||||
/* Given our constraints: w <= 64, h <= 64, taps == 8 we can reduce the
|
||||
* maximum buffer size to 64 * 64 + 7 (+ 1 to make it divisible by 4).
|
||||
*/
|
||||
uint8_t temp[64 * 72];
|
||||
|
||||
// Account for the vertical phase needing 3 lines prior and 4 lines post
|
||||
// (+ 1 to make it divisible by 4).
|
||||
const int intermediate_height = h + 8;
|
||||
|
||||
assert(y_step_q4 == 16);
|
||||
assert(x_step_q4 == 16);
|
||||
|
||||
/* Filter starting 3 lines back. The neon implementation will ignore the given
|
||||
* height and filter a multiple of 4 lines. Since this goes in to the temp
|
||||
* buffer which has lots of extra room and is subsequently discarded this is
|
||||
* safe if somewhat less than ideal. */
|
||||
vpx_convolve8_horiz_neon(src - src_stride * 3, src_stride, temp, w, filter,
|
||||
x0_q4, x_step_q4, y0_q4, y_step_q4, w,
|
||||
intermediate_height);
|
||||
|
||||
/* Step into the temp buffer 3 lines to get the actual frame data */
|
||||
vpx_convolve8_vert_neon(temp + w * 3, w, dst, dst_stride, filter, x0_q4,
|
||||
x_step_q4, y0_q4, y_step_q4, w, h);
|
||||
}
|
||||
|
||||
void vpx_convolve8_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
uint8_t temp[64 * 72];
|
||||
const int intermediate_height = h + 8;
|
||||
|
||||
assert(y_step_q4 == 16);
|
||||
assert(x_step_q4 == 16);
|
||||
|
||||
/* This implementation has the same issues as above. In addition, we only want
|
||||
* to average the values after both passes.
|
||||
*/
|
||||
vpx_convolve8_horiz_neon(src - src_stride * 3, src_stride, temp, w, filter,
|
||||
x0_q4, x_step_q4, y0_q4, y_step_q4, w,
|
||||
intermediate_height);
|
||||
vpx_convolve8_avg_vert_neon(temp + w * 3, w, dst, dst_stride, filter, x0_q4,
|
||||
x_step_q4, y0_q4, y_step_q4, w, h);
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/arm/vpx_convolve8_neon.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
|
||||
static INLINE void scaledconvolve_horiz_w4(
|
||||
const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
|
||||
const ptrdiff_t dst_stride, const InterpKernel *const x_filters,
|
||||
const int x0_q4, const int x_step_q4, const int w, const int h) {
|
||||
DECLARE_ALIGNED(16, uint8_t, temp[4 * 4]);
|
||||
int x, y, z;
|
||||
|
||||
src -= SUBPEL_TAPS / 2 - 1;
|
||||
|
||||
y = h;
|
||||
do {
|
||||
int x_q4 = x0_q4;
|
||||
x = 0;
|
||||
do {
|
||||
// process 4 src_x steps
|
||||
for (z = 0; z < 4; ++z) {
|
||||
const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
|
||||
if (x_q4 & SUBPEL_MASK) {
|
||||
const int16x8_t filters = vld1q_s16(x_filters[x_q4 & SUBPEL_MASK]);
|
||||
const int16x4_t filter3 = vdup_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x4_t filter4 = vdup_lane_s16(vget_high_s16(filters), 0);
|
||||
uint8x8_t s[8], d;
|
||||
int16x8_t ss[4];
|
||||
int16x4_t t[8], tt;
|
||||
|
||||
load_u8_8x4(src_x, src_stride, &s[0], &s[1], &s[2], &s[3]);
|
||||
transpose_u8_8x4(&s[0], &s[1], &s[2], &s[3]);
|
||||
|
||||
ss[0] = vreinterpretq_s16_u16(vmovl_u8(s[0]));
|
||||
ss[1] = vreinterpretq_s16_u16(vmovl_u8(s[1]));
|
||||
ss[2] = vreinterpretq_s16_u16(vmovl_u8(s[2]));
|
||||
ss[3] = vreinterpretq_s16_u16(vmovl_u8(s[3]));
|
||||
t[0] = vget_low_s16(ss[0]);
|
||||
t[1] = vget_low_s16(ss[1]);
|
||||
t[2] = vget_low_s16(ss[2]);
|
||||
t[3] = vget_low_s16(ss[3]);
|
||||
t[4] = vget_high_s16(ss[0]);
|
||||
t[5] = vget_high_s16(ss[1]);
|
||||
t[6] = vget_high_s16(ss[2]);
|
||||
t[7] = vget_high_s16(ss[3]);
|
||||
|
||||
tt = convolve8_4(t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7],
|
||||
filters, filter3, filter4);
|
||||
d = vqrshrun_n_s16(vcombine_s16(tt, tt), 7);
|
||||
vst1_lane_u32((uint32_t *)&temp[4 * z], vreinterpret_u32_u8(d), 0);
|
||||
} else {
|
||||
int i;
|
||||
for (i = 0; i < 4; ++i) {
|
||||
temp[z * 4 + i] = src_x[i * src_stride + 3];
|
||||
}
|
||||
}
|
||||
x_q4 += x_step_q4;
|
||||
}
|
||||
|
||||
// transpose the 4x4 filters values back to dst
|
||||
{
|
||||
const uint8x8x4_t d4 = vld4_u8(temp);
|
||||
vst1_lane_u32((uint32_t *)&dst[x + 0 * dst_stride],
|
||||
vreinterpret_u32_u8(d4.val[0]), 0);
|
||||
vst1_lane_u32((uint32_t *)&dst[x + 1 * dst_stride],
|
||||
vreinterpret_u32_u8(d4.val[1]), 0);
|
||||
vst1_lane_u32((uint32_t *)&dst[x + 2 * dst_stride],
|
||||
vreinterpret_u32_u8(d4.val[2]), 0);
|
||||
vst1_lane_u32((uint32_t *)&dst[x + 3 * dst_stride],
|
||||
vreinterpret_u32_u8(d4.val[3]), 0);
|
||||
}
|
||||
x += 4;
|
||||
} while (x < w);
|
||||
|
||||
src += src_stride * 4;
|
||||
dst += dst_stride * 4;
|
||||
y -= 4;
|
||||
} while (y > 0);
|
||||
}
|
||||
|
||||
static INLINE void scaledconvolve_horiz_w8(
|
||||
const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
|
||||
const ptrdiff_t dst_stride, const InterpKernel *const x_filters,
|
||||
const int x0_q4, const int x_step_q4, const int w, const int h) {
|
||||
DECLARE_ALIGNED(16, uint8_t, temp[8 * 8]);
|
||||
int x, y, z;
|
||||
src -= SUBPEL_TAPS / 2 - 1;
|
||||
|
||||
// This function processes 8x8 areas. The intermediate height is not always
|
||||
// a multiple of 8, so force it to be a multiple of 8 here.
|
||||
y = (h + 7) & ~7;
|
||||
|
||||
do {
|
||||
int x_q4 = x0_q4;
|
||||
x = 0;
|
||||
do {
|
||||
uint8x8_t d[8];
|
||||
// process 8 src_x steps
|
||||
for (z = 0; z < 8; ++z) {
|
||||
const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
|
||||
|
||||
if (x_q4 & SUBPEL_MASK) {
|
||||
const int16x8_t filters = vld1q_s16(x_filters[x_q4 & SUBPEL_MASK]);
|
||||
uint8x8_t s[8];
|
||||
load_u8_8x8(src_x, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4],
|
||||
&s[5], &s[6], &s[7]);
|
||||
transpose_u8_8x8(&s[0], &s[1], &s[2], &s[3], &s[4], &s[5], &s[6],
|
||||
&s[7]);
|
||||
d[0] = scale_filter_8(s, filters);
|
||||
vst1_u8(&temp[8 * z], d[0]);
|
||||
} else {
|
||||
int i;
|
||||
for (i = 0; i < 8; ++i) {
|
||||
temp[z * 8 + i] = src_x[i * src_stride + 3];
|
||||
}
|
||||
}
|
||||
x_q4 += x_step_q4;
|
||||
}
|
||||
|
||||
// transpose the 8x8 filters values back to dst
|
||||
load_u8_8x8(temp, 8, &d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6],
|
||||
&d[7]);
|
||||
transpose_u8_8x8(&d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], &d[7]);
|
||||
vst1_u8(&dst[x + 0 * dst_stride], d[0]);
|
||||
vst1_u8(&dst[x + 1 * dst_stride], d[1]);
|
||||
vst1_u8(&dst[x + 2 * dst_stride], d[2]);
|
||||
vst1_u8(&dst[x + 3 * dst_stride], d[3]);
|
||||
vst1_u8(&dst[x + 4 * dst_stride], d[4]);
|
||||
vst1_u8(&dst[x + 5 * dst_stride], d[5]);
|
||||
vst1_u8(&dst[x + 6 * dst_stride], d[6]);
|
||||
vst1_u8(&dst[x + 7 * dst_stride], d[7]);
|
||||
x += 8;
|
||||
} while (x < w);
|
||||
|
||||
src += src_stride * 8;
|
||||
dst += dst_stride * 8;
|
||||
} while (y -= 8);
|
||||
}
|
||||
|
||||
static INLINE void scaledconvolve_vert_w4(
|
||||
const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
|
||||
const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
|
||||
const int y0_q4, const int y_step_q4, const int w, const int h) {
|
||||
int y;
|
||||
int y_q4 = y0_q4;
|
||||
|
||||
src -= src_stride * (SUBPEL_TAPS / 2 - 1);
|
||||
y = h;
|
||||
do {
|
||||
const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
|
||||
|
||||
if (y_q4 & SUBPEL_MASK) {
|
||||
const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
|
||||
const int16x4_t filter3 = vdup_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x4_t filter4 = vdup_lane_s16(vget_high_s16(filters), 0);
|
||||
uint8x8_t s[8], d;
|
||||
int16x4_t t[8], tt;
|
||||
|
||||
load_u8_8x8(src_y, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
|
||||
&s[6], &s[7]);
|
||||
t[0] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[0])));
|
||||
t[1] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[1])));
|
||||
t[2] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[2])));
|
||||
t[3] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[3])));
|
||||
t[4] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[4])));
|
||||
t[5] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[5])));
|
||||
t[6] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[6])));
|
||||
t[7] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[7])));
|
||||
|
||||
tt = convolve8_4(t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], filters,
|
||||
filter3, filter4);
|
||||
d = vqrshrun_n_s16(vcombine_s16(tt, tt), 7);
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d), 0);
|
||||
} else {
|
||||
memcpy(dst, &src_y[3 * src_stride], w);
|
||||
}
|
||||
|
||||
dst += dst_stride;
|
||||
y_q4 += y_step_q4;
|
||||
} while (--y);
|
||||
}
|
||||
|
||||
static INLINE void scaledconvolve_vert_w8(
|
||||
const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
|
||||
const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
|
||||
const int y0_q4, const int y_step_q4, const int w, const int h) {
|
||||
int y;
|
||||
int y_q4 = y0_q4;
|
||||
|
||||
src -= src_stride * (SUBPEL_TAPS / 2 - 1);
|
||||
y = h;
|
||||
do {
|
||||
const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
|
||||
if (y_q4 & SUBPEL_MASK) {
|
||||
const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
|
||||
uint8x8_t s[8], d;
|
||||
load_u8_8x8(src_y, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
|
||||
&s[6], &s[7]);
|
||||
d = scale_filter_8(s, filters);
|
||||
vst1_u8(dst, d);
|
||||
} else {
|
||||
memcpy(dst, &src_y[3 * src_stride], w);
|
||||
}
|
||||
dst += dst_stride;
|
||||
y_q4 += y_step_q4;
|
||||
} while (--y);
|
||||
}
|
||||
|
||||
static INLINE void scaledconvolve_vert_w16(
|
||||
const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
|
||||
const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
|
||||
const int y0_q4, const int y_step_q4, const int w, const int h) {
|
||||
int x, y;
|
||||
int y_q4 = y0_q4;
|
||||
|
||||
src -= src_stride * (SUBPEL_TAPS / 2 - 1);
|
||||
y = h;
|
||||
do {
|
||||
const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
|
||||
if (y_q4 & SUBPEL_MASK) {
|
||||
x = 0;
|
||||
do {
|
||||
const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
|
||||
uint8x16_t ss[8];
|
||||
uint8x8_t s[8], d[2];
|
||||
load_u8_16x8(src_y, src_stride, &ss[0], &ss[1], &ss[2], &ss[3], &ss[4],
|
||||
&ss[5], &ss[6], &ss[7]);
|
||||
s[0] = vget_low_u8(ss[0]);
|
||||
s[1] = vget_low_u8(ss[1]);
|
||||
s[2] = vget_low_u8(ss[2]);
|
||||
s[3] = vget_low_u8(ss[3]);
|
||||
s[4] = vget_low_u8(ss[4]);
|
||||
s[5] = vget_low_u8(ss[5]);
|
||||
s[6] = vget_low_u8(ss[6]);
|
||||
s[7] = vget_low_u8(ss[7]);
|
||||
d[0] = scale_filter_8(s, filters);
|
||||
|
||||
s[0] = vget_high_u8(ss[0]);
|
||||
s[1] = vget_high_u8(ss[1]);
|
||||
s[2] = vget_high_u8(ss[2]);
|
||||
s[3] = vget_high_u8(ss[3]);
|
||||
s[4] = vget_high_u8(ss[4]);
|
||||
s[5] = vget_high_u8(ss[5]);
|
||||
s[6] = vget_high_u8(ss[6]);
|
||||
s[7] = vget_high_u8(ss[7]);
|
||||
d[1] = scale_filter_8(s, filters);
|
||||
vst1q_u8(&dst[x], vcombine_u8(d[0], d[1]));
|
||||
src_y += 16;
|
||||
x += 16;
|
||||
} while (x < w);
|
||||
} else {
|
||||
memcpy(dst, &src_y[3 * src_stride], w);
|
||||
}
|
||||
dst += dst_stride;
|
||||
y_q4 += y_step_q4;
|
||||
} while (--y);
|
||||
}
|
||||
|
||||
void vpx_scaled_2d_neon(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
|
||||
ptrdiff_t dst_stride, const InterpKernel *filter,
|
||||
int x0_q4, int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h) {
|
||||
// Note: Fixed size intermediate buffer, temp, places limits on parameters.
|
||||
// 2d filtering proceeds in 2 steps:
|
||||
// (1) Interpolate horizontally into an intermediate buffer, temp.
|
||||
// (2) Interpolate temp vertically to derive the sub-pixel result.
|
||||
// Deriving the maximum number of rows in the temp buffer (135):
|
||||
// --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
|
||||
// --Largest block size is 64x64 pixels.
|
||||
// --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
|
||||
// original frame (in 1/16th pixel units).
|
||||
// --Must round-up because block may be located at sub-pixel position.
|
||||
// --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
|
||||
// --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
|
||||
// --Require an additional 8 rows for the horiz_w8 transpose tail.
|
||||
// When calling in frame scaling function, the smallest scaling factor is x1/4
|
||||
// ==> y_step_q4 = 64. Since w and h are at most 16, the temp buffer is still
|
||||
// big enough.
|
||||
DECLARE_ALIGNED(16, uint8_t, temp[(135 + 8) * 64]);
|
||||
const int intermediate_height =
|
||||
(((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
|
||||
|
||||
assert(w <= 64);
|
||||
assert(h <= 64);
|
||||
assert(y_step_q4 <= 32 || (y_step_q4 <= 64 && h <= 32));
|
||||
assert(x_step_q4 <= 64);
|
||||
|
||||
if (w >= 8) {
|
||||
scaledconvolve_horiz_w8(src - src_stride * (SUBPEL_TAPS / 2 - 1),
|
||||
src_stride, temp, 64, filter, x0_q4, x_step_q4, w,
|
||||
intermediate_height);
|
||||
} else {
|
||||
scaledconvolve_horiz_w4(src - src_stride * (SUBPEL_TAPS / 2 - 1),
|
||||
src_stride, temp, 64, filter, x0_q4, x_step_q4, w,
|
||||
intermediate_height);
|
||||
}
|
||||
|
||||
if (w >= 16) {
|
||||
scaledconvolve_vert_w16(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst,
|
||||
dst_stride, filter, y0_q4, y_step_q4, w, h);
|
||||
} else if (w == 8) {
|
||||
scaledconvolve_vert_w8(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst,
|
||||
dst_stride, filter, y0_q4, y_step_q4, w, h);
|
||||
} else {
|
||||
scaledconvolve_vert_w4(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst,
|
||||
dst_stride, filter, y0_q4, y_step_q4, w, h);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user