(61d00a474) v0.9.7.1
This commit is contained in:
Regalis
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_PPC_BITDEPTH_CONVERSION_VSX_H_
|
||||
#define VPX_VPX_DSP_PPC_BITDEPTH_CONVERSION_VSX_H_
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
// Load 8 16 bit values. If the source is 32 bits then pack down with
|
||||
// saturation.
|
||||
static INLINE int16x8_t load_tran_low(int32_t c, const tran_low_t *s) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
int32x4_t u = vec_vsx_ld(c, s);
|
||||
int32x4_t v = vec_vsx_ld(c, s + 4);
|
||||
return vec_packs(u, v);
|
||||
#else
|
||||
return vec_vsx_ld(c, s);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store 8 16 bit values. If the destination is 32 bits then sign extend the
|
||||
// values by multiplying by 1.
|
||||
static INLINE void store_tran_low(int16x8_t v, int32_t c, tran_low_t *s) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
const int16x8_t one = vec_splat_s16(1);
|
||||
const int32x4_t even = vec_mule(v, one);
|
||||
const int32x4_t odd = vec_mulo(v, one);
|
||||
const int32x4_t high = vec_mergeh(even, odd);
|
||||
const int32x4_t low = vec_mergel(even, odd);
|
||||
vec_vsx_st(high, c, s);
|
||||
vec_vsx_st(low, c, s + 4);
|
||||
#else
|
||||
vec_vsx_st(v, c, s);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif // VPX_VPX_DSP_PPC_BITDEPTH_CONVERSION_VSX_H_
|
||||
@@ -0,0 +1,374 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
extern const int16_t vpx_rv[];
|
||||
|
||||
static const uint8x16_t load_merge = { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A,
|
||||
0x0C, 0x0E, 0x18, 0x19, 0x1A, 0x1B,
|
||||
0x1C, 0x1D, 0x1E, 0x1F };
|
||||
|
||||
static const uint8x16_t st8_perm = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
|
||||
0x06, 0x07, 0x18, 0x19, 0x1A, 0x1B,
|
||||
0x1C, 0x1D, 0x1E, 0x1F };
|
||||
|
||||
static INLINE uint8x16_t apply_filter(uint8x16_t ctx[4], uint8x16_t v,
|
||||
uint8x16_t filter) {
|
||||
const uint8x16_t k1 = vec_avg(ctx[0], ctx[1]);
|
||||
const uint8x16_t k2 = vec_avg(ctx[3], ctx[2]);
|
||||
const uint8x16_t k3 = vec_avg(k1, k2);
|
||||
const uint8x16_t f_a = vec_max(vec_absd(v, ctx[0]), vec_absd(v, ctx[1]));
|
||||
const uint8x16_t f_b = vec_max(vec_absd(v, ctx[2]), vec_absd(v, ctx[3]));
|
||||
const bool8x16_t mask = vec_cmplt(vec_max(f_a, f_b), filter);
|
||||
return vec_sel(v, vec_avg(k3, v), mask);
|
||||
}
|
||||
|
||||
static INLINE void vert_ctx(uint8x16_t ctx[4], int col, uint8_t *src,
|
||||
int stride) {
|
||||
ctx[0] = vec_vsx_ld(col - 2 * stride, src);
|
||||
ctx[1] = vec_vsx_ld(col - stride, src);
|
||||
ctx[2] = vec_vsx_ld(col + stride, src);
|
||||
ctx[3] = vec_vsx_ld(col + 2 * stride, src);
|
||||
}
|
||||
|
||||
static INLINE void horz_ctx(uint8x16_t ctx[4], uint8x16_t left_ctx,
|
||||
uint8x16_t v, uint8x16_t right_ctx) {
|
||||
static const uint8x16_t l2_perm = { 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13,
|
||||
0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
|
||||
0x1A, 0x1B, 0x1C, 0x1D };
|
||||
|
||||
static const uint8x16_t l1_perm = { 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14,
|
||||
0x15, 0x16, 0x17, 0x18, 0x19, 0x1A,
|
||||
0x1B, 0x1C, 0x1D, 0x1E };
|
||||
|
||||
static const uint8x16_t r1_perm = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
||||
0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
|
||||
0x0D, 0x0E, 0x0F, 0x10 };
|
||||
|
||||
static const uint8x16_t r2_perm = { 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
||||
0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
|
||||
0x0E, 0x0F, 0x10, 0x11 };
|
||||
ctx[0] = vec_perm(left_ctx, v, l2_perm);
|
||||
ctx[1] = vec_perm(left_ctx, v, l1_perm);
|
||||
ctx[2] = vec_perm(v, right_ctx, r1_perm);
|
||||
ctx[3] = vec_perm(v, right_ctx, r2_perm);
|
||||
}
|
||||
void vpx_post_proc_down_and_across_mb_row_vsx(unsigned char *src_ptr,
|
||||
unsigned char *dst_ptr,
|
||||
int src_pixels_per_line,
|
||||
int dst_pixels_per_line, int cols,
|
||||
unsigned char *f, int size) {
|
||||
int row, col;
|
||||
uint8x16_t ctx[4], out, v, left_ctx;
|
||||
|
||||
for (row = 0; row < size; row++) {
|
||||
for (col = 0; col < cols - 8; col += 16) {
|
||||
const uint8x16_t filter = vec_vsx_ld(col, f);
|
||||
v = vec_vsx_ld(col, src_ptr);
|
||||
vert_ctx(ctx, col, src_ptr, src_pixels_per_line);
|
||||
vec_vsx_st(apply_filter(ctx, v, filter), col, dst_ptr);
|
||||
}
|
||||
|
||||
if (col != cols) {
|
||||
const uint8x16_t filter = vec_vsx_ld(col, f);
|
||||
v = vec_vsx_ld(col, src_ptr);
|
||||
vert_ctx(ctx, col, src_ptr, src_pixels_per_line);
|
||||
out = apply_filter(ctx, v, filter);
|
||||
vec_vsx_st(vec_perm(out, v, st8_perm), col, dst_ptr);
|
||||
}
|
||||
|
||||
/* now post_proc_across */
|
||||
left_ctx = vec_splats(dst_ptr[0]);
|
||||
v = vec_vsx_ld(0, dst_ptr);
|
||||
for (col = 0; col < cols - 8; col += 16) {
|
||||
const uint8x16_t filter = vec_vsx_ld(col, f);
|
||||
const uint8x16_t right_ctx = (col + 16 == cols)
|
||||
? vec_splats(dst_ptr[cols - 1])
|
||||
: vec_vsx_ld(col, dst_ptr + 16);
|
||||
horz_ctx(ctx, left_ctx, v, right_ctx);
|
||||
vec_vsx_st(apply_filter(ctx, v, filter), col, dst_ptr);
|
||||
left_ctx = v;
|
||||
v = right_ctx;
|
||||
}
|
||||
|
||||
if (col != cols) {
|
||||
const uint8x16_t filter = vec_vsx_ld(col, f);
|
||||
const uint8x16_t right_ctx = vec_splats(dst_ptr[cols - 1]);
|
||||
horz_ctx(ctx, left_ctx, v, right_ctx);
|
||||
out = apply_filter(ctx, v, filter);
|
||||
vec_vsx_st(vec_perm(out, v, st8_perm), col, dst_ptr);
|
||||
}
|
||||
|
||||
src_ptr += src_pixels_per_line;
|
||||
dst_ptr += dst_pixels_per_line;
|
||||
}
|
||||
}
|
||||
|
||||
// C: s[c + 7]
|
||||
static INLINE int16x8_t next7l_s16(uint8x16_t c) {
|
||||
static const uint8x16_t next7_perm = {
|
||||
0x07, 0x10, 0x08, 0x11, 0x09, 0x12, 0x0A, 0x13,
|
||||
0x0B, 0x14, 0x0C, 0x15, 0x0D, 0x16, 0x0E, 0x17,
|
||||
};
|
||||
return (int16x8_t)vec_perm(c, vec_zeros_u8, next7_perm);
|
||||
}
|
||||
|
||||
// Slide across window and add.
|
||||
static INLINE int16x8_t slide_sum_s16(int16x8_t x) {
|
||||
// x = A B C D E F G H
|
||||
//
|
||||
// 0 A B C D E F G
|
||||
const int16x8_t sum1 = vec_add(x, vec_slo(x, vec_splats((int8_t)(2 << 3))));
|
||||
// 0 0 A B C D E F
|
||||
const int16x8_t sum2 = vec_add(vec_slo(x, vec_splats((int8_t)(4 << 3))),
|
||||
// 0 0 0 A B C D E
|
||||
vec_slo(x, vec_splats((int8_t)(6 << 3))));
|
||||
// 0 0 0 0 A B C D
|
||||
const int16x8_t sum3 = vec_add(vec_slo(x, vec_splats((int8_t)(8 << 3))),
|
||||
// 0 0 0 0 0 A B C
|
||||
vec_slo(x, vec_splats((int8_t)(10 << 3))));
|
||||
// 0 0 0 0 0 0 A B
|
||||
const int16x8_t sum4 = vec_add(vec_slo(x, vec_splats((int8_t)(12 << 3))),
|
||||
// 0 0 0 0 0 0 0 A
|
||||
vec_slo(x, vec_splats((int8_t)(14 << 3))));
|
||||
return vec_add(vec_add(sum1, sum2), vec_add(sum3, sum4));
|
||||
}
|
||||
|
||||
// Slide across window and add.
|
||||
static INLINE int32x4_t slide_sumsq_s32(int32x4_t xsq_even, int32x4_t xsq_odd) {
|
||||
// 0 A C E
|
||||
// + 0 B D F
|
||||
int32x4_t sumsq_1 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(4 << 3))),
|
||||
vec_slo(xsq_odd, vec_splats((int8_t)(4 << 3))));
|
||||
// 0 0 A C
|
||||
// + 0 0 B D
|
||||
int32x4_t sumsq_2 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(8 << 3))),
|
||||
vec_slo(xsq_odd, vec_splats((int8_t)(8 << 3))));
|
||||
// 0 0 0 A
|
||||
// + 0 0 0 B
|
||||
int32x4_t sumsq_3 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(12 << 3))),
|
||||
vec_slo(xsq_odd, vec_splats((int8_t)(12 << 3))));
|
||||
sumsq_1 = vec_add(sumsq_1, xsq_even);
|
||||
sumsq_2 = vec_add(sumsq_2, sumsq_3);
|
||||
return vec_add(sumsq_1, sumsq_2);
|
||||
}
|
||||
|
||||
// C: (b + sum + val) >> 4
|
||||
static INLINE int16x8_t filter_s16(int16x8_t b, int16x8_t sum, int16x8_t val) {
|
||||
return vec_sra(vec_add(vec_add(b, sum), val), vec_splats((uint16_t)4));
|
||||
}
|
||||
|
||||
// C: sumsq * 15 - sum * sum
|
||||
static INLINE bool16x8_t mask_s16(int32x4_t sumsq_even, int32x4_t sumsq_odd,
|
||||
int16x8_t sum, int32x4_t lim) {
|
||||
static const uint8x16_t mask_merge = { 0x00, 0x01, 0x10, 0x11, 0x04, 0x05,
|
||||
0x14, 0x15, 0x08, 0x09, 0x18, 0x19,
|
||||
0x0C, 0x0D, 0x1C, 0x1D };
|
||||
const int32x4_t sumsq_odd_scaled =
|
||||
vec_mul(sumsq_odd, vec_splats((int32_t)15));
|
||||
const int32x4_t sumsq_even_scaled =
|
||||
vec_mul(sumsq_even, vec_splats((int32_t)15));
|
||||
const int32x4_t thres_odd = vec_sub(sumsq_odd_scaled, vec_mulo(sum, sum));
|
||||
const int32x4_t thres_even = vec_sub(sumsq_even_scaled, vec_mule(sum, sum));
|
||||
|
||||
const bool32x4_t mask_odd = vec_cmplt(thres_odd, lim);
|
||||
const bool32x4_t mask_even = vec_cmplt(thres_even, lim);
|
||||
return vec_perm((bool16x8_t)mask_even, (bool16x8_t)mask_odd, mask_merge);
|
||||
}
|
||||
|
||||
void vpx_mbpost_proc_across_ip_vsx(unsigned char *src, int pitch, int rows,
|
||||
int cols, int flimit) {
|
||||
int row, col;
|
||||
const int32x4_t lim = vec_splats(flimit);
|
||||
|
||||
// 8 columns are processed at a time.
|
||||
assert(cols % 8 == 0);
|
||||
|
||||
for (row = 0; row < rows; row++) {
|
||||
// The sum is signed and requires at most 13 bits.
|
||||
// (8 bits + sign) * 15 (4 bits)
|
||||
int16x8_t sum;
|
||||
// The sum of squares requires at most 20 bits.
|
||||
// (16 bits + sign) * 15 (4 bits)
|
||||
int32x4_t sumsq_even, sumsq_odd;
|
||||
|
||||
// Fill left context with first col.
|
||||
int16x8_t left_ctx = vec_splats((int16_t)src[0]);
|
||||
int16_t s = src[0] * 9;
|
||||
int32_t ssq = src[0] * src[0] * 9 + 16;
|
||||
|
||||
// Fill the next 6 columns of the sliding window with cols 2 to 7.
|
||||
for (col = 1; col <= 6; ++col) {
|
||||
s += src[col];
|
||||
ssq += src[col] * src[col];
|
||||
}
|
||||
// Set this sum to every element in the window.
|
||||
sum = vec_splats(s);
|
||||
sumsq_even = vec_splats(ssq);
|
||||
sumsq_odd = vec_splats(ssq);
|
||||
|
||||
for (col = 0; col < cols; col += 8) {
|
||||
bool16x8_t mask;
|
||||
int16x8_t filtered, masked;
|
||||
uint8x16_t out;
|
||||
|
||||
const uint8x16_t val = vec_vsx_ld(0, src + col);
|
||||
const int16x8_t val_high = unpack_to_s16_h(val);
|
||||
|
||||
// C: s[c + 7]
|
||||
const int16x8_t right_ctx = (col + 8 == cols)
|
||||
? vec_splats((int16_t)src[col + 7])
|
||||
: next7l_s16(val);
|
||||
|
||||
// C: x = s[c + 7] - s[c - 8];
|
||||
const int16x8_t x = vec_sub(right_ctx, left_ctx);
|
||||
const int32x4_t xsq_even =
|
||||
vec_sub(vec_mule(right_ctx, right_ctx), vec_mule(left_ctx, left_ctx));
|
||||
const int32x4_t xsq_odd =
|
||||
vec_sub(vec_mulo(right_ctx, right_ctx), vec_mulo(left_ctx, left_ctx));
|
||||
|
||||
const int32x4_t sumsq_tmp = slide_sumsq_s32(xsq_even, xsq_odd);
|
||||
// A C E G
|
||||
// 0 B D F
|
||||
// 0 A C E
|
||||
// 0 0 B D
|
||||
// 0 0 A C
|
||||
// 0 0 0 B
|
||||
// 0 0 0 A
|
||||
sumsq_even = vec_add(sumsq_even, sumsq_tmp);
|
||||
// B D F G
|
||||
// A C E G
|
||||
// 0 B D F
|
||||
// 0 A C E
|
||||
// 0 0 B D
|
||||
// 0 0 A C
|
||||
// 0 0 0 B
|
||||
// 0 0 0 A
|
||||
sumsq_odd = vec_add(sumsq_odd, vec_add(sumsq_tmp, xsq_odd));
|
||||
|
||||
sum = vec_add(sum, slide_sum_s16(x));
|
||||
|
||||
// C: (8 + sum + s[c]) >> 4
|
||||
filtered = filter_s16(vec_splats((int16_t)8), sum, val_high);
|
||||
// C: sumsq * 15 - sum * sum
|
||||
mask = mask_s16(sumsq_even, sumsq_odd, sum, lim);
|
||||
masked = vec_sel(val_high, filtered, mask);
|
||||
|
||||
out = vec_perm((uint8x16_t)masked, vec_vsx_ld(0, src + col), load_merge);
|
||||
vec_vsx_st(out, 0, src + col);
|
||||
|
||||
// Update window sum and square sum
|
||||
sum = vec_splat(sum, 7);
|
||||
sumsq_even = vec_splat(sumsq_odd, 3);
|
||||
sumsq_odd = vec_splat(sumsq_odd, 3);
|
||||
|
||||
// C: s[c - 8] (for next iteration)
|
||||
left_ctx = val_high;
|
||||
}
|
||||
src += pitch;
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_mbpost_proc_down_vsx(uint8_t *dst, int pitch, int rows, int cols,
|
||||
int flimit) {
|
||||
int col, row, i;
|
||||
int16x8_t window[16];
|
||||
const int32x4_t lim = vec_splats(flimit);
|
||||
|
||||
// 8 columns are processed at a time.
|
||||
assert(cols % 8 == 0);
|
||||
// If rows is less than 8 the bottom border extension fails.
|
||||
assert(rows >= 8);
|
||||
|
||||
for (col = 0; col < cols; col += 8) {
|
||||
// The sum is signed and requires at most 13 bits.
|
||||
// (8 bits + sign) * 15 (4 bits)
|
||||
int16x8_t r1, sum;
|
||||
// The sum of squares requires at most 20 bits.
|
||||
// (16 bits + sign) * 15 (4 bits)
|
||||
int32x4_t sumsq_even, sumsq_odd;
|
||||
|
||||
r1 = unpack_to_s16_h(vec_vsx_ld(0, dst));
|
||||
// Fill sliding window with first row.
|
||||
for (i = 0; i <= 8; i++) {
|
||||
window[i] = r1;
|
||||
}
|
||||
// First 9 rows of the sliding window are the same.
|
||||
// sum = r1 * 9
|
||||
sum = vec_mladd(r1, vec_splats((int16_t)9), vec_zeros_s16);
|
||||
|
||||
// sumsq = r1 * r1 * 9
|
||||
sumsq_even = vec_mule(sum, r1);
|
||||
sumsq_odd = vec_mulo(sum, r1);
|
||||
|
||||
// Fill the next 6 rows of the sliding window with rows 2 to 7.
|
||||
for (i = 1; i <= 6; ++i) {
|
||||
const int16x8_t next_row = unpack_to_s16_h(vec_vsx_ld(i * pitch, dst));
|
||||
window[i + 8] = next_row;
|
||||
sum = vec_add(sum, next_row);
|
||||
sumsq_odd = vec_add(sumsq_odd, vec_mulo(next_row, next_row));
|
||||
sumsq_even = vec_add(sumsq_even, vec_mule(next_row, next_row));
|
||||
}
|
||||
|
||||
for (row = 0; row < rows; row++) {
|
||||
int32x4_t d15_even, d15_odd, d0_even, d0_odd;
|
||||
bool16x8_t mask;
|
||||
int16x8_t filtered, masked;
|
||||
uint8x16_t out;
|
||||
|
||||
const int16x8_t rv = vec_vsx_ld(0, vpx_rv + (row & 127));
|
||||
|
||||
// Move the sliding window
|
||||
if (row + 7 < rows) {
|
||||
window[15] = unpack_to_s16_h(vec_vsx_ld((row + 7) * pitch, dst));
|
||||
} else {
|
||||
window[15] = window[14];
|
||||
}
|
||||
|
||||
// C: sum += s[7 * pitch] - s[-8 * pitch];
|
||||
sum = vec_add(sum, vec_sub(window[15], window[0]));
|
||||
|
||||
// C: sumsq += s[7 * pitch] * s[7 * pitch] - s[-8 * pitch] * s[-8 *
|
||||
// pitch];
|
||||
// Optimization Note: Caching a squared-window for odd and even is
|
||||
// slower than just repeating the multiplies.
|
||||
d15_odd = vec_mulo(window[15], window[15]);
|
||||
d15_even = vec_mule(window[15], window[15]);
|
||||
d0_odd = vec_mulo(window[0], window[0]);
|
||||
d0_even = vec_mule(window[0], window[0]);
|
||||
sumsq_odd = vec_add(sumsq_odd, vec_sub(d15_odd, d0_odd));
|
||||
sumsq_even = vec_add(sumsq_even, vec_sub(d15_even, d0_even));
|
||||
|
||||
// C: (vpx_rv[(r & 127) + (c & 7)] + sum + s[0]) >> 4
|
||||
filtered = filter_s16(rv, sum, window[8]);
|
||||
|
||||
// C: sumsq * 15 - sum * sum
|
||||
mask = mask_s16(sumsq_even, sumsq_odd, sum, lim);
|
||||
masked = vec_sel(window[8], filtered, mask);
|
||||
|
||||
// TODO(ltrudeau) If cols % 16 == 0, we could just process 16 per
|
||||
// iteration
|
||||
out = vec_perm((uint8x16_t)masked, vec_vsx_ld(0, dst + row * pitch),
|
||||
load_merge);
|
||||
vec_vsx_st(out, 0, dst + row * pitch);
|
||||
|
||||
// Optimization Note: Turns out that the following loop is faster than
|
||||
// using pointers to manage the sliding window.
|
||||
for (i = 1; i < 16; i++) {
|
||||
window[i - 1] = window[i];
|
||||
}
|
||||
}
|
||||
dst += 8;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,553 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
|
||||
#include "vpx_dsp/ppc/transpose_vsx.h"
|
||||
#include "vpx_dsp/ppc/txfm_common_vsx.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
// Returns ((a +/- b) * cospi16 + (2 << 13)) >> 14.
|
||||
static INLINE void single_butterfly(int16x8_t a, int16x8_t b, int16x8_t *add,
|
||||
int16x8_t *sub) {
|
||||
// Since a + b can overflow 16 bits, the multiplication is distributed
|
||||
// (a * c +/- b * c).
|
||||
const int32x4_t ac_e = vec_mule(a, cospi16_v);
|
||||
const int32x4_t ac_o = vec_mulo(a, cospi16_v);
|
||||
const int32x4_t bc_e = vec_mule(b, cospi16_v);
|
||||
const int32x4_t bc_o = vec_mulo(b, cospi16_v);
|
||||
|
||||
// Reuse the same multiplies for sum and difference.
|
||||
const int32x4_t sum_e = vec_add(ac_e, bc_e);
|
||||
const int32x4_t sum_o = vec_add(ac_o, bc_o);
|
||||
const int32x4_t diff_e = vec_sub(ac_e, bc_e);
|
||||
const int32x4_t diff_o = vec_sub(ac_o, bc_o);
|
||||
|
||||
// Add rounding offset
|
||||
const int32x4_t rsum_o = vec_add(sum_o, vec_dct_const_rounding);
|
||||
const int32x4_t rsum_e = vec_add(sum_e, vec_dct_const_rounding);
|
||||
const int32x4_t rdiff_o = vec_add(diff_o, vec_dct_const_rounding);
|
||||
const int32x4_t rdiff_e = vec_add(diff_e, vec_dct_const_rounding);
|
||||
|
||||
const int32x4_t ssum_o = vec_sra(rsum_o, vec_dct_const_bits);
|
||||
const int32x4_t ssum_e = vec_sra(rsum_e, vec_dct_const_bits);
|
||||
const int32x4_t sdiff_o = vec_sra(rdiff_o, vec_dct_const_bits);
|
||||
const int32x4_t sdiff_e = vec_sra(rdiff_e, vec_dct_const_bits);
|
||||
|
||||
// There's no pack operation for even and odd, so we need to permute.
|
||||
*add = (int16x8_t)vec_perm(ssum_e, ssum_o, vec_perm_odd_even_pack);
|
||||
*sub = (int16x8_t)vec_perm(sdiff_e, sdiff_o, vec_perm_odd_even_pack);
|
||||
}
|
||||
|
||||
// Returns (a * c1 +/- b * c2 + (2 << 13)) >> 14
|
||||
static INLINE void double_butterfly(int16x8_t a, int16x8_t c1, int16x8_t b,
|
||||
int16x8_t c2, int16x8_t *add,
|
||||
int16x8_t *sub) {
|
||||
const int32x4_t ac1_o = vec_mulo(a, c1);
|
||||
const int32x4_t ac1_e = vec_mule(a, c1);
|
||||
const int32x4_t ac2_o = vec_mulo(a, c2);
|
||||
const int32x4_t ac2_e = vec_mule(a, c2);
|
||||
|
||||
const int32x4_t bc1_o = vec_mulo(b, c1);
|
||||
const int32x4_t bc1_e = vec_mule(b, c1);
|
||||
const int32x4_t bc2_o = vec_mulo(b, c2);
|
||||
const int32x4_t bc2_e = vec_mule(b, c2);
|
||||
|
||||
const int32x4_t sum_o = vec_add(ac1_o, bc2_o);
|
||||
const int32x4_t sum_e = vec_add(ac1_e, bc2_e);
|
||||
const int32x4_t diff_o = vec_sub(ac2_o, bc1_o);
|
||||
const int32x4_t diff_e = vec_sub(ac2_e, bc1_e);
|
||||
|
||||
// Add rounding offset
|
||||
const int32x4_t rsum_o = vec_add(sum_o, vec_dct_const_rounding);
|
||||
const int32x4_t rsum_e = vec_add(sum_e, vec_dct_const_rounding);
|
||||
const int32x4_t rdiff_o = vec_add(diff_o, vec_dct_const_rounding);
|
||||
const int32x4_t rdiff_e = vec_add(diff_e, vec_dct_const_rounding);
|
||||
|
||||
const int32x4_t ssum_o = vec_sra(rsum_o, vec_dct_const_bits);
|
||||
const int32x4_t ssum_e = vec_sra(rsum_e, vec_dct_const_bits);
|
||||
const int32x4_t sdiff_o = vec_sra(rdiff_o, vec_dct_const_bits);
|
||||
const int32x4_t sdiff_e = vec_sra(rdiff_e, vec_dct_const_bits);
|
||||
|
||||
// There's no pack operation for even and odd, so we need to permute.
|
||||
*add = (int16x8_t)vec_perm(ssum_e, ssum_o, vec_perm_odd_even_pack);
|
||||
*sub = (int16x8_t)vec_perm(sdiff_e, sdiff_o, vec_perm_odd_even_pack);
|
||||
}
|
||||
|
||||
// While other architecture combine the load and the stage 1 operations, Power9
|
||||
// benchmarking show no benefit in such an approach.
|
||||
static INLINE void load(const int16_t *a, int stride, int16x8_t *b) {
|
||||
// Tried out different combinations of load and shift instructions, this is
|
||||
// the fastest one.
|
||||
{
|
||||
const int16x8_t l0 = vec_vsx_ld(0, a);
|
||||
const int16x8_t l1 = vec_vsx_ld(0, a + stride);
|
||||
const int16x8_t l2 = vec_vsx_ld(0, a + 2 * stride);
|
||||
const int16x8_t l3 = vec_vsx_ld(0, a + 3 * stride);
|
||||
const int16x8_t l4 = vec_vsx_ld(0, a + 4 * stride);
|
||||
const int16x8_t l5 = vec_vsx_ld(0, a + 5 * stride);
|
||||
const int16x8_t l6 = vec_vsx_ld(0, a + 6 * stride);
|
||||
const int16x8_t l7 = vec_vsx_ld(0, a + 7 * stride);
|
||||
|
||||
const int16x8_t l8 = vec_vsx_ld(0, a + 8 * stride);
|
||||
const int16x8_t l9 = vec_vsx_ld(0, a + 9 * stride);
|
||||
const int16x8_t l10 = vec_vsx_ld(0, a + 10 * stride);
|
||||
const int16x8_t l11 = vec_vsx_ld(0, a + 11 * stride);
|
||||
const int16x8_t l12 = vec_vsx_ld(0, a + 12 * stride);
|
||||
const int16x8_t l13 = vec_vsx_ld(0, a + 13 * stride);
|
||||
const int16x8_t l14 = vec_vsx_ld(0, a + 14 * stride);
|
||||
const int16x8_t l15 = vec_vsx_ld(0, a + 15 * stride);
|
||||
|
||||
b[0] = vec_sl(l0, vec_dct_scale_log2);
|
||||
b[1] = vec_sl(l1, vec_dct_scale_log2);
|
||||
b[2] = vec_sl(l2, vec_dct_scale_log2);
|
||||
b[3] = vec_sl(l3, vec_dct_scale_log2);
|
||||
b[4] = vec_sl(l4, vec_dct_scale_log2);
|
||||
b[5] = vec_sl(l5, vec_dct_scale_log2);
|
||||
b[6] = vec_sl(l6, vec_dct_scale_log2);
|
||||
b[7] = vec_sl(l7, vec_dct_scale_log2);
|
||||
|
||||
b[8] = vec_sl(l8, vec_dct_scale_log2);
|
||||
b[9] = vec_sl(l9, vec_dct_scale_log2);
|
||||
b[10] = vec_sl(l10, vec_dct_scale_log2);
|
||||
b[11] = vec_sl(l11, vec_dct_scale_log2);
|
||||
b[12] = vec_sl(l12, vec_dct_scale_log2);
|
||||
b[13] = vec_sl(l13, vec_dct_scale_log2);
|
||||
b[14] = vec_sl(l14, vec_dct_scale_log2);
|
||||
b[15] = vec_sl(l15, vec_dct_scale_log2);
|
||||
}
|
||||
{
|
||||
const int16x8_t l16 = vec_vsx_ld(0, a + 16 * stride);
|
||||
const int16x8_t l17 = vec_vsx_ld(0, a + 17 * stride);
|
||||
const int16x8_t l18 = vec_vsx_ld(0, a + 18 * stride);
|
||||
const int16x8_t l19 = vec_vsx_ld(0, a + 19 * stride);
|
||||
const int16x8_t l20 = vec_vsx_ld(0, a + 20 * stride);
|
||||
const int16x8_t l21 = vec_vsx_ld(0, a + 21 * stride);
|
||||
const int16x8_t l22 = vec_vsx_ld(0, a + 22 * stride);
|
||||
const int16x8_t l23 = vec_vsx_ld(0, a + 23 * stride);
|
||||
|
||||
const int16x8_t l24 = vec_vsx_ld(0, a + 24 * stride);
|
||||
const int16x8_t l25 = vec_vsx_ld(0, a + 25 * stride);
|
||||
const int16x8_t l26 = vec_vsx_ld(0, a + 26 * stride);
|
||||
const int16x8_t l27 = vec_vsx_ld(0, a + 27 * stride);
|
||||
const int16x8_t l28 = vec_vsx_ld(0, a + 28 * stride);
|
||||
const int16x8_t l29 = vec_vsx_ld(0, a + 29 * stride);
|
||||
const int16x8_t l30 = vec_vsx_ld(0, a + 30 * stride);
|
||||
const int16x8_t l31 = vec_vsx_ld(0, a + 31 * stride);
|
||||
|
||||
b[16] = vec_sl(l16, vec_dct_scale_log2);
|
||||
b[17] = vec_sl(l17, vec_dct_scale_log2);
|
||||
b[18] = vec_sl(l18, vec_dct_scale_log2);
|
||||
b[19] = vec_sl(l19, vec_dct_scale_log2);
|
||||
b[20] = vec_sl(l20, vec_dct_scale_log2);
|
||||
b[21] = vec_sl(l21, vec_dct_scale_log2);
|
||||
b[22] = vec_sl(l22, vec_dct_scale_log2);
|
||||
b[23] = vec_sl(l23, vec_dct_scale_log2);
|
||||
|
||||
b[24] = vec_sl(l24, vec_dct_scale_log2);
|
||||
b[25] = vec_sl(l25, vec_dct_scale_log2);
|
||||
b[26] = vec_sl(l26, vec_dct_scale_log2);
|
||||
b[27] = vec_sl(l27, vec_dct_scale_log2);
|
||||
b[28] = vec_sl(l28, vec_dct_scale_log2);
|
||||
b[29] = vec_sl(l29, vec_dct_scale_log2);
|
||||
b[30] = vec_sl(l30, vec_dct_scale_log2);
|
||||
b[31] = vec_sl(l31, vec_dct_scale_log2);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void store(tran_low_t *a, const int16x8_t *b) {
|
||||
vec_vsx_st(b[0], 0, a);
|
||||
vec_vsx_st(b[8], 0, a + 8);
|
||||
vec_vsx_st(b[16], 0, a + 16);
|
||||
vec_vsx_st(b[24], 0, a + 24);
|
||||
|
||||
vec_vsx_st(b[1], 0, a + 32);
|
||||
vec_vsx_st(b[9], 0, a + 40);
|
||||
vec_vsx_st(b[17], 0, a + 48);
|
||||
vec_vsx_st(b[25], 0, a + 56);
|
||||
|
||||
vec_vsx_st(b[2], 0, a + 64);
|
||||
vec_vsx_st(b[10], 0, a + 72);
|
||||
vec_vsx_st(b[18], 0, a + 80);
|
||||
vec_vsx_st(b[26], 0, a + 88);
|
||||
|
||||
vec_vsx_st(b[3], 0, a + 96);
|
||||
vec_vsx_st(b[11], 0, a + 104);
|
||||
vec_vsx_st(b[19], 0, a + 112);
|
||||
vec_vsx_st(b[27], 0, a + 120);
|
||||
|
||||
vec_vsx_st(b[4], 0, a + 128);
|
||||
vec_vsx_st(b[12], 0, a + 136);
|
||||
vec_vsx_st(b[20], 0, a + 144);
|
||||
vec_vsx_st(b[28], 0, a + 152);
|
||||
|
||||
vec_vsx_st(b[5], 0, a + 160);
|
||||
vec_vsx_st(b[13], 0, a + 168);
|
||||
vec_vsx_st(b[21], 0, a + 176);
|
||||
vec_vsx_st(b[29], 0, a + 184);
|
||||
|
||||
vec_vsx_st(b[6], 0, a + 192);
|
||||
vec_vsx_st(b[14], 0, a + 200);
|
||||
vec_vsx_st(b[22], 0, a + 208);
|
||||
vec_vsx_st(b[30], 0, a + 216);
|
||||
|
||||
vec_vsx_st(b[7], 0, a + 224);
|
||||
vec_vsx_st(b[15], 0, a + 232);
|
||||
vec_vsx_st(b[23], 0, a + 240);
|
||||
vec_vsx_st(b[31], 0, a + 248);
|
||||
}
|
||||
|
||||
// Returns 1 if negative 0 if positive
|
||||
static INLINE int16x8_t vec_sign_s16(int16x8_t a) {
|
||||
return vec_sr(a, vec_shift_sign_s16);
|
||||
}
|
||||
|
||||
// Add 2 if positive, 1 if negative, and shift by 2.
|
||||
static INLINE int16x8_t sub_round_shift(const int16x8_t a) {
|
||||
const int16x8_t sign = vec_sign_s16(a);
|
||||
return vec_sra(vec_sub(vec_add(a, vec_twos_s16), sign), vec_dct_scale_log2);
|
||||
}
|
||||
|
||||
// Add 1 if positive, 2 if negative, and shift by 2.
|
||||
// In practice, add 1, then add the sign bit, then shift without rounding.
|
||||
static INLINE int16x8_t add_round_shift_s16(const int16x8_t a) {
|
||||
const int16x8_t sign = vec_sign_s16(a);
|
||||
return vec_sra(vec_add(vec_add(a, vec_ones_s16), sign), vec_dct_scale_log2);
|
||||
}
|
||||
|
||||
static void fdct32_vsx(const int16x8_t *in, int16x8_t *out, int pass) {
|
||||
int16x8_t temp0[32]; // Hold stages: 1, 4, 7
|
||||
int16x8_t temp1[32]; // Hold stages: 2, 5
|
||||
int16x8_t temp2[32]; // Hold stages: 3, 6
|
||||
int i;
|
||||
|
||||
// Stage 1
|
||||
// Unrolling this loops actually slows down Power9 benchmarks
|
||||
for (i = 0; i < 16; i++) {
|
||||
temp0[i] = vec_add(in[i], in[31 - i]);
|
||||
// pass through to stage 3.
|
||||
temp1[i + 16] = vec_sub(in[15 - i], in[i + 16]);
|
||||
}
|
||||
|
||||
// Stage 2
|
||||
// Unrolling this loops actually slows down Power9 benchmarks
|
||||
for (i = 0; i < 8; i++) {
|
||||
temp1[i] = vec_add(temp0[i], temp0[15 - i]);
|
||||
temp1[i + 8] = vec_sub(temp0[7 - i], temp0[i + 8]);
|
||||
}
|
||||
|
||||
// Apply butterflies (in place) on pass through to stage 3.
|
||||
single_butterfly(temp1[27], temp1[20], &temp1[27], &temp1[20]);
|
||||
single_butterfly(temp1[26], temp1[21], &temp1[26], &temp1[21]);
|
||||
single_butterfly(temp1[25], temp1[22], &temp1[25], &temp1[22]);
|
||||
single_butterfly(temp1[24], temp1[23], &temp1[24], &temp1[23]);
|
||||
|
||||
// dump the magnitude by 4, hence the intermediate values are within
|
||||
// the range of 16 bits.
|
||||
if (pass) {
|
||||
temp1[0] = add_round_shift_s16(temp1[0]);
|
||||
temp1[1] = add_round_shift_s16(temp1[1]);
|
||||
temp1[2] = add_round_shift_s16(temp1[2]);
|
||||
temp1[3] = add_round_shift_s16(temp1[3]);
|
||||
temp1[4] = add_round_shift_s16(temp1[4]);
|
||||
temp1[5] = add_round_shift_s16(temp1[5]);
|
||||
temp1[6] = add_round_shift_s16(temp1[6]);
|
||||
temp1[7] = add_round_shift_s16(temp1[7]);
|
||||
temp1[8] = add_round_shift_s16(temp1[8]);
|
||||
temp1[9] = add_round_shift_s16(temp1[9]);
|
||||
temp1[10] = add_round_shift_s16(temp1[10]);
|
||||
temp1[11] = add_round_shift_s16(temp1[11]);
|
||||
temp1[12] = add_round_shift_s16(temp1[12]);
|
||||
temp1[13] = add_round_shift_s16(temp1[13]);
|
||||
temp1[14] = add_round_shift_s16(temp1[14]);
|
||||
temp1[15] = add_round_shift_s16(temp1[15]);
|
||||
|
||||
temp1[16] = add_round_shift_s16(temp1[16]);
|
||||
temp1[17] = add_round_shift_s16(temp1[17]);
|
||||
temp1[18] = add_round_shift_s16(temp1[18]);
|
||||
temp1[19] = add_round_shift_s16(temp1[19]);
|
||||
temp1[20] = add_round_shift_s16(temp1[20]);
|
||||
temp1[21] = add_round_shift_s16(temp1[21]);
|
||||
temp1[22] = add_round_shift_s16(temp1[22]);
|
||||
temp1[23] = add_round_shift_s16(temp1[23]);
|
||||
temp1[24] = add_round_shift_s16(temp1[24]);
|
||||
temp1[25] = add_round_shift_s16(temp1[25]);
|
||||
temp1[26] = add_round_shift_s16(temp1[26]);
|
||||
temp1[27] = add_round_shift_s16(temp1[27]);
|
||||
temp1[28] = add_round_shift_s16(temp1[28]);
|
||||
temp1[29] = add_round_shift_s16(temp1[29]);
|
||||
temp1[30] = add_round_shift_s16(temp1[30]);
|
||||
temp1[31] = add_round_shift_s16(temp1[31]);
|
||||
}
|
||||
|
||||
// Stage 3
|
||||
temp2[0] = vec_add(temp1[0], temp1[7]);
|
||||
temp2[1] = vec_add(temp1[1], temp1[6]);
|
||||
temp2[2] = vec_add(temp1[2], temp1[5]);
|
||||
temp2[3] = vec_add(temp1[3], temp1[4]);
|
||||
temp2[5] = vec_sub(temp1[2], temp1[5]);
|
||||
temp2[6] = vec_sub(temp1[1], temp1[6]);
|
||||
temp2[8] = temp1[8];
|
||||
temp2[9] = temp1[9];
|
||||
|
||||
single_butterfly(temp1[13], temp1[10], &temp2[13], &temp2[10]);
|
||||
single_butterfly(temp1[12], temp1[11], &temp2[12], &temp2[11]);
|
||||
temp2[14] = temp1[14];
|
||||
temp2[15] = temp1[15];
|
||||
|
||||
temp2[18] = vec_add(temp1[18], temp1[21]);
|
||||
temp2[19] = vec_add(temp1[19], temp1[20]);
|
||||
|
||||
temp2[20] = vec_sub(temp1[19], temp1[20]);
|
||||
temp2[21] = vec_sub(temp1[18], temp1[21]);
|
||||
|
||||
temp2[26] = vec_sub(temp1[29], temp1[26]);
|
||||
temp2[27] = vec_sub(temp1[28], temp1[27]);
|
||||
|
||||
temp2[28] = vec_add(temp1[28], temp1[27]);
|
||||
temp2[29] = vec_add(temp1[29], temp1[26]);
|
||||
|
||||
// Pass through Stage 4
|
||||
temp0[7] = vec_sub(temp1[0], temp1[7]);
|
||||
temp0[4] = vec_sub(temp1[3], temp1[4]);
|
||||
temp0[16] = vec_add(temp1[16], temp1[23]);
|
||||
temp0[17] = vec_add(temp1[17], temp1[22]);
|
||||
temp0[22] = vec_sub(temp1[17], temp1[22]);
|
||||
temp0[23] = vec_sub(temp1[16], temp1[23]);
|
||||
temp0[24] = vec_sub(temp1[31], temp1[24]);
|
||||
temp0[25] = vec_sub(temp1[30], temp1[25]);
|
||||
temp0[30] = vec_add(temp1[30], temp1[25]);
|
||||
temp0[31] = vec_add(temp1[31], temp1[24]);
|
||||
|
||||
// Stage 4
|
||||
temp0[0] = vec_add(temp2[0], temp2[3]);
|
||||
temp0[1] = vec_add(temp2[1], temp2[2]);
|
||||
temp0[2] = vec_sub(temp2[1], temp2[2]);
|
||||
temp0[3] = vec_sub(temp2[0], temp2[3]);
|
||||
single_butterfly(temp2[6], temp2[5], &temp0[6], &temp0[5]);
|
||||
|
||||
temp0[9] = vec_add(temp2[9], temp2[10]);
|
||||
temp0[10] = vec_sub(temp2[9], temp2[10]);
|
||||
temp0[13] = vec_sub(temp2[14], temp2[13]);
|
||||
temp0[14] = vec_add(temp2[14], temp2[13]);
|
||||
|
||||
double_butterfly(temp2[29], cospi8_v, temp2[18], cospi24_v, &temp0[29],
|
||||
&temp0[18]);
|
||||
double_butterfly(temp2[28], cospi8_v, temp2[19], cospi24_v, &temp0[28],
|
||||
&temp0[19]);
|
||||
double_butterfly(temp2[27], cospi24_v, temp2[20], cospi8m_v, &temp0[27],
|
||||
&temp0[20]);
|
||||
double_butterfly(temp2[26], cospi24_v, temp2[21], cospi8m_v, &temp0[26],
|
||||
&temp0[21]);
|
||||
|
||||
// Pass through Stage 5
|
||||
temp1[8] = vec_add(temp2[8], temp2[11]);
|
||||
temp1[11] = vec_sub(temp2[8], temp2[11]);
|
||||
temp1[12] = vec_sub(temp2[15], temp2[12]);
|
||||
temp1[15] = vec_add(temp2[15], temp2[12]);
|
||||
|
||||
// Stage 5
|
||||
// 0 and 1 pass through to 0 and 16 at the end
|
||||
single_butterfly(temp0[0], temp0[1], &out[0], &out[16]);
|
||||
|
||||
// 2 and 3 pass through to 8 and 24 at the end
|
||||
double_butterfly(temp0[3], cospi8_v, temp0[2], cospi24_v, &out[8], &out[24]);
|
||||
|
||||
temp1[4] = vec_add(temp0[4], temp0[5]);
|
||||
temp1[5] = vec_sub(temp0[4], temp0[5]);
|
||||
temp1[6] = vec_sub(temp0[7], temp0[6]);
|
||||
temp1[7] = vec_add(temp0[7], temp0[6]);
|
||||
|
||||
double_butterfly(temp0[14], cospi8_v, temp0[9], cospi24_v, &temp1[14],
|
||||
&temp1[9]);
|
||||
double_butterfly(temp0[13], cospi24_v, temp0[10], cospi8m_v, &temp1[13],
|
||||
&temp1[10]);
|
||||
|
||||
temp1[17] = vec_add(temp0[17], temp0[18]);
|
||||
temp1[18] = vec_sub(temp0[17], temp0[18]);
|
||||
|
||||
temp1[21] = vec_sub(temp0[22], temp0[21]);
|
||||
temp1[22] = vec_add(temp0[22], temp0[21]);
|
||||
|
||||
temp1[25] = vec_add(temp0[25], temp0[26]);
|
||||
temp1[26] = vec_sub(temp0[25], temp0[26]);
|
||||
|
||||
temp1[29] = vec_sub(temp0[30], temp0[29]);
|
||||
temp1[30] = vec_add(temp0[30], temp0[29]);
|
||||
|
||||
// Pass through Stage 6
|
||||
temp2[16] = vec_add(temp0[16], temp0[19]);
|
||||
temp2[19] = vec_sub(temp0[16], temp0[19]);
|
||||
temp2[20] = vec_sub(temp0[23], temp0[20]);
|
||||
temp2[23] = vec_add(temp0[23], temp0[20]);
|
||||
temp2[24] = vec_add(temp0[24], temp0[27]);
|
||||
temp2[27] = vec_sub(temp0[24], temp0[27]);
|
||||
temp2[28] = vec_sub(temp0[31], temp0[28]);
|
||||
temp2[31] = vec_add(temp0[31], temp0[28]);
|
||||
|
||||
// Stage 6
|
||||
// 4 and 7 pass through to 4 and 28 at the end
|
||||
double_butterfly(temp1[7], cospi4_v, temp1[4], cospi28_v, &out[4], &out[28]);
|
||||
// 5 and 6 pass through to 20 and 12 at the end
|
||||
double_butterfly(temp1[6], cospi20_v, temp1[5], cospi12_v, &out[20],
|
||||
&out[12]);
|
||||
temp2[8] = vec_add(temp1[8], temp1[9]);
|
||||
temp2[9] = vec_sub(temp1[8], temp1[9]);
|
||||
temp2[10] = vec_sub(temp1[11], temp1[10]);
|
||||
temp2[11] = vec_add(temp1[11], temp1[10]);
|
||||
temp2[12] = vec_add(temp1[12], temp1[13]);
|
||||
temp2[13] = vec_sub(temp1[12], temp1[13]);
|
||||
temp2[14] = vec_sub(temp1[15], temp1[14]);
|
||||
temp2[15] = vec_add(temp1[15], temp1[14]);
|
||||
|
||||
double_butterfly(temp1[30], cospi4_v, temp1[17], cospi28_v, &temp2[30],
|
||||
&temp2[17]);
|
||||
double_butterfly(temp1[29], cospi28_v, temp1[18], cospi4m_v, &temp2[29],
|
||||
&temp2[18]);
|
||||
double_butterfly(temp1[26], cospi20_v, temp1[21], cospi12_v, &temp2[26],
|
||||
&temp2[21]);
|
||||
double_butterfly(temp1[25], cospi12_v, temp1[22], cospi20m_v, &temp2[25],
|
||||
&temp2[22]);
|
||||
|
||||
// Stage 7
|
||||
double_butterfly(temp2[15], cospi2_v, temp2[8], cospi30_v, &out[2], &out[30]);
|
||||
double_butterfly(temp2[14], cospi18_v, temp2[9], cospi14_v, &out[18],
|
||||
&out[14]);
|
||||
double_butterfly(temp2[13], cospi10_v, temp2[10], cospi22_v, &out[10],
|
||||
&out[22]);
|
||||
double_butterfly(temp2[12], cospi26_v, temp2[11], cospi6_v, &out[26],
|
||||
&out[6]);
|
||||
|
||||
temp0[16] = vec_add(temp2[16], temp2[17]);
|
||||
temp0[17] = vec_sub(temp2[16], temp2[17]);
|
||||
temp0[18] = vec_sub(temp2[19], temp2[18]);
|
||||
temp0[19] = vec_add(temp2[19], temp2[18]);
|
||||
temp0[20] = vec_add(temp2[20], temp2[21]);
|
||||
temp0[21] = vec_sub(temp2[20], temp2[21]);
|
||||
temp0[22] = vec_sub(temp2[23], temp2[22]);
|
||||
temp0[23] = vec_add(temp2[23], temp2[22]);
|
||||
temp0[24] = vec_add(temp2[24], temp2[25]);
|
||||
temp0[25] = vec_sub(temp2[24], temp2[25]);
|
||||
temp0[26] = vec_sub(temp2[27], temp2[26]);
|
||||
temp0[27] = vec_add(temp2[27], temp2[26]);
|
||||
temp0[28] = vec_add(temp2[28], temp2[29]);
|
||||
temp0[29] = vec_sub(temp2[28], temp2[29]);
|
||||
temp0[30] = vec_sub(temp2[31], temp2[30]);
|
||||
temp0[31] = vec_add(temp2[31], temp2[30]);
|
||||
|
||||
// Final stage --- outputs indices are bit-reversed.
|
||||
double_butterfly(temp0[31], cospi1_v, temp0[16], cospi31_v, &out[1],
|
||||
&out[31]);
|
||||
double_butterfly(temp0[30], cospi17_v, temp0[17], cospi15_v, &out[17],
|
||||
&out[15]);
|
||||
double_butterfly(temp0[29], cospi9_v, temp0[18], cospi23_v, &out[9],
|
||||
&out[23]);
|
||||
double_butterfly(temp0[28], cospi25_v, temp0[19], cospi7_v, &out[25],
|
||||
&out[7]);
|
||||
double_butterfly(temp0[27], cospi5_v, temp0[20], cospi27_v, &out[5],
|
||||
&out[27]);
|
||||
double_butterfly(temp0[26], cospi21_v, temp0[21], cospi11_v, &out[21],
|
||||
&out[11]);
|
||||
double_butterfly(temp0[25], cospi13_v, temp0[22], cospi19_v, &out[13],
|
||||
&out[19]);
|
||||
double_butterfly(temp0[24], cospi29_v, temp0[23], cospi3_v, &out[29],
|
||||
&out[3]);
|
||||
|
||||
if (pass == 0) {
|
||||
for (i = 0; i < 32; i++) {
|
||||
out[i] = sub_round_shift(out[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_fdct32x32_rd_vsx(const int16_t *input, tran_low_t *out, int stride) {
|
||||
int16x8_t temp0[32];
|
||||
int16x8_t temp1[32];
|
||||
int16x8_t temp2[32];
|
||||
int16x8_t temp3[32];
|
||||
int16x8_t temp4[32];
|
||||
int16x8_t temp5[32];
|
||||
int16x8_t temp6[32];
|
||||
|
||||
// Process in 8x32 columns.
|
||||
load(input, stride, temp0);
|
||||
fdct32_vsx(temp0, temp1, 0);
|
||||
|
||||
load(input + 8, stride, temp0);
|
||||
fdct32_vsx(temp0, temp2, 0);
|
||||
|
||||
load(input + 16, stride, temp0);
|
||||
fdct32_vsx(temp0, temp3, 0);
|
||||
|
||||
load(input + 24, stride, temp0);
|
||||
fdct32_vsx(temp0, temp4, 0);
|
||||
|
||||
// Generate the top row by munging the first set of 8 from each one
|
||||
// together.
|
||||
transpose_8x8(&temp1[0], &temp0[0]);
|
||||
transpose_8x8(&temp2[0], &temp0[8]);
|
||||
transpose_8x8(&temp3[0], &temp0[16]);
|
||||
transpose_8x8(&temp4[0], &temp0[24]);
|
||||
|
||||
fdct32_vsx(temp0, temp5, 1);
|
||||
|
||||
transpose_8x8(&temp5[0], &temp6[0]);
|
||||
transpose_8x8(&temp5[8], &temp6[8]);
|
||||
transpose_8x8(&temp5[16], &temp6[16]);
|
||||
transpose_8x8(&temp5[24], &temp6[24]);
|
||||
|
||||
store(out, temp6);
|
||||
|
||||
// Second row of 8x32.
|
||||
transpose_8x8(&temp1[8], &temp0[0]);
|
||||
transpose_8x8(&temp2[8], &temp0[8]);
|
||||
transpose_8x8(&temp3[8], &temp0[16]);
|
||||
transpose_8x8(&temp4[8], &temp0[24]);
|
||||
|
||||
fdct32_vsx(temp0, temp5, 1);
|
||||
|
||||
transpose_8x8(&temp5[0], &temp6[0]);
|
||||
transpose_8x8(&temp5[8], &temp6[8]);
|
||||
transpose_8x8(&temp5[16], &temp6[16]);
|
||||
transpose_8x8(&temp5[24], &temp6[24]);
|
||||
|
||||
store(out + 8 * 32, temp6);
|
||||
|
||||
// Third row of 8x32
|
||||
transpose_8x8(&temp1[16], &temp0[0]);
|
||||
transpose_8x8(&temp2[16], &temp0[8]);
|
||||
transpose_8x8(&temp3[16], &temp0[16]);
|
||||
transpose_8x8(&temp4[16], &temp0[24]);
|
||||
|
||||
fdct32_vsx(temp0, temp5, 1);
|
||||
|
||||
transpose_8x8(&temp5[0], &temp6[0]);
|
||||
transpose_8x8(&temp5[8], &temp6[8]);
|
||||
transpose_8x8(&temp5[16], &temp6[16]);
|
||||
transpose_8x8(&temp5[24], &temp6[24]);
|
||||
|
||||
store(out + 16 * 32, temp6);
|
||||
|
||||
// Final row of 8x32.
|
||||
transpose_8x8(&temp1[24], &temp0[0]);
|
||||
transpose_8x8(&temp2[24], &temp0[8]);
|
||||
transpose_8x8(&temp3[24], &temp0[16]);
|
||||
transpose_8x8(&temp4[24], &temp0[24]);
|
||||
|
||||
fdct32_vsx(temp0, temp5, 1);
|
||||
|
||||
transpose_8x8(&temp5[0], &temp6[0]);
|
||||
transpose_8x8(&temp5[8], &temp6[8]);
|
||||
transpose_8x8(&temp5[16], &temp6[16]);
|
||||
transpose_8x8(&temp5[24], &temp6[24]);
|
||||
|
||||
store(out + 24 * 32, temp6);
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
#include "vpx_dsp/ppc/transpose_vsx.h"
|
||||
#include "vpx_dsp/ppc/bitdepth_conversion_vsx.h"
|
||||
|
||||
static void vpx_hadamard_s16_8x8_one_pass(int16x8_t v[8]) {
|
||||
const int16x8_t b0 = vec_add(v[0], v[1]);
|
||||
const int16x8_t b1 = vec_sub(v[0], v[1]);
|
||||
const int16x8_t b2 = vec_add(v[2], v[3]);
|
||||
const int16x8_t b3 = vec_sub(v[2], v[3]);
|
||||
const int16x8_t b4 = vec_add(v[4], v[5]);
|
||||
const int16x8_t b5 = vec_sub(v[4], v[5]);
|
||||
const int16x8_t b6 = vec_add(v[6], v[7]);
|
||||
const int16x8_t b7 = vec_sub(v[6], v[7]);
|
||||
|
||||
const int16x8_t c0 = vec_add(b0, b2);
|
||||
const int16x8_t c1 = vec_add(b1, b3);
|
||||
const int16x8_t c2 = vec_sub(b0, b2);
|
||||
const int16x8_t c3 = vec_sub(b1, b3);
|
||||
const int16x8_t c4 = vec_add(b4, b6);
|
||||
const int16x8_t c5 = vec_add(b5, b7);
|
||||
const int16x8_t c6 = vec_sub(b4, b6);
|
||||
const int16x8_t c7 = vec_sub(b5, b7);
|
||||
|
||||
v[0] = vec_add(c0, c4);
|
||||
v[1] = vec_sub(c2, c6);
|
||||
v[2] = vec_sub(c0, c4);
|
||||
v[3] = vec_add(c2, c6);
|
||||
v[4] = vec_add(c3, c7);
|
||||
v[5] = vec_sub(c3, c7);
|
||||
v[6] = vec_sub(c1, c5);
|
||||
v[7] = vec_add(c1, c5);
|
||||
}
|
||||
|
||||
void vpx_hadamard_8x8_vsx(const int16_t *src_diff, ptrdiff_t src_stride,
|
||||
tran_low_t *coeff) {
|
||||
int16x8_t v[8];
|
||||
|
||||
v[0] = vec_vsx_ld(0, src_diff);
|
||||
v[1] = vec_vsx_ld(0, src_diff + src_stride);
|
||||
v[2] = vec_vsx_ld(0, src_diff + (2 * src_stride));
|
||||
v[3] = vec_vsx_ld(0, src_diff + (3 * src_stride));
|
||||
v[4] = vec_vsx_ld(0, src_diff + (4 * src_stride));
|
||||
v[5] = vec_vsx_ld(0, src_diff + (5 * src_stride));
|
||||
v[6] = vec_vsx_ld(0, src_diff + (6 * src_stride));
|
||||
v[7] = vec_vsx_ld(0, src_diff + (7 * src_stride));
|
||||
|
||||
vpx_hadamard_s16_8x8_one_pass(v);
|
||||
|
||||
vpx_transpose_s16_8x8(v);
|
||||
|
||||
vpx_hadamard_s16_8x8_one_pass(v);
|
||||
|
||||
store_tran_low(v[0], 0, coeff);
|
||||
store_tran_low(v[1], 0, coeff + 8);
|
||||
store_tran_low(v[2], 0, coeff + 16);
|
||||
store_tran_low(v[3], 0, coeff + 24);
|
||||
store_tran_low(v[4], 0, coeff + 32);
|
||||
store_tran_low(v[5], 0, coeff + 40);
|
||||
store_tran_low(v[6], 0, coeff + 48);
|
||||
store_tran_low(v[7], 0, coeff + 56);
|
||||
}
|
||||
|
||||
void vpx_hadamard_16x16_vsx(const int16_t *src_diff, ptrdiff_t src_stride,
|
||||
tran_low_t *coeff) {
|
||||
int i;
|
||||
const uint16x8_t ones = vec_splat_u16(1);
|
||||
|
||||
/* Rearrange 16x16 to 8x32 and remove stride.
|
||||
* Top left first. */
|
||||
vpx_hadamard_8x8_vsx(src_diff, src_stride, coeff);
|
||||
/* Top right. */
|
||||
vpx_hadamard_8x8_vsx(src_diff + 8 + 0 * src_stride, src_stride, coeff + 64);
|
||||
/* Bottom left. */
|
||||
vpx_hadamard_8x8_vsx(src_diff + 0 + 8 * src_stride, src_stride, coeff + 128);
|
||||
/* Bottom right. */
|
||||
vpx_hadamard_8x8_vsx(src_diff + 8 + 8 * src_stride, src_stride, coeff + 192);
|
||||
|
||||
/* Overlay the 8x8 blocks and combine. */
|
||||
for (i = 0; i < 64; i += 8) {
|
||||
const int16x8_t a0 = load_tran_low(0, coeff);
|
||||
const int16x8_t a1 = load_tran_low(0, coeff + 64);
|
||||
const int16x8_t a2 = load_tran_low(0, coeff + 128);
|
||||
const int16x8_t a3 = load_tran_low(0, coeff + 192);
|
||||
|
||||
/* Prevent the result from escaping int16_t. */
|
||||
const int16x8_t b0 = vec_sra(a0, ones);
|
||||
const int16x8_t b1 = vec_sra(a1, ones);
|
||||
const int16x8_t b2 = vec_sra(a2, ones);
|
||||
const int16x8_t b3 = vec_sra(a3, ones);
|
||||
|
||||
const int16x8_t c0 = vec_add(b0, b1);
|
||||
const int16x8_t c2 = vec_add(b2, b3);
|
||||
const int16x8_t c1 = vec_sub(b0, b1);
|
||||
const int16x8_t c3 = vec_sub(b2, b3);
|
||||
|
||||
const int16x8_t d0 = vec_add(c0, c2);
|
||||
const int16x8_t d1 = vec_add(c1, c3);
|
||||
const int16x8_t d2 = vec_sub(c0, c2);
|
||||
const int16x8_t d3 = vec_sub(c1, c3);
|
||||
|
||||
store_tran_low(d0, 0, coeff);
|
||||
store_tran_low(d1, 0, coeff + 64);
|
||||
store_tran_low(d2, 0, coeff + 128);
|
||||
store_tran_low(d3, 0, coeff + 192);
|
||||
|
||||
coeff += 8;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,767 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
void vpx_v_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t d = vec_vsx_ld(0, above);
|
||||
int i;
|
||||
(void)left;
|
||||
|
||||
for (i = 0; i < 16; i++, dst += stride) {
|
||||
vec_vsx_st(d, 0, dst);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_v_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t d0 = vec_vsx_ld(0, above);
|
||||
const uint8x16_t d1 = vec_vsx_ld(16, above);
|
||||
int i;
|
||||
(void)left;
|
||||
|
||||
for (i = 0; i < 32; i++, dst += stride) {
|
||||
vec_vsx_st(d0, 0, dst);
|
||||
vec_vsx_st(d1, 16, dst);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(crbug.com/webm/1522): Fix test failures.
|
||||
#if 0
|
||||
static const uint32x4_t mask4 = { 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF };
|
||||
|
||||
void vpx_h_predictor_4x4_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t d = vec_vsx_ld(0, left);
|
||||
const uint8x16_t v0 = vec_splat(d, 0);
|
||||
const uint8x16_t v1 = vec_splat(d, 1);
|
||||
const uint8x16_t v2 = vec_splat(d, 2);
|
||||
const uint8x16_t v3 = vec_splat(d, 3);
|
||||
|
||||
(void)above;
|
||||
|
||||
vec_vsx_st(vec_sel(v0, vec_vsx_ld(0, dst), (uint8x16_t)mask4), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(vec_sel(v1, vec_vsx_ld(0, dst), (uint8x16_t)mask4), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(vec_sel(v2, vec_vsx_ld(0, dst), (uint8x16_t)mask4), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(vec_sel(v3, vec_vsx_ld(0, dst), (uint8x16_t)mask4), 0, dst);
|
||||
}
|
||||
|
||||
void vpx_h_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t d = vec_vsx_ld(0, left);
|
||||
const uint8x16_t v0 = vec_splat(d, 0);
|
||||
const uint8x16_t v1 = vec_splat(d, 1);
|
||||
const uint8x16_t v2 = vec_splat(d, 2);
|
||||
const uint8x16_t v3 = vec_splat(d, 3);
|
||||
|
||||
const uint8x16_t v4 = vec_splat(d, 4);
|
||||
const uint8x16_t v5 = vec_splat(d, 5);
|
||||
const uint8x16_t v6 = vec_splat(d, 6);
|
||||
const uint8x16_t v7 = vec_splat(d, 7);
|
||||
|
||||
(void)above;
|
||||
|
||||
vec_vsx_st(xxpermdi(v0, vec_vsx_ld(0, dst), 1), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(xxpermdi(v1, vec_vsx_ld(0, dst), 1), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(xxpermdi(v2, vec_vsx_ld(0, dst), 1), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(xxpermdi(v3, vec_vsx_ld(0, dst), 1), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(xxpermdi(v4, vec_vsx_ld(0, dst), 1), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(xxpermdi(v5, vec_vsx_ld(0, dst), 1), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(xxpermdi(v6, vec_vsx_ld(0, dst), 1), 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(xxpermdi(v7, vec_vsx_ld(0, dst), 1), 0, dst);
|
||||
}
|
||||
#endif
|
||||
|
||||
void vpx_h_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t d = vec_vsx_ld(0, left);
|
||||
const uint8x16_t v0 = vec_splat(d, 0);
|
||||
const uint8x16_t v1 = vec_splat(d, 1);
|
||||
const uint8x16_t v2 = vec_splat(d, 2);
|
||||
const uint8x16_t v3 = vec_splat(d, 3);
|
||||
|
||||
const uint8x16_t v4 = vec_splat(d, 4);
|
||||
const uint8x16_t v5 = vec_splat(d, 5);
|
||||
const uint8x16_t v6 = vec_splat(d, 6);
|
||||
const uint8x16_t v7 = vec_splat(d, 7);
|
||||
|
||||
const uint8x16_t v8 = vec_splat(d, 8);
|
||||
const uint8x16_t v9 = vec_splat(d, 9);
|
||||
const uint8x16_t v10 = vec_splat(d, 10);
|
||||
const uint8x16_t v11 = vec_splat(d, 11);
|
||||
|
||||
const uint8x16_t v12 = vec_splat(d, 12);
|
||||
const uint8x16_t v13 = vec_splat(d, 13);
|
||||
const uint8x16_t v14 = vec_splat(d, 14);
|
||||
const uint8x16_t v15 = vec_splat(d, 15);
|
||||
|
||||
(void)above;
|
||||
|
||||
vec_vsx_st(v0, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v1, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v2, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v3, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v4, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v5, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v6, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v7, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v8, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v9, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v10, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v11, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v12, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v13, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v14, 0, dst);
|
||||
dst += stride;
|
||||
vec_vsx_st(v15, 0, dst);
|
||||
}
|
||||
|
||||
#define H_PREDICTOR_32(v) \
|
||||
vec_vsx_st(v, 0, dst); \
|
||||
vec_vsx_st(v, 16, dst); \
|
||||
dst += stride
|
||||
|
||||
void vpx_h_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t d0 = vec_vsx_ld(0, left);
|
||||
const uint8x16_t d1 = vec_vsx_ld(16, left);
|
||||
|
||||
const uint8x16_t v0_0 = vec_splat(d0, 0);
|
||||
const uint8x16_t v1_0 = vec_splat(d0, 1);
|
||||
const uint8x16_t v2_0 = vec_splat(d0, 2);
|
||||
const uint8x16_t v3_0 = vec_splat(d0, 3);
|
||||
const uint8x16_t v4_0 = vec_splat(d0, 4);
|
||||
const uint8x16_t v5_0 = vec_splat(d0, 5);
|
||||
const uint8x16_t v6_0 = vec_splat(d0, 6);
|
||||
const uint8x16_t v7_0 = vec_splat(d0, 7);
|
||||
const uint8x16_t v8_0 = vec_splat(d0, 8);
|
||||
const uint8x16_t v9_0 = vec_splat(d0, 9);
|
||||
const uint8x16_t v10_0 = vec_splat(d0, 10);
|
||||
const uint8x16_t v11_0 = vec_splat(d0, 11);
|
||||
const uint8x16_t v12_0 = vec_splat(d0, 12);
|
||||
const uint8x16_t v13_0 = vec_splat(d0, 13);
|
||||
const uint8x16_t v14_0 = vec_splat(d0, 14);
|
||||
const uint8x16_t v15_0 = vec_splat(d0, 15);
|
||||
|
||||
const uint8x16_t v0_1 = vec_splat(d1, 0);
|
||||
const uint8x16_t v1_1 = vec_splat(d1, 1);
|
||||
const uint8x16_t v2_1 = vec_splat(d1, 2);
|
||||
const uint8x16_t v3_1 = vec_splat(d1, 3);
|
||||
const uint8x16_t v4_1 = vec_splat(d1, 4);
|
||||
const uint8x16_t v5_1 = vec_splat(d1, 5);
|
||||
const uint8x16_t v6_1 = vec_splat(d1, 6);
|
||||
const uint8x16_t v7_1 = vec_splat(d1, 7);
|
||||
const uint8x16_t v8_1 = vec_splat(d1, 8);
|
||||
const uint8x16_t v9_1 = vec_splat(d1, 9);
|
||||
const uint8x16_t v10_1 = vec_splat(d1, 10);
|
||||
const uint8x16_t v11_1 = vec_splat(d1, 11);
|
||||
const uint8x16_t v12_1 = vec_splat(d1, 12);
|
||||
const uint8x16_t v13_1 = vec_splat(d1, 13);
|
||||
const uint8x16_t v14_1 = vec_splat(d1, 14);
|
||||
const uint8x16_t v15_1 = vec_splat(d1, 15);
|
||||
|
||||
(void)above;
|
||||
|
||||
H_PREDICTOR_32(v0_0);
|
||||
H_PREDICTOR_32(v1_0);
|
||||
H_PREDICTOR_32(v2_0);
|
||||
H_PREDICTOR_32(v3_0);
|
||||
|
||||
H_PREDICTOR_32(v4_0);
|
||||
H_PREDICTOR_32(v5_0);
|
||||
H_PREDICTOR_32(v6_0);
|
||||
H_PREDICTOR_32(v7_0);
|
||||
|
||||
H_PREDICTOR_32(v8_0);
|
||||
H_PREDICTOR_32(v9_0);
|
||||
H_PREDICTOR_32(v10_0);
|
||||
H_PREDICTOR_32(v11_0);
|
||||
|
||||
H_PREDICTOR_32(v12_0);
|
||||
H_PREDICTOR_32(v13_0);
|
||||
H_PREDICTOR_32(v14_0);
|
||||
H_PREDICTOR_32(v15_0);
|
||||
|
||||
H_PREDICTOR_32(v0_1);
|
||||
H_PREDICTOR_32(v1_1);
|
||||
H_PREDICTOR_32(v2_1);
|
||||
H_PREDICTOR_32(v3_1);
|
||||
|
||||
H_PREDICTOR_32(v4_1);
|
||||
H_PREDICTOR_32(v5_1);
|
||||
H_PREDICTOR_32(v6_1);
|
||||
H_PREDICTOR_32(v7_1);
|
||||
|
||||
H_PREDICTOR_32(v8_1);
|
||||
H_PREDICTOR_32(v9_1);
|
||||
H_PREDICTOR_32(v10_1);
|
||||
H_PREDICTOR_32(v11_1);
|
||||
|
||||
H_PREDICTOR_32(v12_1);
|
||||
H_PREDICTOR_32(v13_1);
|
||||
H_PREDICTOR_32(v14_1);
|
||||
H_PREDICTOR_32(v15_1);
|
||||
}
|
||||
|
||||
// TODO(crbug.com/webm/1522): Fix test failures.
|
||||
#if 0
|
||||
void vpx_tm_predictor_4x4_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const int16x8_t tl = unpack_to_s16_h(vec_splat(vec_vsx_ld(-1, above), 0));
|
||||
const int16x8_t l = unpack_to_s16_h(vec_vsx_ld(0, left));
|
||||
const int16x8_t a = unpack_to_s16_h(vec_vsx_ld(0, above));
|
||||
int16x8_t tmp, val;
|
||||
uint8x16_t d;
|
||||
|
||||
d = vec_vsx_ld(0, dst);
|
||||
tmp = unpack_to_s16_l(d);
|
||||
val = vec_sub(vec_add(vec_splat(l, 0), a), tl);
|
||||
vec_vsx_st(vec_sel(vec_packsu(val, tmp), d, (uint8x16_t)mask4), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
d = vec_vsx_ld(0, dst);
|
||||
tmp = unpack_to_s16_l(d);
|
||||
val = vec_sub(vec_add(vec_splat(l, 1), a), tl);
|
||||
vec_vsx_st(vec_sel(vec_packsu(val, tmp), d, (uint8x16_t)mask4), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
d = vec_vsx_ld(0, dst);
|
||||
tmp = unpack_to_s16_l(d);
|
||||
val = vec_sub(vec_add(vec_splat(l, 2), a), tl);
|
||||
vec_vsx_st(vec_sel(vec_packsu(val, tmp), d, (uint8x16_t)mask4), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
d = vec_vsx_ld(0, dst);
|
||||
tmp = unpack_to_s16_l(d);
|
||||
val = vec_sub(vec_add(vec_splat(l, 3), a), tl);
|
||||
vec_vsx_st(vec_sel(vec_packsu(val, tmp), d, (uint8x16_t)mask4), 0, dst);
|
||||
}
|
||||
|
||||
void vpx_tm_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const int16x8_t tl = unpack_to_s16_h(vec_splat(vec_vsx_ld(-1, above), 0));
|
||||
const int16x8_t l = unpack_to_s16_h(vec_vsx_ld(0, left));
|
||||
const int16x8_t a = unpack_to_s16_h(vec_vsx_ld(0, above));
|
||||
int16x8_t tmp, val;
|
||||
|
||||
tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
|
||||
val = vec_sub(vec_add(vec_splat(l, 0), a), tl);
|
||||
vec_vsx_st(vec_packsu(val, tmp), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
|
||||
val = vec_sub(vec_add(vec_splat(l, 1), a), tl);
|
||||
vec_vsx_st(vec_packsu(val, tmp), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
|
||||
val = vec_sub(vec_add(vec_splat(l, 2), a), tl);
|
||||
vec_vsx_st(vec_packsu(val, tmp), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
|
||||
val = vec_sub(vec_add(vec_splat(l, 3), a), tl);
|
||||
vec_vsx_st(vec_packsu(val, tmp), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
|
||||
val = vec_sub(vec_add(vec_splat(l, 4), a), tl);
|
||||
vec_vsx_st(vec_packsu(val, tmp), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
|
||||
val = vec_sub(vec_add(vec_splat(l, 5), a), tl);
|
||||
vec_vsx_st(vec_packsu(val, tmp), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
|
||||
val = vec_sub(vec_add(vec_splat(l, 6), a), tl);
|
||||
vec_vsx_st(vec_packsu(val, tmp), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
|
||||
val = vec_sub(vec_add(vec_splat(l, 7), a), tl);
|
||||
vec_vsx_st(vec_packsu(val, tmp), 0, dst);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void tm_predictor_16x8(uint8_t *dst, const ptrdiff_t stride, int16x8_t l,
|
||||
int16x8_t ah, int16x8_t al, int16x8_t tl) {
|
||||
int16x8_t vh, vl, ls;
|
||||
|
||||
ls = vec_splat(l, 0);
|
||||
vh = vec_sub(vec_add(ls, ah), tl);
|
||||
vl = vec_sub(vec_add(ls, al), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
ls = vec_splat(l, 1);
|
||||
vh = vec_sub(vec_add(ls, ah), tl);
|
||||
vl = vec_sub(vec_add(ls, al), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
ls = vec_splat(l, 2);
|
||||
vh = vec_sub(vec_add(ls, ah), tl);
|
||||
vl = vec_sub(vec_add(ls, al), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
ls = vec_splat(l, 3);
|
||||
vh = vec_sub(vec_add(ls, ah), tl);
|
||||
vl = vec_sub(vec_add(ls, al), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
ls = vec_splat(l, 4);
|
||||
vh = vec_sub(vec_add(ls, ah), tl);
|
||||
vl = vec_sub(vec_add(ls, al), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
ls = vec_splat(l, 5);
|
||||
vh = vec_sub(vec_add(ls, ah), tl);
|
||||
vl = vec_sub(vec_add(ls, al), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
ls = vec_splat(l, 6);
|
||||
vh = vec_sub(vec_add(ls, ah), tl);
|
||||
vl = vec_sub(vec_add(ls, al), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 0, dst);
|
||||
dst += stride;
|
||||
|
||||
ls = vec_splat(l, 7);
|
||||
vh = vec_sub(vec_add(ls, ah), tl);
|
||||
vl = vec_sub(vec_add(ls, al), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 0, dst);
|
||||
}
|
||||
|
||||
void vpx_tm_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const int16x8_t tl = unpack_to_s16_h(vec_splat(vec_vsx_ld(-1, above), 0));
|
||||
const uint8x16_t l = vec_vsx_ld(0, left);
|
||||
const int16x8_t lh = unpack_to_s16_h(l);
|
||||
const int16x8_t ll = unpack_to_s16_l(l);
|
||||
const uint8x16_t a = vec_vsx_ld(0, above);
|
||||
const int16x8_t ah = unpack_to_s16_h(a);
|
||||
const int16x8_t al = unpack_to_s16_l(a);
|
||||
|
||||
tm_predictor_16x8(dst, stride, lh, ah, al, tl);
|
||||
|
||||
dst += stride * 8;
|
||||
|
||||
tm_predictor_16x8(dst, stride, ll, ah, al, tl);
|
||||
}
|
||||
|
||||
static INLINE void tm_predictor_32x1(uint8_t *dst, const int16x8_t ls,
|
||||
const int16x8_t a0h, const int16x8_t a0l,
|
||||
const int16x8_t a1h, const int16x8_t a1l,
|
||||
const int16x8_t tl) {
|
||||
int16x8_t vh, vl;
|
||||
|
||||
vh = vec_sub(vec_add(ls, a0h), tl);
|
||||
vl = vec_sub(vec_add(ls, a0l), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 0, dst);
|
||||
vh = vec_sub(vec_add(ls, a1h), tl);
|
||||
vl = vec_sub(vec_add(ls, a1l), tl);
|
||||
vec_vsx_st(vec_packsu(vh, vl), 16, dst);
|
||||
}
|
||||
|
||||
static void tm_predictor_32x8(uint8_t *dst, const ptrdiff_t stride,
|
||||
const int16x8_t l, const uint8x16_t a0,
|
||||
const uint8x16_t a1, const int16x8_t tl) {
|
||||
const int16x8_t a0h = unpack_to_s16_h(a0);
|
||||
const int16x8_t a0l = unpack_to_s16_l(a0);
|
||||
const int16x8_t a1h = unpack_to_s16_h(a1);
|
||||
const int16x8_t a1l = unpack_to_s16_l(a1);
|
||||
|
||||
tm_predictor_32x1(dst, vec_splat(l, 0), a0h, a0l, a1h, a1l, tl);
|
||||
dst += stride;
|
||||
|
||||
tm_predictor_32x1(dst, vec_splat(l, 1), a0h, a0l, a1h, a1l, tl);
|
||||
dst += stride;
|
||||
|
||||
tm_predictor_32x1(dst, vec_splat(l, 2), a0h, a0l, a1h, a1l, tl);
|
||||
dst += stride;
|
||||
|
||||
tm_predictor_32x1(dst, vec_splat(l, 3), a0h, a0l, a1h, a1l, tl);
|
||||
dst += stride;
|
||||
|
||||
tm_predictor_32x1(dst, vec_splat(l, 4), a0h, a0l, a1h, a1l, tl);
|
||||
dst += stride;
|
||||
|
||||
tm_predictor_32x1(dst, vec_splat(l, 5), a0h, a0l, a1h, a1l, tl);
|
||||
dst += stride;
|
||||
|
||||
tm_predictor_32x1(dst, vec_splat(l, 6), a0h, a0l, a1h, a1l, tl);
|
||||
dst += stride;
|
||||
|
||||
tm_predictor_32x1(dst, vec_splat(l, 7), a0h, a0l, a1h, a1l, tl);
|
||||
}
|
||||
|
||||
void vpx_tm_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const int16x8_t tl = unpack_to_s16_h(vec_splat(vec_vsx_ld(-1, above), 0));
|
||||
const uint8x16_t l0 = vec_vsx_ld(0, left);
|
||||
const uint8x16_t l1 = vec_vsx_ld(16, left);
|
||||
const uint8x16_t a0 = vec_vsx_ld(0, above);
|
||||
const uint8x16_t a1 = vec_vsx_ld(16, above);
|
||||
|
||||
tm_predictor_32x8(dst, stride, unpack_to_s16_h(l0), a0, a1, tl);
|
||||
dst += stride * 8;
|
||||
|
||||
tm_predictor_32x8(dst, stride, unpack_to_s16_l(l0), a0, a1, tl);
|
||||
dst += stride * 8;
|
||||
|
||||
tm_predictor_32x8(dst, stride, unpack_to_s16_h(l1), a0, a1, tl);
|
||||
dst += stride * 8;
|
||||
|
||||
tm_predictor_32x8(dst, stride, unpack_to_s16_l(l1), a0, a1, tl);
|
||||
}
|
||||
|
||||
static INLINE void dc_fill_predictor_8x8(uint8_t *dst, const ptrdiff_t stride,
|
||||
const uint8x16_t val) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 8; i++, dst += stride) {
|
||||
const uint8x16_t d = vec_vsx_ld(0, dst);
|
||||
vec_vsx_st(xxpermdi(val, d, 1), 0, dst);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void dc_fill_predictor_16x16(uint8_t *dst, const ptrdiff_t stride,
|
||||
const uint8x16_t val) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 16; i++, dst += stride) {
|
||||
vec_vsx_st(val, 0, dst);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_dc_128_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t v128 = vec_sl(vec_splat_u8(1), vec_splat_u8(7));
|
||||
(void)above;
|
||||
(void)left;
|
||||
|
||||
dc_fill_predictor_16x16(dst, stride, v128);
|
||||
}
|
||||
|
||||
static INLINE void dc_fill_predictor_32x32(uint8_t *dst, const ptrdiff_t stride,
|
||||
const uint8x16_t val) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 32; i++, dst += stride) {
|
||||
vec_vsx_st(val, 0, dst);
|
||||
vec_vsx_st(val, 16, dst);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_dc_128_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t v128 = vec_sl(vec_splat_u8(1), vec_splat_u8(7));
|
||||
(void)above;
|
||||
(void)left;
|
||||
|
||||
dc_fill_predictor_32x32(dst, stride, v128);
|
||||
}
|
||||
|
||||
static uint8x16_t avg16(const uint8_t *values) {
|
||||
const int32x4_t sum4s =
|
||||
(int32x4_t)vec_sum4s(vec_vsx_ld(0, values), vec_splat_u32(0));
|
||||
const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s, vec_splat_s32(8));
|
||||
const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(4));
|
||||
|
||||
return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
|
||||
3);
|
||||
}
|
||||
|
||||
void vpx_dc_left_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above,
|
||||
const uint8_t *left) {
|
||||
(void)above;
|
||||
|
||||
dc_fill_predictor_16x16(dst, stride, avg16(left));
|
||||
}
|
||||
|
||||
void vpx_dc_top_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
(void)left;
|
||||
|
||||
dc_fill_predictor_16x16(dst, stride, avg16(above));
|
||||
}
|
||||
|
||||
static uint8x16_t avg32(const uint8_t *values) {
|
||||
const uint8x16_t v0 = vec_vsx_ld(0, values);
|
||||
const uint8x16_t v1 = vec_vsx_ld(16, values);
|
||||
const int32x4_t v16 = vec_sl(vec_splat_s32(1), vec_splat_u32(4));
|
||||
const int32x4_t sum4s =
|
||||
(int32x4_t)vec_sum4s(v0, vec_sum4s(v1, vec_splat_u32(0)));
|
||||
const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s, v16);
|
||||
const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(5));
|
||||
|
||||
return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
|
||||
3);
|
||||
}
|
||||
|
||||
void vpx_dc_left_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above,
|
||||
const uint8_t *left) {
|
||||
(void)above;
|
||||
|
||||
dc_fill_predictor_32x32(dst, stride, avg32(left));
|
||||
}
|
||||
|
||||
void vpx_dc_top_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
(void)left;
|
||||
|
||||
dc_fill_predictor_32x32(dst, stride, avg32(above));
|
||||
}
|
||||
|
||||
// TODO(crbug.com/webm/1522): Fix test failures.
|
||||
#if 0
|
||||
static uint8x16_t dc_avg8(const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t a0 = vec_vsx_ld(0, above);
|
||||
const uint8x16_t l0 = vec_vsx_ld(0, left);
|
||||
const int32x4_t sum4s =
|
||||
(int32x4_t)vec_sum4s(l0, vec_sum4s(a0, vec_splat_u32(0)));
|
||||
const int32x4_t sum4s8 = xxpermdi(sum4s, vec_splat_s32(0), 1);
|
||||
const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s8, vec_splat_s32(8));
|
||||
const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(4));
|
||||
|
||||
return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
|
||||
3);
|
||||
}
|
||||
#endif
|
||||
|
||||
static uint8x16_t dc_avg16(const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t a0 = vec_vsx_ld(0, above);
|
||||
const uint8x16_t l0 = vec_vsx_ld(0, left);
|
||||
const int32x4_t v16 = vec_sl(vec_splat_s32(1), vec_splat_u32(4));
|
||||
const int32x4_t sum4s =
|
||||
(int32x4_t)vec_sum4s(l0, vec_sum4s(a0, vec_splat_u32(0)));
|
||||
const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s, v16);
|
||||
const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(5));
|
||||
|
||||
return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
|
||||
3);
|
||||
}
|
||||
|
||||
// TODO(crbug.com/webm/1522): Fix test failures.
|
||||
#if 0
|
||||
void vpx_dc_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
dc_fill_predictor_8x8(dst, stride, dc_avg8(above, left));
|
||||
}
|
||||
#endif
|
||||
|
||||
void vpx_dc_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
dc_fill_predictor_16x16(dst, stride, dc_avg16(above, left));
|
||||
}
|
||||
|
||||
static uint8x16_t dc_avg32(const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t a0 = vec_vsx_ld(0, above);
|
||||
const uint8x16_t a1 = vec_vsx_ld(16, above);
|
||||
const uint8x16_t l0 = vec_vsx_ld(0, left);
|
||||
const uint8x16_t l1 = vec_vsx_ld(16, left);
|
||||
const int32x4_t v32 = vec_sl(vec_splat_s32(1), vec_splat_u32(5));
|
||||
const uint32x4_t a_sum = vec_sum4s(a0, vec_sum4s(a1, vec_splat_u32(0)));
|
||||
const int32x4_t sum4s = (int32x4_t)vec_sum4s(l0, vec_sum4s(l1, a_sum));
|
||||
const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s, v32);
|
||||
const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(6));
|
||||
|
||||
return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
|
||||
3);
|
||||
}
|
||||
|
||||
void vpx_dc_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
dc_fill_predictor_32x32(dst, stride, dc_avg32(above, left));
|
||||
}
|
||||
|
||||
static uint8x16_t avg3(const uint8x16_t a, const uint8x16_t b,
|
||||
const uint8x16_t c) {
|
||||
const uint8x16_t ac =
|
||||
vec_adds(vec_and(a, c), vec_sr(vec_xor(a, c), vec_splat_u8(1)));
|
||||
|
||||
return vec_avg(ac, b);
|
||||
}
|
||||
|
||||
// Workaround vec_sld/vec_xxsldi/vec_lsdoi being missing or broken.
|
||||
static const uint8x16_t sl1 = { 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8,
|
||||
0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF, 0x10 };
|
||||
|
||||
// TODO(crbug.com/webm/1522): Fix test failures.
|
||||
#if 0
|
||||
void vpx_d45_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t af = vec_vsx_ld(0, above);
|
||||
const uint8x16_t above_right = vec_splat(af, 7);
|
||||
const uint8x16_t a = xxpermdi(af, above_right, 1);
|
||||
const uint8x16_t b = vec_perm(a, above_right, sl1);
|
||||
const uint8x16_t c = vec_perm(b, above_right, sl1);
|
||||
uint8x16_t row = avg3(a, b, c);
|
||||
int i;
|
||||
(void)left;
|
||||
|
||||
for (i = 0; i < 8; i++) {
|
||||
const uint8x16_t d = vec_vsx_ld(0, dst);
|
||||
vec_vsx_st(xxpermdi(row, d, 1), 0, dst);
|
||||
dst += stride;
|
||||
row = vec_perm(row, above_right, sl1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void vpx_d45_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t a = vec_vsx_ld(0, above);
|
||||
const uint8x16_t above_right = vec_splat(a, 15);
|
||||
const uint8x16_t b = vec_perm(a, above_right, sl1);
|
||||
const uint8x16_t c = vec_perm(b, above_right, sl1);
|
||||
uint8x16_t row = avg3(a, b, c);
|
||||
int i;
|
||||
(void)left;
|
||||
|
||||
for (i = 0; i < 16; i++) {
|
||||
vec_vsx_st(row, 0, dst);
|
||||
dst += stride;
|
||||
row = vec_perm(row, above_right, sl1);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_d45_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t a0 = vec_vsx_ld(0, above);
|
||||
const uint8x16_t a1 = vec_vsx_ld(16, above);
|
||||
const uint8x16_t above_right = vec_splat(a1, 15);
|
||||
const uint8x16_t b0 = vec_perm(a0, a1, sl1);
|
||||
const uint8x16_t b1 = vec_perm(a1, above_right, sl1);
|
||||
const uint8x16_t c0 = vec_perm(b0, b1, sl1);
|
||||
const uint8x16_t c1 = vec_perm(b1, above_right, sl1);
|
||||
uint8x16_t row0 = avg3(a0, b0, c0);
|
||||
uint8x16_t row1 = avg3(a1, b1, c1);
|
||||
int i;
|
||||
(void)left;
|
||||
|
||||
for (i = 0; i < 32; i++) {
|
||||
vec_vsx_st(row0, 0, dst);
|
||||
vec_vsx_st(row1, 16, dst);
|
||||
dst += stride;
|
||||
row0 = vec_perm(row0, row1, sl1);
|
||||
row1 = vec_perm(row1, above_right, sl1);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(crbug.com/webm/1522): Fix test failures.
|
||||
#if 0
|
||||
void vpx_d63_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t af = vec_vsx_ld(0, above);
|
||||
const uint8x16_t above_right = vec_splat(af, 9);
|
||||
const uint8x16_t a = xxpermdi(af, above_right, 1);
|
||||
const uint8x16_t b = vec_perm(a, above_right, sl1);
|
||||
const uint8x16_t c = vec_perm(b, above_right, sl1);
|
||||
uint8x16_t row0 = vec_avg(a, b);
|
||||
uint8x16_t row1 = avg3(a, b, c);
|
||||
int i;
|
||||
(void)left;
|
||||
|
||||
for (i = 0; i < 4; i++) {
|
||||
const uint8x16_t d0 = vec_vsx_ld(0, dst);
|
||||
const uint8x16_t d1 = vec_vsx_ld(0, dst + stride);
|
||||
vec_vsx_st(xxpermdi(row0, d0, 1), 0, dst);
|
||||
vec_vsx_st(xxpermdi(row1, d1, 1), 0, dst + stride);
|
||||
dst += stride * 2;
|
||||
row0 = vec_perm(row0, above_right, sl1);
|
||||
row1 = vec_perm(row1, above_right, sl1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void vpx_d63_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t a0 = vec_vsx_ld(0, above);
|
||||
const uint8x16_t a1 = vec_vsx_ld(16, above);
|
||||
const uint8x16_t above_right = vec_splat(a1, 0);
|
||||
const uint8x16_t b = vec_perm(a0, above_right, sl1);
|
||||
const uint8x16_t c = vec_perm(b, above_right, sl1);
|
||||
uint8x16_t row0 = vec_avg(a0, b);
|
||||
uint8x16_t row1 = avg3(a0, b, c);
|
||||
int i;
|
||||
(void)left;
|
||||
|
||||
for (i = 0; i < 8; i++) {
|
||||
vec_vsx_st(row0, 0, dst);
|
||||
vec_vsx_st(row1, 0, dst + stride);
|
||||
dst += stride * 2;
|
||||
row0 = vec_perm(row0, above_right, sl1);
|
||||
row1 = vec_perm(row1, above_right, sl1);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_d63_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
|
||||
const uint8_t *above, const uint8_t *left) {
|
||||
const uint8x16_t a0 = vec_vsx_ld(0, above);
|
||||
const uint8x16_t a1 = vec_vsx_ld(16, above);
|
||||
const uint8x16_t a2 = vec_vsx_ld(32, above);
|
||||
const uint8x16_t above_right = vec_splat(a2, 0);
|
||||
const uint8x16_t b0 = vec_perm(a0, a1, sl1);
|
||||
const uint8x16_t b1 = vec_perm(a1, above_right, sl1);
|
||||
const uint8x16_t c0 = vec_perm(b0, b1, sl1);
|
||||
const uint8x16_t c1 = vec_perm(b1, above_right, sl1);
|
||||
uint8x16_t row0_0 = vec_avg(a0, b0);
|
||||
uint8x16_t row0_1 = vec_avg(a1, b1);
|
||||
uint8x16_t row1_0 = avg3(a0, b0, c0);
|
||||
uint8x16_t row1_1 = avg3(a1, b1, c1);
|
||||
int i;
|
||||
(void)left;
|
||||
|
||||
for (i = 0; i < 16; i++) {
|
||||
vec_vsx_st(row0_0, 0, dst);
|
||||
vec_vsx_st(row0_1, 16, dst);
|
||||
vec_vsx_st(row1_0, 0, dst + stride);
|
||||
vec_vsx_st(row1_1, 16, dst + stride);
|
||||
dst += stride * 2;
|
||||
row0_0 = vec_perm(row0_0, row0_1, sl1);
|
||||
row0_1 = vec_perm(row0_1, above_right, sl1);
|
||||
row1_0 = vec_perm(row1_0, row1_1, sl1);
|
||||
row1_1 = vec_perm(row1_1, above_right, sl1);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_PPC_INV_TXFM_VSX_H_
|
||||
#define VPX_VPX_DSP_PPC_INV_TXFM_VSX_H_
|
||||
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
void vpx_round_store4x4_vsx(int16x8_t *in, int16x8_t *out, uint8_t *dest,
|
||||
int stride);
|
||||
void vpx_idct4_vsx(int16x8_t *in, int16x8_t *out);
|
||||
void vp9_iadst4_vsx(int16x8_t *in, int16x8_t *out);
|
||||
|
||||
void vpx_round_store8x8_vsx(int16x8_t *in, uint8_t *dest, int stride);
|
||||
void vpx_idct8_vsx(int16x8_t *in, int16x8_t *out);
|
||||
void vp9_iadst8_vsx(int16x8_t *in, int16x8_t *out);
|
||||
|
||||
#define LOAD_INPUT16(load, source, offset, step, in) \
|
||||
in[0] = load(offset, source); \
|
||||
in[1] = load((step) + (offset), source); \
|
||||
in[2] = load(2 * (step) + (offset), source); \
|
||||
in[3] = load(3 * (step) + (offset), source); \
|
||||
in[4] = load(4 * (step) + (offset), source); \
|
||||
in[5] = load(5 * (step) + (offset), source); \
|
||||
in[6] = load(6 * (step) + (offset), source); \
|
||||
in[7] = load(7 * (step) + (offset), source); \
|
||||
in[8] = load(8 * (step) + (offset), source); \
|
||||
in[9] = load(9 * (step) + (offset), source); \
|
||||
in[10] = load(10 * (step) + (offset), source); \
|
||||
in[11] = load(11 * (step) + (offset), source); \
|
||||
in[12] = load(12 * (step) + (offset), source); \
|
||||
in[13] = load(13 * (step) + (offset), source); \
|
||||
in[14] = load(14 * (step) + (offset), source); \
|
||||
in[15] = load(15 * (step) + (offset), source);
|
||||
|
||||
void vpx_round_store16x16_vsx(int16x8_t *src0, int16x8_t *src1, uint8_t *dest,
|
||||
int stride);
|
||||
void vpx_idct16_vsx(int16x8_t *src0, int16x8_t *src1);
|
||||
void vpx_iadst16_vsx(int16x8_t *src0, int16x8_t *src1);
|
||||
|
||||
#endif // VPX_VPX_DSP_PPC_INV_TXFM_VSX_H_
|
||||
@@ -0,0 +1,305 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
// Negate 16-bit integers in a when the corresponding signed 16-bit
|
||||
// integer in b is negative.
|
||||
static INLINE int16x8_t vec_sign(int16x8_t a, int16x8_t b) {
|
||||
const int16x8_t mask = vec_sra(b, vec_shift_sign_s16);
|
||||
return vec_xor(vec_add(a, mask), mask);
|
||||
}
|
||||
|
||||
// Sets the value of a 32-bit integers to 1 when the corresponding value in a is
|
||||
// negative.
|
||||
static INLINE int32x4_t vec_is_neg(int32x4_t a) {
|
||||
return vec_sr(a, vec_shift_sign_s32);
|
||||
}
|
||||
|
||||
// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit
|
||||
// integers, and return the high 16 bits of the intermediate integers.
|
||||
// (a * b) >> 16
|
||||
static INLINE int16x8_t vec_mulhi(int16x8_t a, int16x8_t b) {
|
||||
// madds does ((A * B) >>15) + C, we need >> 16, so we perform an extra right
|
||||
// shift.
|
||||
return vec_sra(vec_madds(a, b, vec_zeros_s16), vec_ones_u16);
|
||||
}
|
||||
|
||||
// Quantization function used for 4x4, 8x8 and 16x16 blocks.
|
||||
static INLINE int16x8_t quantize_coeff(int16x8_t coeff, int16x8_t coeff_abs,
|
||||
int16x8_t round, int16x8_t quant,
|
||||
int16x8_t quant_shift, bool16x8_t mask) {
|
||||
const int16x8_t rounded = vec_vaddshs(coeff_abs, round);
|
||||
int16x8_t qcoeff = vec_mulhi(rounded, quant);
|
||||
qcoeff = vec_add(qcoeff, rounded);
|
||||
qcoeff = vec_mulhi(qcoeff, quant_shift);
|
||||
qcoeff = vec_sign(qcoeff, coeff);
|
||||
return vec_and(qcoeff, mask);
|
||||
}
|
||||
|
||||
// Quantization function used for 32x32 blocks.
|
||||
static INLINE int16x8_t quantize_coeff_32(int16x8_t coeff, int16x8_t coeff_abs,
|
||||
int16x8_t round, int16x8_t quant,
|
||||
int16x8_t quant_shift,
|
||||
bool16x8_t mask) {
|
||||
const int16x8_t rounded = vec_vaddshs(coeff_abs, round);
|
||||
int16x8_t qcoeff = vec_mulhi(rounded, quant);
|
||||
qcoeff = vec_add(qcoeff, rounded);
|
||||
// 32x32 blocks require an extra multiplication by 2, this compensates for the
|
||||
// extra right shift added in vec_mulhi, as such vec_madds can be used
|
||||
// directly instead of vec_mulhi (((a * b) >> 15) >> 1) << 1 == (a * b >> 15)
|
||||
qcoeff = vec_madds(qcoeff, quant_shift, vec_zeros_s16);
|
||||
qcoeff = vec_sign(qcoeff, coeff);
|
||||
return vec_and(qcoeff, mask);
|
||||
}
|
||||
|
||||
// DeQuantization function used for 32x32 blocks. Quantized coeff of 32x32
|
||||
// blocks are twice as big as for other block sizes. As such, using
|
||||
// vec_mladd results in overflow.
|
||||
static INLINE int16x8_t dequantize_coeff_32(int16x8_t qcoeff,
|
||||
int16x8_t dequant) {
|
||||
int32x4_t dqcoeffe = vec_mule(qcoeff, dequant);
|
||||
int32x4_t dqcoeffo = vec_mulo(qcoeff, dequant);
|
||||
// Add 1 if negative to round towards zero because the C uses division.
|
||||
dqcoeffe = vec_add(dqcoeffe, vec_is_neg(dqcoeffe));
|
||||
dqcoeffo = vec_add(dqcoeffo, vec_is_neg(dqcoeffo));
|
||||
dqcoeffe = vec_sra(dqcoeffe, vec_ones_u32);
|
||||
dqcoeffo = vec_sra(dqcoeffo, vec_ones_u32);
|
||||
return (int16x8_t)vec_perm(dqcoeffe, dqcoeffo, vec_perm_odd_even_pack);
|
||||
}
|
||||
|
||||
static INLINE int16x8_t nonzero_scanindex(int16x8_t qcoeff, bool16x8_t mask,
|
||||
const int16_t *iscan_ptr, int index) {
|
||||
int16x8_t scan = vec_vsx_ld(index, iscan_ptr);
|
||||
bool16x8_t zero_coeff = vec_cmpeq(qcoeff, vec_zeros_s16);
|
||||
scan = vec_sub(scan, mask);
|
||||
return vec_andc(scan, zero_coeff);
|
||||
}
|
||||
|
||||
// Compare packed 16-bit integers across a, and return the maximum value in
|
||||
// every element. Returns a vector containing the biggest value across vector a.
|
||||
static INLINE int16x8_t vec_max_across(int16x8_t a) {
|
||||
a = vec_max(a, vec_perm(a, a, vec_perm64));
|
||||
a = vec_max(a, vec_perm(a, a, vec_perm32));
|
||||
return vec_max(a, vec_perm(a, a, vec_perm16));
|
||||
}
|
||||
|
||||
void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
|
||||
int skip_block, const int16_t *zbin_ptr,
|
||||
const int16_t *round_ptr, const int16_t *quant_ptr,
|
||||
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
|
||||
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
|
||||
uint16_t *eob_ptr, const int16_t *scan_ptr,
|
||||
const int16_t *iscan_ptr) {
|
||||
int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob;
|
||||
bool16x8_t zero_mask0, zero_mask1;
|
||||
|
||||
// First set of 8 coeff starts with DC + 7 AC
|
||||
int16x8_t zbin = vec_vsx_ld(0, zbin_ptr);
|
||||
int16x8_t round = vec_vsx_ld(0, round_ptr);
|
||||
int16x8_t quant = vec_vsx_ld(0, quant_ptr);
|
||||
int16x8_t dequant = vec_vsx_ld(0, dequant_ptr);
|
||||
int16x8_t quant_shift = vec_vsx_ld(0, quant_shift_ptr);
|
||||
|
||||
int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr);
|
||||
int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr);
|
||||
|
||||
int16x8_t coeff0_abs = vec_abs(coeff0);
|
||||
int16x8_t coeff1_abs = vec_abs(coeff1);
|
||||
|
||||
zero_mask0 = vec_cmpge(coeff0_abs, zbin);
|
||||
zbin = vec_splat(zbin, 1);
|
||||
zero_mask1 = vec_cmpge(coeff1_abs, zbin);
|
||||
|
||||
(void)scan_ptr;
|
||||
(void)skip_block;
|
||||
assert(!skip_block);
|
||||
|
||||
qcoeff0 =
|
||||
quantize_coeff(coeff0, coeff0_abs, round, quant, quant_shift, zero_mask0);
|
||||
vec_vsx_st(qcoeff0, 0, qcoeff_ptr);
|
||||
round = vec_splat(round, 1);
|
||||
quant = vec_splat(quant, 1);
|
||||
quant_shift = vec_splat(quant_shift, 1);
|
||||
qcoeff1 =
|
||||
quantize_coeff(coeff1, coeff1_abs, round, quant, quant_shift, zero_mask1);
|
||||
vec_vsx_st(qcoeff1, 16, qcoeff_ptr);
|
||||
|
||||
dqcoeff0 = vec_mladd(qcoeff0, dequant, vec_zeros_s16);
|
||||
vec_vsx_st(dqcoeff0, 0, dqcoeff_ptr);
|
||||
dequant = vec_splat(dequant, 1);
|
||||
dqcoeff1 = vec_mladd(qcoeff1, dequant, vec_zeros_s16);
|
||||
vec_vsx_st(dqcoeff1, 16, dqcoeff_ptr);
|
||||
|
||||
eob = vec_max(nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, 0),
|
||||
nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, 16));
|
||||
|
||||
if (n_coeffs > 16) {
|
||||
int index = 16;
|
||||
int off0 = 32;
|
||||
int off1 = 48;
|
||||
int off2 = 64;
|
||||
do {
|
||||
int16x8_t coeff2, coeff2_abs, qcoeff2, dqcoeff2, eob2;
|
||||
bool16x8_t zero_mask2;
|
||||
coeff0 = vec_vsx_ld(off0, coeff_ptr);
|
||||
coeff1 = vec_vsx_ld(off1, coeff_ptr);
|
||||
coeff2 = vec_vsx_ld(off2, coeff_ptr);
|
||||
coeff0_abs = vec_abs(coeff0);
|
||||
coeff1_abs = vec_abs(coeff1);
|
||||
coeff2_abs = vec_abs(coeff2);
|
||||
zero_mask0 = vec_cmpge(coeff0_abs, zbin);
|
||||
zero_mask1 = vec_cmpge(coeff1_abs, zbin);
|
||||
zero_mask2 = vec_cmpge(coeff2_abs, zbin);
|
||||
qcoeff0 = quantize_coeff(coeff0, coeff0_abs, round, quant, quant_shift,
|
||||
zero_mask0);
|
||||
qcoeff1 = quantize_coeff(coeff1, coeff1_abs, round, quant, quant_shift,
|
||||
zero_mask1);
|
||||
qcoeff2 = quantize_coeff(coeff2, coeff2_abs, round, quant, quant_shift,
|
||||
zero_mask2);
|
||||
vec_vsx_st(qcoeff0, off0, qcoeff_ptr);
|
||||
vec_vsx_st(qcoeff1, off1, qcoeff_ptr);
|
||||
vec_vsx_st(qcoeff2, off2, qcoeff_ptr);
|
||||
|
||||
dqcoeff0 = vec_mladd(qcoeff0, dequant, vec_zeros_s16);
|
||||
dqcoeff1 = vec_mladd(qcoeff1, dequant, vec_zeros_s16);
|
||||
dqcoeff2 = vec_mladd(qcoeff2, dequant, vec_zeros_s16);
|
||||
|
||||
vec_vsx_st(dqcoeff0, off0, dqcoeff_ptr);
|
||||
vec_vsx_st(dqcoeff1, off1, dqcoeff_ptr);
|
||||
vec_vsx_st(dqcoeff2, off2, dqcoeff_ptr);
|
||||
|
||||
eob =
|
||||
vec_max(eob, nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, off0));
|
||||
eob2 = vec_max(nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, off1),
|
||||
nonzero_scanindex(qcoeff2, zero_mask2, iscan_ptr, off2));
|
||||
eob = vec_max(eob, eob2);
|
||||
|
||||
index += 24;
|
||||
off0 += 48;
|
||||
off1 += 48;
|
||||
off2 += 48;
|
||||
} while (index < n_coeffs);
|
||||
}
|
||||
|
||||
eob = vec_max_across(eob);
|
||||
*eob_ptr = eob[0];
|
||||
}
|
||||
|
||||
void vpx_quantize_b_32x32_vsx(
|
||||
const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block,
|
||||
const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr,
|
||||
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
|
||||
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
|
||||
const int16_t *scan_ptr, const int16_t *iscan_ptr) {
|
||||
// In stage 1, we quantize 16 coeffs (DC + 15 AC)
|
||||
// In stage 2, we loop 42 times and quantize 24 coeffs per iteration
|
||||
// (32 * 32 - 16) / 24 = 42
|
||||
int num_itr = 42;
|
||||
// Offsets are in bytes, 16 coeffs = 32 bytes
|
||||
int off0 = 32;
|
||||
int off1 = 48;
|
||||
int off2 = 64;
|
||||
|
||||
int16x8_t qcoeff0, qcoeff1, eob;
|
||||
bool16x8_t zero_mask0, zero_mask1;
|
||||
|
||||
int16x8_t zbin = vec_vsx_ld(0, zbin_ptr);
|
||||
int16x8_t round = vec_vsx_ld(0, round_ptr);
|
||||
int16x8_t quant = vec_vsx_ld(0, quant_ptr);
|
||||
int16x8_t dequant = vec_vsx_ld(0, dequant_ptr);
|
||||
int16x8_t quant_shift = vec_vsx_ld(0, quant_shift_ptr);
|
||||
|
||||
int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr);
|
||||
int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr);
|
||||
|
||||
int16x8_t coeff0_abs = vec_abs(coeff0);
|
||||
int16x8_t coeff1_abs = vec_abs(coeff1);
|
||||
|
||||
(void)scan_ptr;
|
||||
(void)skip_block;
|
||||
(void)n_coeffs;
|
||||
assert(!skip_block);
|
||||
|
||||
// 32x32 quantization requires that zbin and round be divided by 2
|
||||
zbin = vec_sra(vec_add(zbin, vec_ones_s16), vec_ones_u16);
|
||||
round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16);
|
||||
|
||||
zero_mask0 = vec_cmpge(coeff0_abs, zbin);
|
||||
zbin = vec_splat(zbin, 1); // remove DC from zbin
|
||||
zero_mask1 = vec_cmpge(coeff1_abs, zbin);
|
||||
|
||||
qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift,
|
||||
zero_mask0);
|
||||
round = vec_splat(round, 1); // remove DC from round
|
||||
quant = vec_splat(quant, 1); // remove DC from quant
|
||||
quant_shift = vec_splat(quant_shift, 1); // remove DC from quant_shift
|
||||
qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift,
|
||||
zero_mask1);
|
||||
|
||||
vec_vsx_st(qcoeff0, 0, qcoeff_ptr);
|
||||
vec_vsx_st(qcoeff1, 16, qcoeff_ptr);
|
||||
|
||||
vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), 0, dqcoeff_ptr);
|
||||
dequant = vec_splat(dequant, 1); // remove DC from dequant
|
||||
vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), 16, dqcoeff_ptr);
|
||||
|
||||
eob = vec_max(nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, 0),
|
||||
nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, 16));
|
||||
|
||||
do {
|
||||
int16x8_t coeff2, coeff2_abs, qcoeff2, eob2;
|
||||
bool16x8_t zero_mask2;
|
||||
|
||||
coeff0 = vec_vsx_ld(off0, coeff_ptr);
|
||||
coeff1 = vec_vsx_ld(off1, coeff_ptr);
|
||||
coeff2 = vec_vsx_ld(off2, coeff_ptr);
|
||||
|
||||
coeff0_abs = vec_abs(coeff0);
|
||||
coeff1_abs = vec_abs(coeff1);
|
||||
coeff2_abs = vec_abs(coeff2);
|
||||
|
||||
zero_mask0 = vec_cmpge(coeff0_abs, zbin);
|
||||
zero_mask1 = vec_cmpge(coeff1_abs, zbin);
|
||||
zero_mask2 = vec_cmpge(coeff2_abs, zbin);
|
||||
|
||||
qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift,
|
||||
zero_mask0);
|
||||
qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift,
|
||||
zero_mask1);
|
||||
qcoeff2 = quantize_coeff_32(coeff2, coeff2_abs, round, quant, quant_shift,
|
||||
zero_mask2);
|
||||
|
||||
vec_vsx_st(qcoeff0, off0, qcoeff_ptr);
|
||||
vec_vsx_st(qcoeff1, off1, qcoeff_ptr);
|
||||
vec_vsx_st(qcoeff2, off2, qcoeff_ptr);
|
||||
|
||||
vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), off0, dqcoeff_ptr);
|
||||
vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), off1, dqcoeff_ptr);
|
||||
vec_vsx_st(dequantize_coeff_32(qcoeff2, dequant), off2, dqcoeff_ptr);
|
||||
|
||||
eob = vec_max(eob, nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, off0));
|
||||
eob2 = vec_max(nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, off1),
|
||||
nonzero_scanindex(qcoeff2, zero_mask2, iscan_ptr, off2));
|
||||
eob = vec_max(eob, eob2);
|
||||
|
||||
// 24 int16_t is 48 bytes
|
||||
off0 += 48;
|
||||
off1 += 48;
|
||||
off2 += 48;
|
||||
num_itr--;
|
||||
} while (num_itr != 0);
|
||||
|
||||
eob = vec_max_across(eob);
|
||||
*eob_ptr = eob[0];
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
|
||||
#define PROCESS16(offset) \
|
||||
v_a = vec_vsx_ld(offset, a); \
|
||||
v_b = vec_vsx_ld(offset, b); \
|
||||
v_abs = vec_absd(v_a, v_b); \
|
||||
v_sad = vec_sum4s(v_abs, v_sad);
|
||||
|
||||
#define SAD8(height) \
|
||||
unsigned int vpx_sad8x##height##_vsx(const uint8_t *a, int a_stride, \
|
||||
const uint8_t *b, int b_stride) { \
|
||||
int y = 0; \
|
||||
uint8x16_t v_a, v_b, v_abs; \
|
||||
uint32x4_t v_sad = vec_zeros_u32; \
|
||||
\
|
||||
do { \
|
||||
PROCESS16(0) \
|
||||
\
|
||||
a += a_stride; \
|
||||
b += b_stride; \
|
||||
y++; \
|
||||
} while (y < height); \
|
||||
\
|
||||
return v_sad[1] + v_sad[0]; \
|
||||
}
|
||||
|
||||
#define SAD16(height) \
|
||||
unsigned int vpx_sad16x##height##_vsx(const uint8_t *a, int a_stride, \
|
||||
const uint8_t *b, int b_stride) { \
|
||||
int y = 0; \
|
||||
uint8x16_t v_a, v_b, v_abs; \
|
||||
uint32x4_t v_sad = vec_zeros_u32; \
|
||||
\
|
||||
do { \
|
||||
PROCESS16(0); \
|
||||
\
|
||||
a += a_stride; \
|
||||
b += b_stride; \
|
||||
y++; \
|
||||
} while (y < height); \
|
||||
\
|
||||
return v_sad[3] + v_sad[2] + v_sad[1] + v_sad[0]; \
|
||||
}
|
||||
|
||||
#define SAD32(height) \
|
||||
unsigned int vpx_sad32x##height##_vsx(const uint8_t *a, int a_stride, \
|
||||
const uint8_t *b, int b_stride) { \
|
||||
int y = 0; \
|
||||
uint8x16_t v_a, v_b, v_abs; \
|
||||
uint32x4_t v_sad = vec_zeros_u32; \
|
||||
\
|
||||
do { \
|
||||
PROCESS16(0); \
|
||||
PROCESS16(16); \
|
||||
\
|
||||
a += a_stride; \
|
||||
b += b_stride; \
|
||||
y++; \
|
||||
} while (y < height); \
|
||||
\
|
||||
return v_sad[3] + v_sad[2] + v_sad[1] + v_sad[0]; \
|
||||
}
|
||||
|
||||
#define SAD64(height) \
|
||||
unsigned int vpx_sad64x##height##_vsx(const uint8_t *a, int a_stride, \
|
||||
const uint8_t *b, int b_stride) { \
|
||||
int y = 0; \
|
||||
uint8x16_t v_a, v_b, v_abs; \
|
||||
uint32x4_t v_sad = vec_zeros_u32; \
|
||||
\
|
||||
do { \
|
||||
PROCESS16(0); \
|
||||
PROCESS16(16); \
|
||||
PROCESS16(32); \
|
||||
PROCESS16(48); \
|
||||
\
|
||||
a += a_stride; \
|
||||
b += b_stride; \
|
||||
y++; \
|
||||
} while (y < height); \
|
||||
\
|
||||
return v_sad[3] + v_sad[2] + v_sad[1] + v_sad[0]; \
|
||||
}
|
||||
|
||||
SAD8(4);
|
||||
SAD8(8);
|
||||
SAD8(16);
|
||||
SAD16(8);
|
||||
SAD16(16);
|
||||
SAD16(32);
|
||||
SAD32(16);
|
||||
SAD32(32);
|
||||
SAD32(64);
|
||||
SAD64(32);
|
||||
SAD64(64);
|
||||
|
||||
#define SAD16AVG(height) \
|
||||
unsigned int vpx_sad16x##height##_avg_vsx( \
|
||||
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
|
||||
const uint8_t *second_pred) { \
|
||||
DECLARE_ALIGNED(16, uint8_t, comp_pred[16 * (height)]); \
|
||||
vpx_comp_avg_pred_vsx(comp_pred, second_pred, 16, height, ref, \
|
||||
ref_stride); \
|
||||
\
|
||||
return vpx_sad16x##height##_vsx(src, src_stride, comp_pred, 16); \
|
||||
}
|
||||
|
||||
#define SAD32AVG(height) \
|
||||
unsigned int vpx_sad32x##height##_avg_vsx( \
|
||||
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
|
||||
const uint8_t *second_pred) { \
|
||||
DECLARE_ALIGNED(32, uint8_t, comp_pred[32 * (height)]); \
|
||||
vpx_comp_avg_pred_vsx(comp_pred, second_pred, 32, height, ref, \
|
||||
ref_stride); \
|
||||
\
|
||||
return vpx_sad32x##height##_vsx(src, src_stride, comp_pred, 32); \
|
||||
}
|
||||
|
||||
#define SAD64AVG(height) \
|
||||
unsigned int vpx_sad64x##height##_avg_vsx( \
|
||||
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
|
||||
const uint8_t *second_pred) { \
|
||||
DECLARE_ALIGNED(64, uint8_t, comp_pred[64 * (height)]); \
|
||||
vpx_comp_avg_pred_vsx(comp_pred, second_pred, 64, height, ref, \
|
||||
ref_stride); \
|
||||
return vpx_sad64x##height##_vsx(src, src_stride, comp_pred, 64); \
|
||||
}
|
||||
|
||||
SAD16AVG(8);
|
||||
SAD16AVG(16);
|
||||
SAD16AVG(32);
|
||||
SAD32AVG(16);
|
||||
SAD32AVG(32);
|
||||
SAD32AVG(64);
|
||||
SAD64AVG(32);
|
||||
SAD64AVG(64);
|
||||
|
||||
#define PROCESS16_4D(offset, ref, v_h, v_l) \
|
||||
v_b = vec_vsx_ld(offset, ref); \
|
||||
v_bh = unpack_to_s16_h(v_b); \
|
||||
v_bl = unpack_to_s16_l(v_b); \
|
||||
v_subh = vec_sub(v_h, v_bh); \
|
||||
v_subl = vec_sub(v_l, v_bl); \
|
||||
v_absh = vec_abs(v_subh); \
|
||||
v_absl = vec_abs(v_subl); \
|
||||
v_sad = vec_sum4s(v_absh, v_sad); \
|
||||
v_sad = vec_sum4s(v_absl, v_sad);
|
||||
|
||||
#define UNPACK_SRC(offset, srcv_h, srcv_l) \
|
||||
v_a = vec_vsx_ld(offset, src); \
|
||||
srcv_h = unpack_to_s16_h(v_a); \
|
||||
srcv_l = unpack_to_s16_l(v_a);
|
||||
|
||||
#define SAD16_4D(height) \
|
||||
void vpx_sad16x##height##x4d_vsx(const uint8_t *src, int src_stride, \
|
||||
const uint8_t *const ref_array[], \
|
||||
int ref_stride, uint32_t *sad_array) { \
|
||||
int i; \
|
||||
int y; \
|
||||
unsigned int sad[4]; \
|
||||
uint8x16_t v_a, v_b; \
|
||||
int16x8_t v_ah, v_al, v_bh, v_bl, v_absh, v_absl, v_subh, v_subl; \
|
||||
\
|
||||
for (i = 0; i < 4; i++) sad_array[i] = 0; \
|
||||
\
|
||||
for (y = 0; y < height; y++) { \
|
||||
UNPACK_SRC(y *src_stride, v_ah, v_al); \
|
||||
for (i = 0; i < 4; i++) { \
|
||||
int32x4_t v_sad = vec_splat_s32(0); \
|
||||
PROCESS16_4D(y *ref_stride, ref_array[i], v_ah, v_al); \
|
||||
\
|
||||
vec_vsx_st((uint32x4_t)v_sad, 0, sad); \
|
||||
sad_array[i] += (sad[3] + sad[2] + sad[1] + sad[0]); \
|
||||
} \
|
||||
} \
|
||||
}
|
||||
|
||||
#define SAD32_4D(height) \
|
||||
void vpx_sad32x##height##x4d_vsx(const uint8_t *src, int src_stride, \
|
||||
const uint8_t *const ref_array[], \
|
||||
int ref_stride, uint32_t *sad_array) { \
|
||||
int i; \
|
||||
int y; \
|
||||
unsigned int sad[4]; \
|
||||
uint8x16_t v_a, v_b; \
|
||||
int16x8_t v_ah1, v_al1, v_ah2, v_al2, v_bh, v_bl; \
|
||||
int16x8_t v_absh, v_absl, v_subh, v_subl; \
|
||||
\
|
||||
for (i = 0; i < 4; i++) sad_array[i] = 0; \
|
||||
\
|
||||
for (y = 0; y < height; y++) { \
|
||||
UNPACK_SRC(y *src_stride, v_ah1, v_al1); \
|
||||
UNPACK_SRC(y *src_stride + 16, v_ah2, v_al2); \
|
||||
for (i = 0; i < 4; i++) { \
|
||||
int32x4_t v_sad = vec_splat_s32(0); \
|
||||
PROCESS16_4D(y *ref_stride, ref_array[i], v_ah1, v_al1); \
|
||||
PROCESS16_4D(y *ref_stride + 16, ref_array[i], v_ah2, v_al2); \
|
||||
\
|
||||
vec_vsx_st((uint32x4_t)v_sad, 0, sad); \
|
||||
sad_array[i] += (sad[3] + sad[2] + sad[1] + sad[0]); \
|
||||
} \
|
||||
} \
|
||||
}
|
||||
|
||||
#define SAD64_4D(height) \
|
||||
void vpx_sad64x##height##x4d_vsx(const uint8_t *src, int src_stride, \
|
||||
const uint8_t *const ref_array[], \
|
||||
int ref_stride, uint32_t *sad_array) { \
|
||||
int i; \
|
||||
int y; \
|
||||
unsigned int sad[4]; \
|
||||
uint8x16_t v_a, v_b; \
|
||||
int16x8_t v_ah1, v_al1, v_ah2, v_al2, v_bh, v_bl; \
|
||||
int16x8_t v_ah3, v_al3, v_ah4, v_al4; \
|
||||
int16x8_t v_absh, v_absl, v_subh, v_subl; \
|
||||
\
|
||||
for (i = 0; i < 4; i++) sad_array[i] = 0; \
|
||||
\
|
||||
for (y = 0; y < height; y++) { \
|
||||
UNPACK_SRC(y *src_stride, v_ah1, v_al1); \
|
||||
UNPACK_SRC(y *src_stride + 16, v_ah2, v_al2); \
|
||||
UNPACK_SRC(y *src_stride + 32, v_ah3, v_al3); \
|
||||
UNPACK_SRC(y *src_stride + 48, v_ah4, v_al4); \
|
||||
for (i = 0; i < 4; i++) { \
|
||||
int32x4_t v_sad = vec_splat_s32(0); \
|
||||
PROCESS16_4D(y *ref_stride, ref_array[i], v_ah1, v_al1); \
|
||||
PROCESS16_4D(y *ref_stride + 16, ref_array[i], v_ah2, v_al2); \
|
||||
PROCESS16_4D(y *ref_stride + 32, ref_array[i], v_ah3, v_al3); \
|
||||
PROCESS16_4D(y *ref_stride + 48, ref_array[i], v_ah4, v_al4); \
|
||||
\
|
||||
vec_vsx_st((uint32x4_t)v_sad, 0, sad); \
|
||||
sad_array[i] += (sad[3] + sad[2] + sad[1] + sad[0]); \
|
||||
} \
|
||||
} \
|
||||
}
|
||||
|
||||
SAD16_4D(8);
|
||||
SAD16_4D(16);
|
||||
SAD16_4D(32);
|
||||
SAD32_4D(16);
|
||||
SAD32_4D(32);
|
||||
SAD32_4D(64);
|
||||
SAD64_4D(32);
|
||||
SAD64_4D(64);
|
||||
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
static VPX_FORCE_INLINE void subtract_block4x4(
|
||||
int16_t *diff, ptrdiff_t diff_stride, const uint8_t *src,
|
||||
ptrdiff_t src_stride, const uint8_t *pred, ptrdiff_t pred_stride) {
|
||||
int16_t *diff1 = diff + 2 * diff_stride;
|
||||
const uint8_t *src1 = src + 2 * src_stride;
|
||||
const uint8_t *pred1 = pred + 2 * pred_stride;
|
||||
|
||||
const int16x8_t d0 = vec_vsx_ld(0, diff);
|
||||
const int16x8_t d1 = vec_vsx_ld(0, diff + diff_stride);
|
||||
const int16x8_t d2 = vec_vsx_ld(0, diff1);
|
||||
const int16x8_t d3 = vec_vsx_ld(0, diff1 + diff_stride);
|
||||
|
||||
const uint8x16_t s0 = read4x2(src, (int)src_stride);
|
||||
const uint8x16_t p0 = read4x2(pred, (int)pred_stride);
|
||||
const uint8x16_t s1 = read4x2(src1, (int)src_stride);
|
||||
const uint8x16_t p1 = read4x2(pred1, (int)pred_stride);
|
||||
|
||||
const int16x8_t da = vec_sub(unpack_to_s16_h(s0), unpack_to_s16_h(p0));
|
||||
const int16x8_t db = vec_sub(unpack_to_s16_h(s1), unpack_to_s16_h(p1));
|
||||
|
||||
vec_vsx_st(xxpermdi(da, d0, 1), 0, diff);
|
||||
vec_vsx_st(xxpermdi(da, d1, 3), 0, diff + diff_stride);
|
||||
vec_vsx_st(xxpermdi(db, d2, 1), 0, diff1);
|
||||
vec_vsx_st(xxpermdi(db, d3, 3), 0, diff1 + diff_stride);
|
||||
}
|
||||
|
||||
void vpx_subtract_block_vsx(int rows, int cols, int16_t *diff,
|
||||
ptrdiff_t diff_stride, const uint8_t *src,
|
||||
ptrdiff_t src_stride, const uint8_t *pred,
|
||||
ptrdiff_t pred_stride) {
|
||||
int r = rows, c;
|
||||
|
||||
switch (cols) {
|
||||
case 64:
|
||||
case 32:
|
||||
do {
|
||||
for (c = 0; c < cols; c += 32) {
|
||||
const uint8x16_t s0 = vec_vsx_ld(0, src + c);
|
||||
const uint8x16_t s1 = vec_vsx_ld(16, src + c);
|
||||
const uint8x16_t p0 = vec_vsx_ld(0, pred + c);
|
||||
const uint8x16_t p1 = vec_vsx_ld(16, pred + c);
|
||||
const int16x8_t d0l =
|
||||
vec_sub(unpack_to_s16_l(s0), unpack_to_s16_l(p0));
|
||||
const int16x8_t d0h =
|
||||
vec_sub(unpack_to_s16_h(s0), unpack_to_s16_h(p0));
|
||||
const int16x8_t d1l =
|
||||
vec_sub(unpack_to_s16_l(s1), unpack_to_s16_l(p1));
|
||||
const int16x8_t d1h =
|
||||
vec_sub(unpack_to_s16_h(s1), unpack_to_s16_h(p1));
|
||||
vec_vsx_st(d0h, 0, diff + c);
|
||||
vec_vsx_st(d0l, 16, diff + c);
|
||||
vec_vsx_st(d1h, 0, diff + c + 16);
|
||||
vec_vsx_st(d1l, 16, diff + c + 16);
|
||||
}
|
||||
diff += diff_stride;
|
||||
pred += pred_stride;
|
||||
src += src_stride;
|
||||
} while (--r);
|
||||
break;
|
||||
case 16:
|
||||
do {
|
||||
const uint8x16_t s0 = vec_vsx_ld(0, src);
|
||||
const uint8x16_t p0 = vec_vsx_ld(0, pred);
|
||||
const int16x8_t d0l = vec_sub(unpack_to_s16_l(s0), unpack_to_s16_l(p0));
|
||||
const int16x8_t d0h = vec_sub(unpack_to_s16_h(s0), unpack_to_s16_h(p0));
|
||||
vec_vsx_st(d0h, 0, diff);
|
||||
vec_vsx_st(d0l, 16, diff);
|
||||
diff += diff_stride;
|
||||
pred += pred_stride;
|
||||
src += src_stride;
|
||||
} while (--r);
|
||||
break;
|
||||
case 8:
|
||||
do {
|
||||
const uint8x16_t s0 = vec_vsx_ld(0, src);
|
||||
const uint8x16_t p0 = vec_vsx_ld(0, pred);
|
||||
const int16x8_t d0h = vec_sub(unpack_to_s16_h(s0), unpack_to_s16_h(p0));
|
||||
vec_vsx_st(d0h, 0, diff);
|
||||
diff += diff_stride;
|
||||
pred += pred_stride;
|
||||
src += src_stride;
|
||||
} while (--r);
|
||||
break;
|
||||
case 4:
|
||||
subtract_block4x4(diff, diff_stride, src, src_stride, pred, pred_stride);
|
||||
if (r > 4) {
|
||||
diff += 4 * diff_stride;
|
||||
pred += 4 * pred_stride;
|
||||
src += 4 * src_stride;
|
||||
|
||||
subtract_block4x4(diff, diff_stride,
|
||||
|
||||
src, src_stride,
|
||||
|
||||
pred, pred_stride);
|
||||
}
|
||||
break;
|
||||
default: assert(0); // unreachable
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_PPC_TRANSPOSE_VSX_H_
|
||||
#define VPX_VPX_DSP_PPC_TRANSPOSE_VSX_H_
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
static INLINE void vpx_transpose_s16_8x8(int16x8_t v[8]) {
|
||||
// d = vec_mergeh(a,b):
|
||||
// The even elements of the result are obtained left-to-right,
|
||||
// from the high elements of a.
|
||||
// The odd elements of the result are obtained left-to-right,
|
||||
// from the high elements of b.
|
||||
//
|
||||
// d = vec_mergel(a,b):
|
||||
// The even elements of the result are obtained left-to-right,
|
||||
// from the low elements of a.
|
||||
// The odd elements of the result are obtained left-to-right,
|
||||
// from the low elements of b.
|
||||
|
||||
// Example, starting with:
|
||||
// v[0]: 00 01 02 03 04 05 06 07
|
||||
// v[1]: 10 11 12 13 14 15 16 17
|
||||
// v[2]: 20 21 22 23 24 25 26 27
|
||||
// v[3]: 30 31 32 33 34 35 36 37
|
||||
// v[4]: 40 41 42 43 44 45 46 47
|
||||
// v[5]: 50 51 52 53 54 55 56 57
|
||||
// v[6]: 60 61 62 63 64 65 66 67
|
||||
// v[7]: 70 71 72 73 74 75 76 77
|
||||
|
||||
int16x8_t b0, b1, b2, b3, b4, b5, b6, b7;
|
||||
int16x8_t c0, c1, c2, c3, c4, c5, c6, c7;
|
||||
|
||||
b0 = vec_mergeh(v[0], v[4]);
|
||||
b1 = vec_mergel(v[0], v[4]);
|
||||
b2 = vec_mergeh(v[1], v[5]);
|
||||
b3 = vec_mergel(v[1], v[5]);
|
||||
b4 = vec_mergeh(v[2], v[6]);
|
||||
b5 = vec_mergel(v[2], v[6]);
|
||||
b6 = vec_mergeh(v[3], v[7]);
|
||||
b7 = vec_mergel(v[3], v[7]);
|
||||
|
||||
// After first merge operation
|
||||
// b0: 00 40 01 41 02 42 03 43
|
||||
// b1: 04 44 05 45 06 46 07 47
|
||||
// b2: 10 50 11 51 12 52 13 53
|
||||
// b3: 14 54 15 55 16 56 17 57
|
||||
// b4: 20 60 21 61 22 62 23 63
|
||||
// b5: 24 64 25 65 26 66 27 67
|
||||
// b6: 30 70 31 71 32 62 33 73
|
||||
// b7: 34 74 35 75 36 76 37 77
|
||||
|
||||
c0 = vec_mergeh(b0, b4);
|
||||
c1 = vec_mergel(b0, b4);
|
||||
c2 = vec_mergeh(b1, b5);
|
||||
c3 = vec_mergel(b1, b5);
|
||||
c4 = vec_mergeh(b2, b6);
|
||||
c5 = vec_mergel(b2, b6);
|
||||
c6 = vec_mergeh(b3, b7);
|
||||
c7 = vec_mergel(b3, b7);
|
||||
|
||||
// After second merge operation
|
||||
// c0: 00 20 40 60 01 21 41 61
|
||||
// c1: 02 22 42 62 03 23 43 63
|
||||
// c2: 04 24 44 64 05 25 45 65
|
||||
// c3: 06 26 46 66 07 27 47 67
|
||||
// c4: 10 30 50 70 11 31 51 71
|
||||
// c5: 12 32 52 72 13 33 53 73
|
||||
// c6: 14 34 54 74 15 35 55 75
|
||||
// c7: 16 36 56 76 17 37 57 77
|
||||
|
||||
v[0] = vec_mergeh(c0, c4);
|
||||
v[1] = vec_mergel(c0, c4);
|
||||
v[2] = vec_mergeh(c1, c5);
|
||||
v[3] = vec_mergel(c1, c5);
|
||||
v[4] = vec_mergeh(c2, c6);
|
||||
v[5] = vec_mergel(c2, c6);
|
||||
v[6] = vec_mergeh(c3, c7);
|
||||
v[7] = vec_mergel(c3, c7);
|
||||
|
||||
// After last merge operation
|
||||
// v[0]: 00 10 20 30 40 50 60 70
|
||||
// v[1]: 01 11 21 31 41 51 61 71
|
||||
// v[2]: 02 12 22 32 42 52 62 72
|
||||
// v[3]: 03 13 23 33 43 53 63 73
|
||||
// v[4]: 04 14 24 34 44 54 64 74
|
||||
// v[5]: 05 15 25 35 45 55 65 75
|
||||
// v[6]: 06 16 26 36 46 56 66 76
|
||||
// v[7]: 07 17 27 37 47 57 67 77
|
||||
}
|
||||
|
||||
static INLINE void transpose_8x8(const int16x8_t *a, int16x8_t *b) {
|
||||
// Stage 1
|
||||
const int16x8_t s1_0 = vec_mergeh(a[0], a[4]);
|
||||
const int16x8_t s1_1 = vec_mergel(a[0], a[4]);
|
||||
const int16x8_t s1_2 = vec_mergeh(a[1], a[5]);
|
||||
const int16x8_t s1_3 = vec_mergel(a[1], a[5]);
|
||||
const int16x8_t s1_4 = vec_mergeh(a[2], a[6]);
|
||||
const int16x8_t s1_5 = vec_mergel(a[2], a[6]);
|
||||
const int16x8_t s1_6 = vec_mergeh(a[3], a[7]);
|
||||
const int16x8_t s1_7 = vec_mergel(a[3], a[7]);
|
||||
|
||||
// Stage 2
|
||||
const int16x8_t s2_0 = vec_mergeh(s1_0, s1_4);
|
||||
const int16x8_t s2_1 = vec_mergel(s1_0, s1_4);
|
||||
const int16x8_t s2_2 = vec_mergeh(s1_1, s1_5);
|
||||
const int16x8_t s2_3 = vec_mergel(s1_1, s1_5);
|
||||
const int16x8_t s2_4 = vec_mergeh(s1_2, s1_6);
|
||||
const int16x8_t s2_5 = vec_mergel(s1_2, s1_6);
|
||||
const int16x8_t s2_6 = vec_mergeh(s1_3, s1_7);
|
||||
const int16x8_t s2_7 = vec_mergel(s1_3, s1_7);
|
||||
|
||||
// Stage 2
|
||||
b[0] = vec_mergeh(s2_0, s2_4);
|
||||
b[1] = vec_mergel(s2_0, s2_4);
|
||||
b[2] = vec_mergeh(s2_1, s2_5);
|
||||
b[3] = vec_mergel(s2_1, s2_5);
|
||||
b[4] = vec_mergeh(s2_2, s2_6);
|
||||
b[5] = vec_mergel(s2_2, s2_6);
|
||||
b[6] = vec_mergeh(s2_3, s2_7);
|
||||
b[7] = vec_mergel(s2_3, s2_7);
|
||||
}
|
||||
|
||||
#endif // VPX_VPX_DSP_PPC_TRANSPOSE_VSX_H_
|
||||
@@ -0,0 +1,90 @@
|
||||
/*
|
||||
* Copyright (c) 2018 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_PPC_TXFM_COMMON_VSX_H_
|
||||
#define VPX_VPX_DSP_PPC_TXFM_COMMON_VSX_H_
|
||||
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
static const int32x4_t vec_dct_const_rounding = { 8192, 8192, 8192, 8192 };
|
||||
|
||||
static const uint32x4_t vec_dct_const_bits = { 14, 14, 14, 14 };
|
||||
|
||||
static const uint16x8_t vec_dct_scale_log2 = { 2, 2, 2, 2, 2, 2, 2, 2 };
|
||||
|
||||
static const int16x8_t cospi1_v = { 16364, 16364, 16364, 16364,
|
||||
16364, 16364, 16364, 16364 };
|
||||
static const int16x8_t cospi2_v = { 16305, 16305, 16305, 16305,
|
||||
16305, 16305, 16305, 16305 };
|
||||
static const int16x8_t cospi3_v = { 16207, 16207, 16207, 16207,
|
||||
16207, 16207, 16207, 16207 };
|
||||
static const int16x8_t cospi4_v = { 16069, 16069, 16069, 16069,
|
||||
16069, 16069, 16069, 16069 };
|
||||
static const int16x8_t cospi4m_v = { -16069, -16069, -16069, -16069,
|
||||
-16069, -16069, -16069, -16069 };
|
||||
static const int16x8_t cospi5_v = { 15893, 15893, 15893, 15893,
|
||||
15893, 15893, 15893, 15893 };
|
||||
static const int16x8_t cospi6_v = { 15679, 15679, 15679, 15679,
|
||||
15679, 15679, 15679, 15679 };
|
||||
static const int16x8_t cospi7_v = { 15426, 15426, 15426, 15426,
|
||||
15426, 15426, 15426, 15426 };
|
||||
static const int16x8_t cospi8_v = { 15137, 15137, 15137, 15137,
|
||||
15137, 15137, 15137, 15137 };
|
||||
static const int16x8_t cospi8m_v = { -15137, -15137, -15137, -15137,
|
||||
-15137, -15137, -15137, -15137 };
|
||||
static const int16x8_t cospi9_v = { 14811, 14811, 14811, 14811,
|
||||
14811, 14811, 14811, 14811 };
|
||||
static const int16x8_t cospi10_v = { 14449, 14449, 14449, 14449,
|
||||
14449, 14449, 14449, 14449 };
|
||||
static const int16x8_t cospi11_v = { 14053, 14053, 14053, 14053,
|
||||
14053, 14053, 14053, 14053 };
|
||||
static const int16x8_t cospi12_v = { 13623, 13623, 13623, 13623,
|
||||
13623, 13623, 13623, 13623 };
|
||||
static const int16x8_t cospi13_v = { 13160, 13160, 13160, 13160,
|
||||
13160, 13160, 13160, 13160 };
|
||||
static const int16x8_t cospi14_v = { 12665, 12665, 12665, 12665,
|
||||
12665, 12665, 12665, 12665 };
|
||||
static const int16x8_t cospi15_v = { 12140, 12140, 12140, 12140,
|
||||
12140, 12140, 12140, 12140 };
|
||||
static const int16x8_t cospi16_v = { 11585, 11585, 11585, 11585,
|
||||
11585, 11585, 11585, 11585 };
|
||||
static const int16x8_t cospi17_v = { 11003, 11003, 11003, 11003,
|
||||
11003, 11003, 11003, 11003 };
|
||||
static const int16x8_t cospi18_v = { 10394, 10394, 10394, 10394,
|
||||
10394, 10394, 10394, 10394 };
|
||||
static const int16x8_t cospi19_v = { 9760, 9760, 9760, 9760,
|
||||
9760, 9760, 9760, 9760 };
|
||||
static const int16x8_t cospi20_v = { 9102, 9102, 9102, 9102,
|
||||
9102, 9102, 9102, 9102 };
|
||||
static const int16x8_t cospi20m_v = { -9102, -9102, -9102, -9102,
|
||||
-9102, -9102, -9102, -9102 };
|
||||
static const int16x8_t cospi21_v = { 8423, 8423, 8423, 8423,
|
||||
8423, 8423, 8423, 8423 };
|
||||
static const int16x8_t cospi22_v = { 7723, 7723, 7723, 7723,
|
||||
7723, 7723, 7723, 7723 };
|
||||
static const int16x8_t cospi23_v = { 7005, 7005, 7005, 7005,
|
||||
7005, 7005, 7005, 7005 };
|
||||
static const int16x8_t cospi24_v = { 6270, 6270, 6270, 6270,
|
||||
6270, 6270, 6270, 6270 };
|
||||
static const int16x8_t cospi25_v = { 5520, 5520, 5520, 5520,
|
||||
5520, 5520, 5520, 5520 };
|
||||
static const int16x8_t cospi26_v = { 4756, 4756, 4756, 4756,
|
||||
4756, 4756, 4756, 4756 };
|
||||
static const int16x8_t cospi27_v = { 3981, 3981, 3981, 3981,
|
||||
3981, 3981, 3981, 3981 };
|
||||
static const int16x8_t cospi28_v = { 3196, 3196, 3196, 3196,
|
||||
3196, 3196, 3196, 3196 };
|
||||
static const int16x8_t cospi29_v = { 2404, 2404, 2404, 2404,
|
||||
2404, 2404, 2404, 2404 };
|
||||
static const int16x8_t cospi30_v = { 1606, 1606, 1606, 1606,
|
||||
1606, 1606, 1606, 1606 };
|
||||
static const int16x8_t cospi31_v = { 804, 804, 804, 804, 804, 804, 804, 804 };
|
||||
|
||||
#endif // VPX_VPX_DSP_PPC_TXFM_COMMON_VSX_H_
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_PPC_TYPES_VSX_H_
|
||||
#define VPX_VPX_DSP_PPC_TYPES_VSX_H_
|
||||
|
||||
#include <altivec.h>
|
||||
|
||||
typedef vector signed char int8x16_t;
|
||||
typedef vector unsigned char uint8x16_t;
|
||||
typedef vector signed short int16x8_t;
|
||||
typedef vector unsigned short uint16x8_t;
|
||||
typedef vector signed int int32x4_t;
|
||||
typedef vector unsigned int uint32x4_t;
|
||||
typedef vector bool char bool8x16_t;
|
||||
typedef vector bool short bool16x8_t;
|
||||
typedef vector bool int bool32x4_t;
|
||||
|
||||
#if defined(__clang__) && __clang_major__ < 6
|
||||
static const uint8x16_t xxpermdi0_perm = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
|
||||
0x06, 0x07, 0x10, 0x11, 0x12, 0x13,
|
||||
0x14, 0x15, 0x16, 0x17 };
|
||||
static const uint8x16_t xxpermdi1_perm = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
|
||||
0x06, 0x07, 0x18, 0x19, 0x1A, 0x1B,
|
||||
0x1C, 0x1D, 0x1E, 0x1F };
|
||||
static const uint8x16_t xxpermdi2_perm = { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
|
||||
0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13,
|
||||
0x14, 0x15, 0x16, 0x17 };
|
||||
static const uint8x16_t xxpermdi3_perm = { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
|
||||
0x0E, 0x0F, 0x18, 0x19, 0x1A, 0x1B,
|
||||
0x1C, 0x1D, 0x1E, 0x1F };
|
||||
#define xxpermdi(a, b, c) vec_perm(a, b, xxpermdi##c##_perm)
|
||||
#elif defined(__GNUC__) && \
|
||||
(__GNUC__ > 6 || (__GNUC__ == 6 && __GNUC_MINOR__ >= 3))
|
||||
#define xxpermdi(a, b, c) vec_xxpermdi(a, b, c)
|
||||
#endif
|
||||
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
#define unpack_to_u16_h(v) \
|
||||
(uint16x8_t) vec_mergeh(vec_splat_u8(0), (uint8x16_t)v)
|
||||
#define unpack_to_u16_l(v) \
|
||||
(uint16x8_t) vec_mergel(vec_splat_u8(0), (uint8x16_t)v)
|
||||
#define unpack_to_s16_h(v) \
|
||||
(int16x8_t) vec_mergeh(vec_splat_u8(0), (uint8x16_t)v)
|
||||
#define unpack_to_s16_l(v) \
|
||||
(int16x8_t) vec_mergel(vec_splat_u8(0), (uint8x16_t)v)
|
||||
#ifndef xxpermdi
|
||||
#define xxpermdi(a, b, c) vec_xxpermdi(a, b, c)
|
||||
#endif
|
||||
#else
|
||||
#define unpack_to_u16_h(v) \
|
||||
(uint16x8_t) vec_mergeh((uint8x16_t)v, vec_splat_u8(0))
|
||||
#define unpack_to_u16_l(v) \
|
||||
(uint16x8_t) vec_mergel((uint8x16_t)v, vec_splat_u8(0))
|
||||
#define unpack_to_s16_h(v) \
|
||||
(int16x8_t) vec_mergeh((uint8x16_t)v, vec_splat_u8(0))
|
||||
#define unpack_to_s16_l(v) \
|
||||
(int16x8_t) vec_mergel((uint8x16_t)v, vec_splat_u8(0))
|
||||
#ifndef xxpermdi
|
||||
#define xxpermdi(a, b, c) vec_xxpermdi(b, a, (((c) >> 1) | ((c)&1) << 1) ^ 3)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
static INLINE uint8x16_t read4x2(const uint8_t *a, int stride) {
|
||||
const uint32x4_t a0 = (uint32x4_t)vec_vsx_ld(0, a);
|
||||
const uint32x4_t a1 = (uint32x4_t)vec_vsx_ld(0, a + stride);
|
||||
|
||||
return (uint8x16_t)vec_mergeh(a0, a1);
|
||||
}
|
||||
|
||||
#ifndef __POWER9_VECTOR__
|
||||
#define vec_absd(a, b) vec_sub(vec_max(a, b), vec_min(a, b))
|
||||
#endif
|
||||
|
||||
static const uint8x16_t vec_zeros_u8 = { 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0 };
|
||||
static const int16x8_t vec_zeros_s16 = { 0, 0, 0, 0, 0, 0, 0, 0 };
|
||||
static const int16x8_t vec_ones_s16 = { 1, 1, 1, 1, 1, 1, 1, 1 };
|
||||
static const int16x8_t vec_twos_s16 = { 2, 2, 2, 2, 2, 2, 2, 2 };
|
||||
static const uint16x8_t vec_ones_u16 = { 1, 1, 1, 1, 1, 1, 1, 1 };
|
||||
static const uint32x4_t vec_ones_u32 = { 1, 1, 1, 1 };
|
||||
static const int32x4_t vec_zeros_s32 = { 0, 0, 0, 0 };
|
||||
static const uint32x4_t vec_zeros_u32 = { 0, 0, 0, 0 };
|
||||
static const uint16x8_t vec_shift_sign_s16 = { 15, 15, 15, 15, 15, 15, 15, 15 };
|
||||
static const uint32x4_t vec_shift_sign_s32 = { 31, 31, 31, 31 };
|
||||
static const uint8x16_t vec_perm64 = { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
|
||||
0x0E, 0x0F, 0x00, 0x01, 0x02, 0x03,
|
||||
0x04, 0x05, 0x06, 0x07 };
|
||||
static const uint8x16_t vec_perm32 = { 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
|
||||
0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
|
||||
0x00, 0x01, 0x02, 0x03 };
|
||||
static const uint8x16_t vec_perm16 = { 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
||||
0x08, 0x09, 0x0A, 0x0B, 0x0E, 0x0D,
|
||||
0x0E, 0x0F, 0x00, 0x01 };
|
||||
|
||||
static const uint8x16_t vec_perm_odd_even_pack = { 0x00, 0x01, 0x10, 0x11,
|
||||
0x04, 0x05, 0x14, 0x15,
|
||||
0x08, 0x09, 0x18, 0x19,
|
||||
0x0C, 0x0D, 0x1C, 0x1D };
|
||||
|
||||
#endif // VPX_VPX_DSP_PPC_TYPES_VSX_H_
|
||||
@@ -0,0 +1,271 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
|
||||
uint32_t vpx_get4x4sse_cs_vsx(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride) {
|
||||
int distortion;
|
||||
|
||||
const int16x8_t a0 = unpack_to_s16_h(read4x2(src_ptr, src_stride));
|
||||
const int16x8_t a1 =
|
||||
unpack_to_s16_h(read4x2(src_ptr + src_stride * 2, src_stride));
|
||||
const int16x8_t b0 = unpack_to_s16_h(read4x2(ref_ptr, ref_stride));
|
||||
const int16x8_t b1 =
|
||||
unpack_to_s16_h(read4x2(ref_ptr + ref_stride * 2, ref_stride));
|
||||
const int16x8_t d0 = vec_sub(a0, b0);
|
||||
const int16x8_t d1 = vec_sub(a1, b1);
|
||||
const int32x4_t ds = vec_msum(d1, d1, vec_msum(d0, d0, vec_splat_s32(0)));
|
||||
const int32x4_t d = vec_splat(vec_sums(ds, vec_splat_s32(0)), 3);
|
||||
|
||||
vec_ste(d, 0, &distortion);
|
||||
|
||||
return distortion;
|
||||
}
|
||||
|
||||
// TODO(lu_zero): Unroll
|
||||
uint32_t vpx_get_mb_ss_vsx(const int16_t *src_ptr) {
|
||||
unsigned int i, sum = 0;
|
||||
int32x4_t s = vec_splat_s32(0);
|
||||
|
||||
for (i = 0; i < 256; i += 8) {
|
||||
const int16x8_t v = vec_vsx_ld(0, src_ptr + i);
|
||||
s = vec_msum(v, v, s);
|
||||
}
|
||||
|
||||
s = vec_splat(vec_sums(s, vec_splat_s32(0)), 3);
|
||||
|
||||
vec_ste((uint32x4_t)s, 0, &sum);
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
void vpx_comp_avg_pred_vsx(uint8_t *comp_pred, const uint8_t *pred, int width,
|
||||
int height, const uint8_t *ref, int ref_stride) {
|
||||
int i, j;
|
||||
/* comp_pred and pred must be 16 byte aligned. */
|
||||
assert(((intptr_t)comp_pred & 0xf) == 0);
|
||||
assert(((intptr_t)pred & 0xf) == 0);
|
||||
if (width >= 16) {
|
||||
for (i = 0; i < height; ++i) {
|
||||
for (j = 0; j < width; j += 16) {
|
||||
const uint8x16_t v = vec_avg(vec_vsx_ld(j, pred), vec_vsx_ld(j, ref));
|
||||
vec_vsx_st(v, j, comp_pred);
|
||||
}
|
||||
comp_pred += width;
|
||||
pred += width;
|
||||
ref += ref_stride;
|
||||
}
|
||||
} else if (width == 8) {
|
||||
// Process 2 lines at time
|
||||
for (i = 0; i < height / 2; ++i) {
|
||||
const uint8x16_t r0 = vec_vsx_ld(0, ref);
|
||||
const uint8x16_t r1 = vec_vsx_ld(0, ref + ref_stride);
|
||||
const uint8x16_t r = xxpermdi(r0, r1, 0);
|
||||
const uint8x16_t v = vec_avg(vec_vsx_ld(0, pred), r);
|
||||
vec_vsx_st(v, 0, comp_pred);
|
||||
comp_pred += 16; // width * 2;
|
||||
pred += 16; // width * 2;
|
||||
ref += ref_stride * 2;
|
||||
}
|
||||
} else {
|
||||
assert(width == 4);
|
||||
// process 4 lines at time
|
||||
for (i = 0; i < height / 4; ++i) {
|
||||
const uint32x4_t r0 = (uint32x4_t)vec_vsx_ld(0, ref);
|
||||
const uint32x4_t r1 = (uint32x4_t)vec_vsx_ld(0, ref + ref_stride);
|
||||
const uint32x4_t r2 = (uint32x4_t)vec_vsx_ld(0, ref + ref_stride * 2);
|
||||
const uint32x4_t r3 = (uint32x4_t)vec_vsx_ld(0, ref + ref_stride * 3);
|
||||
const uint8x16_t r =
|
||||
(uint8x16_t)xxpermdi(vec_mergeh(r0, r1), vec_mergeh(r2, r3), 0);
|
||||
const uint8x16_t v = vec_avg(vec_vsx_ld(0, pred), r);
|
||||
vec_vsx_st(v, 0, comp_pred);
|
||||
comp_pred += 16; // width * 4;
|
||||
pred += 16; // width * 4;
|
||||
ref += ref_stride * 4;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void variance_inner_32(const uint8_t *src_ptr,
|
||||
const uint8_t *ref_ptr,
|
||||
int32x4_t *sum_squared, int32x4_t *sum) {
|
||||
int32x4_t s = *sum;
|
||||
int32x4_t ss = *sum_squared;
|
||||
|
||||
const uint8x16_t va0 = vec_vsx_ld(0, src_ptr);
|
||||
const uint8x16_t vb0 = vec_vsx_ld(0, ref_ptr);
|
||||
const uint8x16_t va1 = vec_vsx_ld(16, src_ptr);
|
||||
const uint8x16_t vb1 = vec_vsx_ld(16, ref_ptr);
|
||||
|
||||
const int16x8_t a0 = unpack_to_s16_h(va0);
|
||||
const int16x8_t b0 = unpack_to_s16_h(vb0);
|
||||
const int16x8_t a1 = unpack_to_s16_l(va0);
|
||||
const int16x8_t b1 = unpack_to_s16_l(vb0);
|
||||
const int16x8_t a2 = unpack_to_s16_h(va1);
|
||||
const int16x8_t b2 = unpack_to_s16_h(vb1);
|
||||
const int16x8_t a3 = unpack_to_s16_l(va1);
|
||||
const int16x8_t b3 = unpack_to_s16_l(vb1);
|
||||
const int16x8_t d0 = vec_sub(a0, b0);
|
||||
const int16x8_t d1 = vec_sub(a1, b1);
|
||||
const int16x8_t d2 = vec_sub(a2, b2);
|
||||
const int16x8_t d3 = vec_sub(a3, b3);
|
||||
|
||||
s = vec_sum4s(d0, s);
|
||||
ss = vec_msum(d0, d0, ss);
|
||||
s = vec_sum4s(d1, s);
|
||||
ss = vec_msum(d1, d1, ss);
|
||||
s = vec_sum4s(d2, s);
|
||||
ss = vec_msum(d2, d2, ss);
|
||||
s = vec_sum4s(d3, s);
|
||||
ss = vec_msum(d3, d3, ss);
|
||||
*sum = s;
|
||||
*sum_squared = ss;
|
||||
}
|
||||
|
||||
static INLINE void variance(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride, int w,
|
||||
int h, uint32_t *sse, int *sum) {
|
||||
int i;
|
||||
|
||||
int32x4_t s = vec_splat_s32(0);
|
||||
int32x4_t ss = vec_splat_s32(0);
|
||||
|
||||
switch (w) {
|
||||
case 4:
|
||||
for (i = 0; i < h / 2; ++i) {
|
||||
const int16x8_t a0 = unpack_to_s16_h(read4x2(src_ptr, src_stride));
|
||||
const int16x8_t b0 = unpack_to_s16_h(read4x2(ref_ptr, ref_stride));
|
||||
const int16x8_t d = vec_sub(a0, b0);
|
||||
s = vec_sum4s(d, s);
|
||||
ss = vec_msum(d, d, ss);
|
||||
src_ptr += src_stride * 2;
|
||||
ref_ptr += ref_stride * 2;
|
||||
}
|
||||
break;
|
||||
case 8:
|
||||
for (i = 0; i < h; ++i) {
|
||||
const int16x8_t a0 = unpack_to_s16_h(vec_vsx_ld(0, src_ptr));
|
||||
const int16x8_t b0 = unpack_to_s16_h(vec_vsx_ld(0, ref_ptr));
|
||||
const int16x8_t d = vec_sub(a0, b0);
|
||||
|
||||
s = vec_sum4s(d, s);
|
||||
ss = vec_msum(d, d, ss);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
}
|
||||
break;
|
||||
case 16:
|
||||
for (i = 0; i < h; ++i) {
|
||||
const uint8x16_t va = vec_vsx_ld(0, src_ptr);
|
||||
const uint8x16_t vb = vec_vsx_ld(0, ref_ptr);
|
||||
const int16x8_t a0 = unpack_to_s16_h(va);
|
||||
const int16x8_t b0 = unpack_to_s16_h(vb);
|
||||
const int16x8_t a1 = unpack_to_s16_l(va);
|
||||
const int16x8_t b1 = unpack_to_s16_l(vb);
|
||||
const int16x8_t d0 = vec_sub(a0, b0);
|
||||
const int16x8_t d1 = vec_sub(a1, b1);
|
||||
|
||||
s = vec_sum4s(d0, s);
|
||||
ss = vec_msum(d0, d0, ss);
|
||||
s = vec_sum4s(d1, s);
|
||||
ss = vec_msum(d1, d1, ss);
|
||||
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
}
|
||||
break;
|
||||
case 32:
|
||||
for (i = 0; i < h; ++i) {
|
||||
variance_inner_32(src_ptr, ref_ptr, &ss, &s);
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
}
|
||||
break;
|
||||
case 64:
|
||||
for (i = 0; i < h; ++i) {
|
||||
variance_inner_32(src_ptr, ref_ptr, &ss, &s);
|
||||
variance_inner_32(src_ptr + 32, ref_ptr + 32, &ss, &s);
|
||||
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
s = vec_splat(vec_sums(s, vec_splat_s32(0)), 3);
|
||||
|
||||
vec_ste(s, 0, sum);
|
||||
|
||||
ss = vec_splat(vec_sums(ss, vec_splat_s32(0)), 3);
|
||||
|
||||
vec_ste((uint32x4_t)ss, 0, sse);
|
||||
}
|
||||
|
||||
/* Identical to the variance call except it takes an additional parameter, sum,
|
||||
* and returns that value using pass-by-reference instead of returning
|
||||
* sse - sum^2 / w*h
|
||||
*/
|
||||
#define GET_VAR(W, H) \
|
||||
void vpx_get##W##x##H##var_vsx(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride, \
|
||||
uint32_t *sse, int *sum) { \
|
||||
variance(src_ptr, src_stride, ref_ptr, ref_stride, W, H, sse, sum); \
|
||||
}
|
||||
|
||||
/* Identical to the variance call except it does not calculate the
|
||||
* sse - sum^2 / w*h and returns sse in addtion to modifying the passed in
|
||||
* variable.
|
||||
*/
|
||||
#define MSE(W, H) \
|
||||
uint32_t vpx_mse##W##x##H##_vsx(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride, \
|
||||
uint32_t *sse) { \
|
||||
int sum; \
|
||||
variance(src_ptr, src_stride, ref_ptr, ref_stride, W, H, sse, &sum); \
|
||||
return *sse; \
|
||||
}
|
||||
|
||||
#define VAR(W, H) \
|
||||
uint32_t vpx_variance##W##x##H##_vsx(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride, \
|
||||
uint32_t *sse) { \
|
||||
int sum; \
|
||||
variance(src_ptr, src_stride, ref_ptr, ref_stride, W, H, sse, &sum); \
|
||||
return *sse - (uint32_t)(((int64_t)sum * sum) / ((W) * (H))); \
|
||||
}
|
||||
|
||||
#define VARIANCES(W, H) VAR(W, H)
|
||||
|
||||
VARIANCES(64, 64)
|
||||
VARIANCES(64, 32)
|
||||
VARIANCES(32, 64)
|
||||
VARIANCES(32, 32)
|
||||
VARIANCES(32, 16)
|
||||
VARIANCES(16, 32)
|
||||
VARIANCES(16, 16)
|
||||
VARIANCES(16, 8)
|
||||
VARIANCES(8, 16)
|
||||
VARIANCES(8, 8)
|
||||
VARIANCES(8, 4)
|
||||
VARIANCES(4, 8)
|
||||
VARIANCES(4, 4)
|
||||
|
||||
GET_VAR(16, 16)
|
||||
GET_VAR(8, 8)
|
||||
|
||||
MSE(16, 16)
|
||||
MSE(16, 8)
|
||||
MSE(8, 16)
|
||||
MSE(8, 8)
|
||||
@@ -0,0 +1,408 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/ppc/types_vsx.h"
|
||||
#include "vpx_dsp/vpx_filter.h"
|
||||
|
||||
// TODO(lu_zero): unroll
|
||||
static VPX_FORCE_INLINE void copy_w16(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
int32_t h) {
|
||||
int i;
|
||||
|
||||
for (i = h; i--;) {
|
||||
vec_vsx_st(vec_vsx_ld(0, src), 0, dst);
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void copy_w32(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
int32_t h) {
|
||||
int i;
|
||||
|
||||
for (i = h; i--;) {
|
||||
vec_vsx_st(vec_vsx_ld(0, src), 0, dst);
|
||||
vec_vsx_st(vec_vsx_ld(16, src), 16, dst);
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void copy_w64(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
int32_t h) {
|
||||
int i;
|
||||
|
||||
for (i = h; i--;) {
|
||||
vec_vsx_st(vec_vsx_ld(0, src), 0, dst);
|
||||
vec_vsx_st(vec_vsx_ld(16, src), 16, dst);
|
||||
vec_vsx_st(vec_vsx_ld(32, src), 32, dst);
|
||||
vec_vsx_st(vec_vsx_ld(48, src), 48, dst);
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_convolve_copy_vsx(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4, int x_step_q4,
|
||||
int y0_q4, int32_t y_step_q4, int32_t w, int32_t h) {
|
||||
(void)filter;
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
switch (w) {
|
||||
case 16: {
|
||||
copy_w16(src, src_stride, dst, dst_stride, h);
|
||||
break;
|
||||
}
|
||||
case 32: {
|
||||
copy_w32(src, src_stride, dst, dst_stride, h);
|
||||
break;
|
||||
}
|
||||
case 64: {
|
||||
copy_w64(src, src_stride, dst, dst_stride, h);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
int i;
|
||||
for (i = h; i--;) {
|
||||
memcpy(dst, src, w);
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void avg_w16(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
int32_t h) {
|
||||
int i;
|
||||
|
||||
for (i = h; i--;) {
|
||||
const uint8x16_t v = vec_avg(vec_vsx_ld(0, src), vec_vsx_ld(0, dst));
|
||||
vec_vsx_st(v, 0, dst);
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void avg_w32(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
int32_t h) {
|
||||
int i;
|
||||
|
||||
for (i = h; i--;) {
|
||||
const uint8x16_t v0 = vec_avg(vec_vsx_ld(0, src), vec_vsx_ld(0, dst));
|
||||
const uint8x16_t v1 = vec_avg(vec_vsx_ld(16, src), vec_vsx_ld(16, dst));
|
||||
vec_vsx_st(v0, 0, dst);
|
||||
vec_vsx_st(v1, 16, dst);
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void avg_w64(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
int32_t h) {
|
||||
int i;
|
||||
|
||||
for (i = h; i--;) {
|
||||
const uint8x16_t v0 = vec_avg(vec_vsx_ld(0, src), vec_vsx_ld(0, dst));
|
||||
const uint8x16_t v1 = vec_avg(vec_vsx_ld(16, src), vec_vsx_ld(16, dst));
|
||||
const uint8x16_t v2 = vec_avg(vec_vsx_ld(32, src), vec_vsx_ld(32, dst));
|
||||
const uint8x16_t v3 = vec_avg(vec_vsx_ld(48, src), vec_vsx_ld(48, dst));
|
||||
vec_vsx_st(v0, 0, dst);
|
||||
vec_vsx_st(v1, 16, dst);
|
||||
vec_vsx_st(v2, 32, dst);
|
||||
vec_vsx_st(v3, 48, dst);
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_convolve_avg_vsx(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4, int x_step_q4,
|
||||
int y0_q4, int32_t y_step_q4, int32_t w, int32_t h) {
|
||||
switch (w) {
|
||||
case 16: {
|
||||
avg_w16(src, src_stride, dst, dst_stride, h);
|
||||
break;
|
||||
}
|
||||
case 32: {
|
||||
avg_w32(src, src_stride, dst, dst_stride, h);
|
||||
break;
|
||||
}
|
||||
case 64: {
|
||||
avg_w64(src, src_stride, dst, dst_stride, h);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
vpx_convolve_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4,
|
||||
x_step_q4, y0_q4, y_step_q4, w, h);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void convolve_line(uint8_t *dst, const int16x8_t s,
|
||||
const int16x8_t f) {
|
||||
const int32x4_t sum = vec_msum(s, f, vec_splat_s32(0));
|
||||
const int32x4_t bias =
|
||||
vec_sl(vec_splat_s32(1), vec_splat_u32(FILTER_BITS - 1));
|
||||
const int32x4_t avg = vec_sr(vec_sums(sum, bias), vec_splat_u32(FILTER_BITS));
|
||||
const uint8x16_t v = vec_splat(
|
||||
vec_packsu(vec_pack(avg, vec_splat_s32(0)), vec_splat_s16(0)), 3);
|
||||
vec_ste(v, 0, dst);
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void convolve_line_h(uint8_t *dst,
|
||||
const uint8_t *const src_x,
|
||||
const int16_t *const x_filter) {
|
||||
const int16x8_t s = unpack_to_s16_h(vec_vsx_ld(0, src_x));
|
||||
const int16x8_t f = vec_vsx_ld(0, x_filter);
|
||||
|
||||
convolve_line(dst, s, f);
|
||||
}
|
||||
|
||||
// TODO(lu_zero): Implement 8x8 and bigger block special cases
|
||||
static VPX_FORCE_INLINE void convolve_horiz(const uint8_t *src,
|
||||
ptrdiff_t src_stride, uint8_t *dst,
|
||||
ptrdiff_t dst_stride,
|
||||
const InterpKernel *x_filters,
|
||||
int x0_q4, int x_step_q4, int w,
|
||||
int h) {
|
||||
int x, y;
|
||||
src -= SUBPEL_TAPS / 2 - 1;
|
||||
|
||||
for (y = 0; y < h; ++y) {
|
||||
int x_q4 = x0_q4;
|
||||
for (x = 0; x < w; ++x) {
|
||||
convolve_line_h(dst + x, &src[x_q4 >> SUBPEL_BITS],
|
||||
x_filters[x_q4 & SUBPEL_MASK]);
|
||||
x_q4 += x_step_q4;
|
||||
}
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void convolve_avg_horiz(
|
||||
const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
|
||||
ptrdiff_t dst_stride, const InterpKernel *x_filters, int x0_q4,
|
||||
int x_step_q4, int w, int h) {
|
||||
int x, y;
|
||||
src -= SUBPEL_TAPS / 2 - 1;
|
||||
|
||||
for (y = 0; y < h; ++y) {
|
||||
int x_q4 = x0_q4;
|
||||
for (x = 0; x < w; ++x) {
|
||||
uint8_t v;
|
||||
convolve_line_h(&v, &src[x_q4 >> SUBPEL_BITS],
|
||||
x_filters[x_q4 & SUBPEL_MASK]);
|
||||
dst[x] = ROUND_POWER_OF_TWO(dst[x] + v, 1);
|
||||
x_q4 += x_step_q4;
|
||||
}
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static uint8x16_t transpose_line_u8_8x8(uint8x16_t a, uint8x16_t b,
|
||||
uint8x16_t c, uint8x16_t d,
|
||||
uint8x16_t e, uint8x16_t f,
|
||||
uint8x16_t g, uint8x16_t h) {
|
||||
uint16x8_t ab = (uint16x8_t)vec_mergeh(a, b);
|
||||
uint16x8_t cd = (uint16x8_t)vec_mergeh(c, d);
|
||||
uint16x8_t ef = (uint16x8_t)vec_mergeh(e, f);
|
||||
uint16x8_t gh = (uint16x8_t)vec_mergeh(g, h);
|
||||
|
||||
uint32x4_t abcd = (uint32x4_t)vec_mergeh(ab, cd);
|
||||
uint32x4_t efgh = (uint32x4_t)vec_mergeh(ef, gh);
|
||||
|
||||
return (uint8x16_t)vec_mergeh(abcd, efgh);
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void convolve_line_v(uint8_t *dst,
|
||||
const uint8_t *const src_y,
|
||||
ptrdiff_t src_stride,
|
||||
const int16_t *const y_filter) {
|
||||
uint8x16_t s0 = vec_vsx_ld(0, src_y + 0 * src_stride);
|
||||
uint8x16_t s1 = vec_vsx_ld(0, src_y + 1 * src_stride);
|
||||
uint8x16_t s2 = vec_vsx_ld(0, src_y + 2 * src_stride);
|
||||
uint8x16_t s3 = vec_vsx_ld(0, src_y + 3 * src_stride);
|
||||
uint8x16_t s4 = vec_vsx_ld(0, src_y + 4 * src_stride);
|
||||
uint8x16_t s5 = vec_vsx_ld(0, src_y + 5 * src_stride);
|
||||
uint8x16_t s6 = vec_vsx_ld(0, src_y + 6 * src_stride);
|
||||
uint8x16_t s7 = vec_vsx_ld(0, src_y + 7 * src_stride);
|
||||
const int16x8_t f = vec_vsx_ld(0, y_filter);
|
||||
uint8_t buf[16];
|
||||
const uint8x16_t s = transpose_line_u8_8x8(s0, s1, s2, s3, s4, s5, s6, s7);
|
||||
|
||||
vec_vsx_st(s, 0, buf);
|
||||
|
||||
convolve_line(dst, unpack_to_s16_h(s), f);
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void convolve_vert(const uint8_t *src,
|
||||
ptrdiff_t src_stride, uint8_t *dst,
|
||||
ptrdiff_t dst_stride,
|
||||
const InterpKernel *y_filters,
|
||||
int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
int x, y;
|
||||
src -= src_stride * (SUBPEL_TAPS / 2 - 1);
|
||||
|
||||
for (x = 0; x < w; ++x) {
|
||||
int y_q4 = y0_q4;
|
||||
for (y = 0; y < h; ++y) {
|
||||
convolve_line_v(dst + y * dst_stride,
|
||||
&src[(y_q4 >> SUBPEL_BITS) * src_stride], src_stride,
|
||||
y_filters[y_q4 & SUBPEL_MASK]);
|
||||
y_q4 += y_step_q4;
|
||||
}
|
||||
++src;
|
||||
++dst;
|
||||
}
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void convolve_avg_vert(
|
||||
const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
|
||||
ptrdiff_t dst_stride, const InterpKernel *y_filters, int y0_q4,
|
||||
int y_step_q4, int w, int h) {
|
||||
int x, y;
|
||||
src -= src_stride * (SUBPEL_TAPS / 2 - 1);
|
||||
|
||||
for (x = 0; x < w; ++x) {
|
||||
int y_q4 = y0_q4;
|
||||
for (y = 0; y < h; ++y) {
|
||||
uint8_t v;
|
||||
convolve_line_v(&v, &src[(y_q4 >> SUBPEL_BITS) * src_stride], src_stride,
|
||||
y_filters[y_q4 & SUBPEL_MASK]);
|
||||
dst[y * dst_stride] = ROUND_POWER_OF_TWO(dst[y * dst_stride] + v, 1);
|
||||
y_q4 += y_step_q4;
|
||||
}
|
||||
++src;
|
||||
++dst;
|
||||
}
|
||||
}
|
||||
|
||||
static VPX_FORCE_INLINE void convolve(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *const filter,
|
||||
int x0_q4, int x_step_q4, int y0_q4,
|
||||
int y_step_q4, int w, int h) {
|
||||
// Note: Fixed size intermediate buffer, temp, places limits on parameters.
|
||||
// 2d filtering proceeds in 2 steps:
|
||||
// (1) Interpolate horizontally into an intermediate buffer, temp.
|
||||
// (2) Interpolate temp vertically to derive the sub-pixel result.
|
||||
// Deriving the maximum number of rows in the temp buffer (135):
|
||||
// --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
|
||||
// --Largest block size is 64x64 pixels.
|
||||
// --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
|
||||
// original frame (in 1/16th pixel units).
|
||||
// --Must round-up because block may be located at sub-pixel position.
|
||||
// --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
|
||||
// --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
|
||||
DECLARE_ALIGNED(16, uint8_t, temp[64 * 135]);
|
||||
const int intermediate_height =
|
||||
(((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
|
||||
|
||||
assert(w <= 64);
|
||||
assert(h <= 64);
|
||||
assert(y_step_q4 <= 32);
|
||||
assert(x_step_q4 <= 32);
|
||||
|
||||
convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride, temp, 64,
|
||||
filter, x0_q4, x_step_q4, w, intermediate_height);
|
||||
convolve_vert(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst, dst_stride, filter,
|
||||
y0_q4, y_step_q4, w, h);
|
||||
}
|
||||
|
||||
void vpx_convolve8_horiz_vsx(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
convolve_horiz(src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, w,
|
||||
h);
|
||||
}
|
||||
|
||||
void vpx_convolve8_avg_horiz_vsx(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
convolve_avg_horiz(src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4,
|
||||
w, h);
|
||||
}
|
||||
|
||||
void vpx_convolve8_vert_vsx(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
|
||||
convolve_vert(src, src_stride, dst, dst_stride, filter, y0_q4, y_step_q4, w,
|
||||
h);
|
||||
}
|
||||
|
||||
void vpx_convolve8_avg_vert_vsx(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
|
||||
convolve_avg_vert(src, src_stride, dst, dst_stride, filter, y0_q4, y_step_q4,
|
||||
w, h);
|
||||
}
|
||||
|
||||
void vpx_convolve8_vsx(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
|
||||
ptrdiff_t dst_stride, const InterpKernel *filter,
|
||||
int x0_q4, int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h) {
|
||||
convolve(src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, y0_q4,
|
||||
y_step_q4, w, h);
|
||||
}
|
||||
|
||||
void vpx_convolve8_avg_vsx(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4, int x_step_q4,
|
||||
int y0_q4, int y_step_q4, int w, int h) {
|
||||
// Fixed size intermediate buffer places limits on parameters.
|
||||
DECLARE_ALIGNED(16, uint8_t, temp[64 * 64]);
|
||||
assert(w <= 64);
|
||||
assert(h <= 64);
|
||||
|
||||
vpx_convolve8_vsx(src, src_stride, temp, 64, filter, x0_q4, x_step_q4, y0_q4,
|
||||
y_step_q4, w, h);
|
||||
vpx_convolve_avg_vsx(temp, 64, dst, dst_stride, NULL, 0, 0, 0, 0, w, h);
|
||||
}
|
||||
Reference in New Issue
Block a user