From: "Swinney, Jonathan" <jswinney@amazon.com>
To: "ffmpeg-devel@ffmpeg.org" <ffmpeg-devel@ffmpeg.org>
Cc: "Martin Storsjö" <martin@martin.st>, "Pop, Sebastian" <spop@amazon.com>
Subject: [FFmpeg-devel] [PATCH 2/2] swscale/aarch64: add vscale specializations
Date: Fri, 15 Apr 2022 21:37:06 +0000
Message-ID: <fb9f751291d84d36bc66b0c65028e741@EX13D07UWB004.ant.amazon.com> (raw)
This commit adds new code paths for vscale when filterSize is 2, 4, or 8. By
using specialized code with unrolling to match the filterSize we can improve
performance.
| (seconds) | c6g | | |
| ------------| ----- | ----- | ----- |
| filterSize | 2 | 4 | 8 |
| original | 0.581 | 0.974 | 1.744 |
| optimized | 0.399 | 0.569 | 1.052 |
| improvement | 31.1% | 41.6% | 39.7% |
Signed-off-by: Jonathan Swinney <jswinney@amazon.com>
---
libswscale/aarch64/output.S | 147 +++++++++++++++++++++++++++++++++--
libswscale/aarch64/swscale.c | 12 +++
2 files changed, 153 insertions(+), 6 deletions(-)
diff --git a/libswscale/aarch64/output.S b/libswscale/aarch64/output.S
index af71de6050..9c99c3bea9 100644
--- a/libswscale/aarch64/output.S
+++ b/libswscale/aarch64/output.S
@@ -21,12 +21,27 @@
#include "libavutil/aarch64/asm.S"
function ff_yuv2planeX_8_neon, export=1
+// x0 - const int16_t *filter,
+// x1 - int filterSize,
+// x2 - const int16_t **src,
+// x3 - uint8_t *dest,
+// x4 - int dstW,
+// x5 - const uint8_t *dither,
+// x6 - int offset
+
ld1 {v0.8B}, [x5] // load 8x8-bit dither
cbz w6, 1f // check if offsetting present
ext v0.8B, v0.8B, v0.8B, #3 // honor offsetting which can be 0 or 3 only
1: uxtl v0.8H, v0.8B // extend dither to 16-bit
ushll v1.4S, v0.4H, #12 // extend dither to 32-bit with left shift by 12 (part 1)
ushll2 v2.4S, v0.8H, #12 // extend dither to 32-bit with left shift by 12 (part 2)
+ cmp w1, #8 // if filterSize == 8, branch to specialized version
+ b.eq 5f
+ cmp w1, #4 // if filterSize == 4, branch to specialized version
+ b.eq 7f
+ cmp w1, #2 // if filterSize == 2, branch to specialized version
+ b.eq 9f
+
mov x7, #0 // i = 0
2: mov v3.16B, v1.16B // initialize accumulator part 1 with dithering value
mov v4.16B, v2.16B // initialize accumulator part 2 with dithering value
@@ -34,16 +49,15 @@ function ff_yuv2planeX_8_neon, export=1
mov x9, x2 // srcp = src
mov x10, x0 // filterp = filter
3: ldp x11, x12, [x9], #16 // get 2 pointers: src[j] and src[j+1]
+ ld2r {v16.8H, v17.8H}, [x10], #4 // read 2x16-bit coeff X and Y at filter[j] and filter[j+1]
add x11, x11, x7, lsl #1 // &src[j ][i]
add x12, x12, x7, lsl #1 // &src[j+1][i]
ld1 {v5.8H}, [x11] // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
ld1 {v6.8H}, [x12] // read 8x16-bit @ src[j+1][i + {0..7}]: I,J,K,L,M,N,O,P
- ld1r {v7.8H}, [x10], #2 // read 1x16-bit coeff X at filter[j ] and duplicate across lanes
- ld1r {v16.8H}, [x10], #2 // read 1x16-bit coeff Y at filter[j+1] and duplicate across lanes
- smlal v3.4S, v5.4H, v7.4H // val0 += {A,B,C,D} * X
- smlal2 v4.4S, v5.8H, v7.8H // val1 += {E,F,G,H} * X
- smlal v3.4S, v6.4H, v16.4H // val0 += {I,J,K,L} * Y
- smlal2 v4.4S, v6.8H, v16.8H // val1 += {M,N,O,P} * Y
+ smlal v3.4S, v5.4H, v16.4H // val0 += {A,B,C,D} * X
+ smlal2 v4.4S, v5.8H, v16.8H // val1 += {E,F,G,H} * X
+ smlal v3.4S, v6.4H, v17.4H // val0 += {I,J,K,L} * Y
+ smlal2 v4.4S, v6.8H, v17.8H // val1 += {M,N,O,P} * Y
subs w8, w8, #2 // tmpfilterSize -= 2
b.gt 3b // loop until filterSize consumed
@@ -55,4 +69,125 @@ function ff_yuv2planeX_8_neon, export=1
add x7, x7, #8 // i += 8
b.gt 2b // loop until width consumed
ret
+
+5: // fs=8
+ ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
+ ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]
+ ldp x10, x11, [x2, #32] // load 2 pointers: src[j+4] and src[j+5]
+ ldp x12, x13, [x2, #48] // load 2 pointers: src[j+6] and src[j+7]
+
+ // load 8x16-bit values for filter[j], where j=0..7 and replicated across lanes
+ ld4r {v16.8H, v17.8H, v18.8H, v19.8H}, [x0], #8
+ ld4r {v20.8H, v21.8H, v22.8H, v23.8H}, [x0]
+6:
+ mov v3.16B, v1.16B // initialize accumulator part 1 with dithering value
+ mov v4.16B, v2.16B // initialize accumulator part 2 with dithering value
+
+ ld1 {v24.8H}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
+ ld1 {v25.8H}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
+ ld1 {v26.8H}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]
+ ld1 {v27.8H}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]
+ ld1 {v28.8H}, [x10], #16 // load 8x16-bit values for src[j + 4][i + {0..7}]
+ ld1 {v29.8H}, [x11], #16 // load 8x16-bit values for src[j + 5][i + {0..7}]
+ ld1 {v30.8H}, [x12], #16 // load 8x16-bit values for src[j + 6][i + {0..7}]
+ ld1 {v31.8H}, [x13], #16 // load 8x16-bit values for src[j + 7][i + {0..7}]
+
+ smlal v3.4S, v24.4H, v16.4H // val0 += src[0][i + {0..3}] * filter[0]
+ smlal2 v4.4S, v24.8H, v16.8H // val1 += src[0][i + {4..7}] * filter[0]
+ smlal v3.4S, v25.4H, v17.4H // val0 += src[1][i + {0..3}] * filter[1]
+ smlal2 v4.4S, v25.8H, v17.8H // val1 += src[1][i + {4..7}] * filter[1]
+ smlal v3.4S, v26.4H, v18.4H // val0 += src[2][i + {0..3}] * filter[2]
+ smlal2 v4.4S, v26.8H, v18.8H // val1 += src[2][i + {4..7}] * filter[2]
+ smlal v3.4S, v27.4H, v19.4H // val0 += src[3][i + {0..3}] * filter[3]
+ smlal2 v4.4S, v27.8H, v19.8H // val1 += src[3][i + {4..7}] * filter[3]
+ smlal v3.4S, v28.4H, v20.4H // val0 += src[4][i + {0..3}] * filter[4]
+ smlal2 v4.4S, v28.8H, v20.8H // val1 += src[4][i + {4..7}] * filter[4]
+ smlal v3.4S, v29.4H, v21.4H // val0 += src[5][i + {0..3}] * filter[5]
+ smlal2 v4.4S, v29.8H, v21.8H // val1 += src[5][i + {4..7}] * filter[5]
+ smlal v3.4S, v30.4H, v22.4H // val0 += src[6][i + {0..3}] * filter[6]
+ smlal2 v4.4S, v30.8H, v22.8H // val1 += src[6][i + {4..7}] * filter[6]
+ smlal v3.4S, v31.4H, v23.4H // val0 += src[7][i + {0..3}] * filter[7]
+ smlal2 v4.4S, v31.8H, v23.8H // val1 += src[7][i + {4..7}] * filter[7]
+
+ sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
+ sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
+ uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
+ st1 {v3.8b}, [x3], #8 // write to destination
+ subs w4, w4, #8 // dstW -= 8
+ b.gt 6b // loop until width consumed
+ ret
+
+7: // fs=4
+ ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
+ ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]
+
+ // load 4x16-bit values for filter[j], where j=0..3 and replicated across lanes
+ ld4r {v16.8H, v17.8H, v18.8H, v19.8H}, [x0]
+8:
+ mov v3.16B, v1.16B // initialize accumulator part 1 with dithering value
+ mov v4.16B, v2.16B // initialize accumulator part 2 with dithering value
+
+ ld1 {v24.8H}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
+ ld1 {v25.8H}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
+ ld1 {v26.8H}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]
+ ld1 {v27.8H}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]
+
+ smlal v3.4S, v24.4H, v16.4H // val0 += src[0][i + {0..3}] * filter[0]
+ smlal2 v4.4S, v24.8H, v16.8H // val1 += src[0][i + {4..7}] * filter[0]
+ smlal v3.4S, v25.4H, v17.4H // val0 += src[1][i + {0..3}] * filter[1]
+ smlal2 v4.4S, v25.8H, v17.8H // val1 += src[1][i + {4..7}] * filter[1]
+ smlal v3.4S, v26.4H, v18.4H // val0 += src[2][i + {0..3}] * filter[2]
+ smlal2 v4.4S, v26.8H, v18.8H // val1 += src[2][i + {4..7}] * filter[2]
+ smlal v3.4S, v27.4H, v19.4H // val0 += src[3][i + {0..3}] * filter[3]
+ smlal2 v4.4S, v27.8H, v19.8H // val1 += src[3][i + {4..7}] * filter[3]
+
+ sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
+ sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
+ uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
+ st1 {v3.8b}, [x3], #8 // write to destination
+ subs w4, w4, #8 // dstW -= 8
+ b.gt 8b // loop until width consumed
+ ret
+
+9: // fs=2
+ ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
+
+ // load 2x16-bit values for filter[j], where j=0..1 and replicated across lanes
+ ld2r {v16.8H, v17.8H}, [x0]
+10:
+ mov v3.16B, v1.16B // initialize accumulator part 1 with dithering value
+ mov v4.16B, v2.16B // initialize accumulator part 2 with dithering value
+
+ ld1 {v24.8H}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
+ ld1 {v25.8H}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
+
+ smlal v3.4S, v24.4H, v16.4H // val0 += src[0][i + {0..3}] * filter[0]
+ smlal2 v4.4S, v24.8H, v16.8H // val1 += src[0][i + {4..7}] * filter[0]
+ smlal v3.4S, v25.4H, v17.4H // val0 += src[1][i + {0..3}] * filter[1]
+ smlal2 v4.4S, v25.8H, v17.8H // val1 += src[1][i + {4..7}] * filter[1]
+
+ sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
+ sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
+ uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
+ st1 {v3.8b}, [x3], #8 // write to destination
+ subs w4, w4, #8 // dstW -= 8
+ b.gt 10b // loop until width consumed
+ ret
+endfunc
+function ff_yuv2plane1_8_neon, export=1
+ ld1 {v0.8B}, [x5] // load 8x8-bit dither
+ cbz w6, 1f // check if offsetting present
+ ext v0.8B, v0.8B, v0.8B, #3 // honor offsetting which can be 0 or 3 only
+1: uxtl v0.8H, v0.8B // extend dither to 16-bit
+
+
+2:
+ ld1 {v5.8H}, [x0], #16 // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
+ add v3.8H, v0.8H, v5.8H
+
+ uqshrn v3.8b, v3.8h, #7 // clip8(val>>7)
+ st1 {v3.8b}, [x1], #8 // write to destination
+ subs w2, w2, #8 // dstW -= 8
+ b.gt 2b // loop until width consumed
+ ret
endfunc
diff --git a/libswscale/aarch64/swscale.c b/libswscale/aarch64/swscale.c
index 2ea4ccb3a6..0cc821bf11 100644
--- a/libswscale/aarch64/swscale.c
+++ b/libswscale/aarch64/swscale.c
@@ -40,6 +40,12 @@ ALL_SCALE_FUNCS(neon);
void ff_yuv2planeX_8_neon(const int16_t *filter, int filterSize,
const int16_t **src, uint8_t *dest, int dstW,
const uint8_t *dither, int offset);
+void ff_yuv2plane1_8_neon(
+ const int16_t *src,
+ uint8_t *dest,
+ int dstW,
+ const uint8_t *dither,
+ int offset);
#define ASSIGN_SCALE_FUNC2(hscalefn, filtersize, opt) do { \
if (c->srcBpc == 8 && c->dstBpc <= 14) { \
@@ -56,6 +62,11 @@ void ff_yuv2planeX_8_neon(const int16_t *filter, int filterSize,
ASSIGN_SCALE_FUNC2(hscalefn, X8, opt); \
break; \
}
+#define ASSIGN_VSCALE_FUNC(vscalefn, opt1) \
+ switch(c->dstBpc){ \
+ case 8: vscalefn = ff_yuv2plane1_8_ ## opt1; break; \
+ default: break; \
+ }
av_cold void ff_sws_init_swscale_aarch64(SwsContext *c)
{
@@ -64,6 +75,7 @@ av_cold void ff_sws_init_swscale_aarch64(SwsContext *c)
if (have_neon(cpu_flags)) {
ASSIGN_SCALE_FUNC(c->hyScale, c->hLumFilterSize, neon);
ASSIGN_SCALE_FUNC(c->hcScale, c->hChrFilterSize, neon);
+ ASSIGN_VSCALE_FUNC(c->yuv2plane1, neon);
if (c->dstBpc == 8) {
c->yuv2planeX = ff_yuv2planeX_8_neon;
}
--
2.32.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
next reply other threads:[~2022-04-15 21:37 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-04-15 21:37 Swinney, Jonathan [this message]
2022-04-16 21:32 ` Martin Storsjö
2022-04-19 21:12 ` Martin Storsjö
2022-06-13 16:36 Swinney, Jonathan
2022-06-21 20:23 ` Martin Storsjö
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=fb9f751291d84d36bc66b0c65028e741@EX13D07UWB004.ant.amazon.com \
--to=jswinney@amazon.com \
--cc=ffmpeg-devel@ffmpeg.org \
--cc=martin@martin.st \
--cc=spop@amazon.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git