From: Ben Avison <bavison@riscosopen.org>
To: ffmpeg-devel@ffmpeg.org
Cc: Ben Avison <bavison@riscosopen.org>
Subject: [FFmpeg-devel] [PATCH v3 09/10] avcodec/vc1: Arm 64-bit NEON unescape fast path
Date: Thu, 31 Mar 2022 18:23:50 +0100
Message-ID: <20220331172351.550818-10-bavison@riscosopen.org> (raw)
In-Reply-To: <20220331172351.550818-1-bavison@riscosopen.org>
checkasm benchmarks on 1.5 GHz Cortex-A72 are as follows.
vc1dsp.vc1_unescape_buffer_c: 655617.7
vc1dsp.vc1_unescape_buffer_neon: 118237.0
Signed-off-by: Ben Avison <bavison@riscosopen.org>
---
libavcodec/aarch64/vc1dsp_init_aarch64.c | 61 ++++++++
libavcodec/aarch64/vc1dsp_neon.S | 176 +++++++++++++++++++++++
2 files changed, 237 insertions(+)
diff --git a/libavcodec/aarch64/vc1dsp_init_aarch64.c b/libavcodec/aarch64/vc1dsp_init_aarch64.c
index e0eb52dd63..a7976fd596 100644
--- a/libavcodec/aarch64/vc1dsp_init_aarch64.c
+++ b/libavcodec/aarch64/vc1dsp_init_aarch64.c
@@ -21,6 +21,7 @@
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
+#include "libavutil/intreadwrite.h"
#include "libavcodec/vc1dsp.h"
#include "config.h"
@@ -51,6 +52,64 @@ void ff_put_vc1_chroma_mc4_neon(uint8_t *dst, uint8_t *src, ptrdiff_t stride,
void ff_avg_vc1_chroma_mc4_neon(uint8_t *dst, uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
+int ff_vc1_unescape_buffer_helper_neon(const uint8_t *src, int size, uint8_t *dst);
+
+static int vc1_unescape_buffer_neon(const uint8_t *src, int size, uint8_t *dst)
+{
+ /* Dealing with starting and stopping, and removing escape bytes, are
+ * comparatively less time-sensitive, so are more clearly expressed using
+ * a C wrapper around the assembly inner loop. Note that we assume a
+ * little-endian machine that supports unaligned loads. */
+ int dsize = 0;
+ while (size >= 4)
+ {
+ int found = 0;
+ while (!found && (((uintptr_t) dst) & 7) && size >= 4)
+ {
+ found = (AV_RL32(src) &~ 0x03000000) == 0x00030000;
+ if (!found)
+ {
+ *dst++ = *src++;
+ --size;
+ ++dsize;
+ }
+ }
+ if (!found)
+ {
+ int skip = size - ff_vc1_unescape_buffer_helper_neon(src, size, dst);
+ dst += skip;
+ src += skip;
+ size -= skip;
+ dsize += skip;
+ while (!found && size >= 4)
+ {
+ found = (AV_RL32(src) &~ 0x03000000) == 0x00030000;
+ if (!found)
+ {
+ *dst++ = *src++;
+ --size;
+ ++dsize;
+ }
+ }
+ }
+ if (found)
+ {
+ *dst++ = *src++;
+ *dst++ = *src++;
+ ++src;
+ size -= 3;
+ dsize += 2;
+ }
+ }
+ while (size > 0)
+ {
+ *dst++ = *src++;
+ --size;
+ ++dsize;
+ }
+ return dsize;
+}
+
av_cold void ff_vc1dsp_init_aarch64(VC1DSPContext *dsp)
{
int cpu_flags = av_get_cpu_flags();
@@ -76,5 +135,7 @@ av_cold void ff_vc1dsp_init_aarch64(VC1DSPContext *dsp)
dsp->avg_no_rnd_vc1_chroma_pixels_tab[0] = ff_avg_vc1_chroma_mc8_neon;
dsp->put_no_rnd_vc1_chroma_pixels_tab[1] = ff_put_vc1_chroma_mc4_neon;
dsp->avg_no_rnd_vc1_chroma_pixels_tab[1] = ff_avg_vc1_chroma_mc4_neon;
+
+ dsp->vc1_unescape_buffer = vc1_unescape_buffer_neon;
}
}
diff --git a/libavcodec/aarch64/vc1dsp_neon.S b/libavcodec/aarch64/vc1dsp_neon.S
index 0201db4f78..9a96c2523c 100644
--- a/libavcodec/aarch64/vc1dsp_neon.S
+++ b/libavcodec/aarch64/vc1dsp_neon.S
@@ -1368,3 +1368,179 @@ function ff_vc1_h_loop_filter16_neon, export=1
st2 {v2.b, v3.b}[7], [x6]
4: ret
endfunc
+
+// Copy at most the specified number of bytes from source to destination buffer,
+// stopping at a multiple of 32 bytes, none of which are the start of an escape sequence
+// On entry:
+// x0 -> source buffer
+// w1 = max number of bytes to copy
+// x2 -> destination buffer, optimally 8-byte aligned
+// On exit:
+// w0 = number of bytes not copied
+function ff_vc1_unescape_buffer_helper_neon, export=1
+ // Offset by 80 to screen out cases that are too short for us to handle,
+ // and also make it easy to test for loop termination, or to determine
+ // whether we need an odd number of half-iterations of the loop.
+ subs w1, w1, #80
+ b.mi 90f
+
+ // Set up useful constants
+ movi v20.4s, #3, lsl #24
+ movi v21.4s, #3, lsl #16
+
+ tst w1, #32
+ b.ne 1f
+
+ ld1 {v0.16b, v1.16b, v2.16b}, [x0], #48
+ ext v25.16b, v0.16b, v1.16b, #1
+ ext v26.16b, v0.16b, v1.16b, #2
+ ext v27.16b, v0.16b, v1.16b, #3
+ ext v29.16b, v1.16b, v2.16b, #1
+ ext v30.16b, v1.16b, v2.16b, #2
+ ext v31.16b, v1.16b, v2.16b, #3
+ bic v24.16b, v0.16b, v20.16b
+ bic v25.16b, v25.16b, v20.16b
+ bic v26.16b, v26.16b, v20.16b
+ bic v27.16b, v27.16b, v20.16b
+ bic v28.16b, v1.16b, v20.16b
+ bic v29.16b, v29.16b, v20.16b
+ bic v30.16b, v30.16b, v20.16b
+ bic v31.16b, v31.16b, v20.16b
+ eor v24.16b, v24.16b, v21.16b
+ eor v25.16b, v25.16b, v21.16b
+ eor v26.16b, v26.16b, v21.16b
+ eor v27.16b, v27.16b, v21.16b
+ eor v28.16b, v28.16b, v21.16b
+ eor v29.16b, v29.16b, v21.16b
+ eor v30.16b, v30.16b, v21.16b
+ eor v31.16b, v31.16b, v21.16b
+ cmeq v24.4s, v24.4s, #0
+ cmeq v25.4s, v25.4s, #0
+ cmeq v26.4s, v26.4s, #0
+ cmeq v27.4s, v27.4s, #0
+ add w1, w1, #32
+ b 3f
+
+1: ld1 {v3.16b, v4.16b, v5.16b}, [x0], #48
+ ext v25.16b, v3.16b, v4.16b, #1
+ ext v26.16b, v3.16b, v4.16b, #2
+ ext v27.16b, v3.16b, v4.16b, #3
+ ext v29.16b, v4.16b, v5.16b, #1
+ ext v30.16b, v4.16b, v5.16b, #2
+ ext v31.16b, v4.16b, v5.16b, #3
+ bic v24.16b, v3.16b, v20.16b
+ bic v25.16b, v25.16b, v20.16b
+ bic v26.16b, v26.16b, v20.16b
+ bic v27.16b, v27.16b, v20.16b
+ bic v28.16b, v4.16b, v20.16b
+ bic v29.16b, v29.16b, v20.16b
+ bic v30.16b, v30.16b, v20.16b
+ bic v31.16b, v31.16b, v20.16b
+ eor v24.16b, v24.16b, v21.16b
+ eor v25.16b, v25.16b, v21.16b
+ eor v26.16b, v26.16b, v21.16b
+ eor v27.16b, v27.16b, v21.16b
+ eor v28.16b, v28.16b, v21.16b
+ eor v29.16b, v29.16b, v21.16b
+ eor v30.16b, v30.16b, v21.16b
+ eor v31.16b, v31.16b, v21.16b
+ cmeq v24.4s, v24.4s, #0
+ cmeq v25.4s, v25.4s, #0
+ cmeq v26.4s, v26.4s, #0
+ cmeq v27.4s, v27.4s, #0
+ // Drop through...
+2: mov v0.16b, v5.16b
+ ld1 {v1.16b, v2.16b}, [x0], #32
+ cmeq v28.4s, v28.4s, #0
+ cmeq v29.4s, v29.4s, #0
+ cmeq v30.4s, v30.4s, #0
+ cmeq v31.4s, v31.4s, #0
+ orr v24.16b, v24.16b, v25.16b
+ orr v26.16b, v26.16b, v27.16b
+ orr v28.16b, v28.16b, v29.16b
+ orr v30.16b, v30.16b, v31.16b
+ ext v25.16b, v0.16b, v1.16b, #1
+ orr v22.16b, v24.16b, v26.16b
+ ext v26.16b, v0.16b, v1.16b, #2
+ ext v27.16b, v0.16b, v1.16b, #3
+ ext v29.16b, v1.16b, v2.16b, #1
+ orr v23.16b, v28.16b, v30.16b
+ ext v30.16b, v1.16b, v2.16b, #2
+ ext v31.16b, v1.16b, v2.16b, #3
+ bic v24.16b, v0.16b, v20.16b
+ bic v25.16b, v25.16b, v20.16b
+ bic v26.16b, v26.16b, v20.16b
+ orr v22.16b, v22.16b, v23.16b
+ bic v27.16b, v27.16b, v20.16b
+ bic v28.16b, v1.16b, v20.16b
+ bic v29.16b, v29.16b, v20.16b
+ bic v30.16b, v30.16b, v20.16b
+ bic v31.16b, v31.16b, v20.16b
+ addv s22, v22.4s
+ eor v24.16b, v24.16b, v21.16b
+ eor v25.16b, v25.16b, v21.16b
+ eor v26.16b, v26.16b, v21.16b
+ eor v27.16b, v27.16b, v21.16b
+ eor v28.16b, v28.16b, v21.16b
+ mov w3, v22.s[0]
+ eor v29.16b, v29.16b, v21.16b
+ eor v30.16b, v30.16b, v21.16b
+ eor v31.16b, v31.16b, v21.16b
+ cmeq v24.4s, v24.4s, #0
+ cmeq v25.4s, v25.4s, #0
+ cmeq v26.4s, v26.4s, #0
+ cmeq v27.4s, v27.4s, #0
+ cbnz w3, 90f
+ st1 {v3.16b, v4.16b}, [x2], #32
+3: mov v3.16b, v2.16b
+ ld1 {v4.16b, v5.16b}, [x0], #32
+ cmeq v28.4s, v28.4s, #0
+ cmeq v29.4s, v29.4s, #0
+ cmeq v30.4s, v30.4s, #0
+ cmeq v31.4s, v31.4s, #0
+ orr v24.16b, v24.16b, v25.16b
+ orr v26.16b, v26.16b, v27.16b
+ orr v28.16b, v28.16b, v29.16b
+ orr v30.16b, v30.16b, v31.16b
+ ext v25.16b, v3.16b, v4.16b, #1
+ orr v22.16b, v24.16b, v26.16b
+ ext v26.16b, v3.16b, v4.16b, #2
+ ext v27.16b, v3.16b, v4.16b, #3
+ ext v29.16b, v4.16b, v5.16b, #1
+ orr v23.16b, v28.16b, v30.16b
+ ext v30.16b, v4.16b, v5.16b, #2
+ ext v31.16b, v4.16b, v5.16b, #3
+ bic v24.16b, v3.16b, v20.16b
+ bic v25.16b, v25.16b, v20.16b
+ bic v26.16b, v26.16b, v20.16b
+ orr v22.16b, v22.16b, v23.16b
+ bic v27.16b, v27.16b, v20.16b
+ bic v28.16b, v4.16b, v20.16b
+ bic v29.16b, v29.16b, v20.16b
+ bic v30.16b, v30.16b, v20.16b
+ bic v31.16b, v31.16b, v20.16b
+ addv s22, v22.4s
+ eor v24.16b, v24.16b, v21.16b
+ eor v25.16b, v25.16b, v21.16b
+ eor v26.16b, v26.16b, v21.16b
+ eor v27.16b, v27.16b, v21.16b
+ eor v28.16b, v28.16b, v21.16b
+ mov w3, v22.s[0]
+ eor v29.16b, v29.16b, v21.16b
+ eor v30.16b, v30.16b, v21.16b
+ eor v31.16b, v31.16b, v21.16b
+ cmeq v24.4s, v24.4s, #0
+ cmeq v25.4s, v25.4s, #0
+ cmeq v26.4s, v26.4s, #0
+ cmeq v27.4s, v27.4s, #0
+ cbnz w3, 91f
+ st1 {v0.16b, v1.16b}, [x2], #32
+ subs w1, w1, #64
+ b.pl 2b
+
+90: add w0, w1, #80
+ ret
+
+91: sub w1, w1, #32
+ b 90b
+endfunc
--
2.25.1
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
next prev parent reply other threads:[~2022-03-31 17:25 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-03-31 17:23 [FFmpeg-devel] [PATCH v3 00/10] avcodec/vc1: Arm optimisations Ben Avison
2022-03-31 17:23 ` [FFmpeg-devel] [PATCH v3 01/10] checkasm: Add vc1dsp in-loop deblocking filter tests Ben Avison
2022-03-31 17:23 ` [FFmpeg-devel] [PATCH v3 02/10] checkasm: Add vc1dsp inverse transform tests Ben Avison
2022-03-31 17:23 ` [FFmpeg-devel] [PATCH v3 03/10] checkasm: Add idctdsp add/put-pixels-clamped tests Ben Avison
2022-03-31 17:23 ` [FFmpeg-devel] [PATCH v3 04/10] avcodec/vc1: Introduce fast path for unescaping bitstream buffer Ben Avison
2022-03-31 17:23 ` [FFmpeg-devel] [PATCH v3 05/10] avcodec/vc1: Arm 64-bit NEON deblocking filter fast paths Ben Avison
2022-03-31 17:23 ` [FFmpeg-devel] [PATCH v3 06/10] avcodec/vc1: Arm 32-bit " Ben Avison
2022-03-31 17:23 ` [FFmpeg-devel] [PATCH v3 07/10] avcodec/vc1: Arm 64-bit NEON inverse transform " Ben Avison
2022-03-31 17:23 ` [FFmpeg-devel] [PATCH v3 08/10] avcodec/idctdsp: Arm 64-bit NEON block add and clamp " Ben Avison
2022-03-31 17:23 ` Ben Avison [this message]
2022-03-31 17:23 ` [FFmpeg-devel] [PATCH v3 10/10] avcodec/vc1: Arm 32-bit NEON unescape fast path Ben Avison
2022-03-31 21:50 ` [FFmpeg-devel] [PATCH v3 00/10] avcodec/vc1: Arm optimisations Martin Storsjö
2022-04-01 7:08 ` Martin Storsjö
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220331172351.550818-10-bavison@riscosopen.org \
--to=bavison@riscosopen.org \
--cc=ffmpeg-devel@ffmpeg.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git