* [FFmpeg-devel] [PATCH v6 1/3] lavc/hashtable: create generic robin hood hash table
@ 2025-05-30 23:20 Emma Worley
2025-05-30 23:20 ` [FFmpeg-devel] [PATCH v6 2/3] lavc/dxvenc: migrate DXT1 encoder to lavc hashtable Emma Worley
2025-05-30 23:20 ` [FFmpeg-devel] [PATCH v6 3/3] lavc/dxvenc: improve compatibility with Resolume products Emma Worley
0 siblings, 2 replies; 3+ messages in thread
From: Emma Worley @ 2025-05-30 23:20 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Emma Worley
Adds a generic hash table with the DXV encoder as an initial use case.
Signed-off-by: Emma Worley <emma@emma.gg>
---
libavcodec/Makefile | 2 +
libavcodec/hashtable.c | 214 +++++++++++++++++++++++++++++++++++
libavcodec/hashtable.h | 94 +++++++++++++++
libavcodec/tests/hashtable.c | 110 ++++++++++++++++++
4 files changed, 420 insertions(+)
create mode 100644 libavcodec/hashtable.c
create mode 100644 libavcodec/hashtable.h
create mode 100644 libavcodec/tests/hashtable.c
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 77734dff24..1bb5d4f4a7 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -42,6 +42,7 @@ OBJS = ac3_parser.o \
dv_profile.o \
encode.o \
get_buffer.o \
+ hashtable.o \
imgconvert.o \
jni.o \
lcevcdec.o \
@@ -1325,6 +1326,7 @@ TESTPROGS = avcodec \
bitstream_le \
celp_math \
codec_desc \
+ hashtable \
htmlsubtitles \
jpeg2000dwt \
mathops \
diff --git a/libavcodec/hashtable.c b/libavcodec/hashtable.c
new file mode 100644
index 0000000000..151476176b
--- /dev/null
+++ b/libavcodec/hashtable.c
@@ -0,0 +1,214 @@
+/*
+ * Generic hashtable
+ * Copyright (C) 2025 Emma Worley <emma@emma.gg>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdint.h>
+#include <string.h>
+
+#include "libavutil/crc.h"
+#include "libavutil/error.h"
+#include "libavutil/mem.h"
+#include "hashtable.h"
+
+#define ALIGN _Alignof(size_t)
+
+struct FFHashtableContext {
+ size_t key_size;
+ size_t key_size_aligned;
+ size_t val_size;
+ size_t val_size_aligned;
+ size_t entry_size;
+ size_t max_entries;
+ size_t nb_entries;
+ const AVCRC *crc;
+ uint8_t *table;
+ uint8_t *swapbuf;
+};
+
+/*
+ * Hash table entries are comprised of a probe sequence length (PSL), key, and
+ * value. When the PSL of an entry is zero, it means it is not occupied by a
+ * key/value pair. When the PSL is non-zero, it represents the "distance" of
+ * the entry from its "home" location plus one, where the "home" location is
+ * hash(key) % max_entries.
+ */
+
+#define ENTRY_PSL_VAL(entry) (*(size_t*)(entry))
+#define ENTRY_KEY_PTR(entry) ((entry) + FFALIGN(sizeof(size_t), ALIGN))
+#define ENTRY_VAL_PTR(entry) (ENTRY_KEY_PTR(entry) + ctx->key_size_aligned)
+
+#define KEYS_EQUAL(k1, k2) (!memcmp((k1), (k2), ctx->key_size))
+
+int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries)
+{
+ struct FFHashtableContext *res = av_malloc(sizeof(struct FFHashtableContext));
+ if (!res)
+ return AVERROR(ENOMEM);
+ res->key_size = key_size;
+ res->key_size_aligned = FFALIGN(key_size, ALIGN);
+ res->val_size = val_size;
+ res->val_size_aligned = FFALIGN(val_size, ALIGN);
+ res->entry_size = FFALIGN(sizeof(size_t), ALIGN)
+ + res->key_size_aligned
+ + res->val_size_aligned;
+ res->max_entries = max_entries;
+ res->nb_entries = 0;
+ res->crc = av_crc_get_table(AV_CRC_32_IEEE);
+ if (!res->crc) {
+ ff_hashtable_freep(&res);
+ return AVERROR_BUG;
+ }
+ res->table = av_calloc(res->max_entries, res->entry_size);
+ if (!res->table) {
+ ff_hashtable_freep(&res);
+ return AVERROR(ENOMEM);
+ }
+
+ res->swapbuf = av_calloc(2, res->key_size_aligned + res->val_size_aligned);
+ if (!res->swapbuf) {
+ ff_hashtable_freep(&res);
+ return AVERROR(ENOMEM);
+ }
+ *ctx = res;
+ return 0;
+}
+
+static size_t hash_key(const struct FFHashtableContext *ctx, const void *key)
+{
+ return av_crc(ctx->crc, 0, key, ctx->key_size) % ctx->max_entries;
+}
+
+int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val)
+{
+ if (!ctx->nb_entries)
+ return 0;
+
+ size_t hash = hash_key(ctx, key);
+
+ for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
+ size_t wrapped_index = (hash + psl) % ctx->max_entries;
+ uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
+ if (ENTRY_PSL_VAL(entry) < psl)
+ // When PSL stops increasing it means there are no further entries
+ // with the same key hash.
+ return 0;
+ if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
+ memcpy(val, ENTRY_VAL_PTR(entry), ctx->val_size);
+ return 1;
+ }
+ }
+ return 0;
+}
+
+int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val)
+{
+ int swapping = 0;
+ size_t psl = 1;
+ size_t hash = hash_key(ctx, key);
+ size_t wrapped_index = hash % ctx->max_entries;
+ uint8_t *set = ctx->swapbuf;
+ uint8_t *tmp = ctx->swapbuf + ctx->key_size_aligned + ctx->val_size_aligned;
+
+ memcpy(set, key, ctx->key_size);
+ memcpy(set + ctx->key_size_aligned, val, ctx->val_size);
+
+ for (size_t i = 0; i < ctx->max_entries; i++) {
+ if (++wrapped_index == ctx->max_entries)
+ wrapped_index = 0;
+ uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
+ if (!ENTRY_PSL_VAL(entry) || (!swapping && KEYS_EQUAL(ENTRY_KEY_PTR(entry), set))) {
+ if (!ENTRY_PSL_VAL(entry))
+ ctx->nb_entries++;
+ ENTRY_PSL_VAL(entry) = psl;
+ memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size);
+ return 1;
+ }
+ if (ENTRY_PSL_VAL(entry) < psl) {
+ // When PSL stops increasing it means there are no further entries
+ // with the same key hash. We can only hope to find an unoccupied
+ // entry.
+ if (ctx->nb_entries == ctx->max_entries)
+ // The table is full so inserts are impossible.
+ return 0;
+ // Robin Hood hash tables "steal from the rich" by minimizing the
+ // PSL of the inserted entry.
+ swapping = 1;
+ // set needs to swap with entry
+ memcpy(tmp, ENTRY_KEY_PTR(entry), ctx->key_size_aligned + ctx->val_size_aligned);
+ memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size_aligned);
+ FFSWAP(uint8_t*, set, tmp);
+ FFSWAP(size_t, psl, ENTRY_PSL_VAL(entry));
+ }
+ psl++;
+ }
+ return 0;
+}
+
+int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key)
+{
+ if (!ctx->nb_entries)
+ return 0;
+
+ uint8_t *next_entry;
+ size_t hash = hash_key(ctx, key);
+ size_t wrapped_index = hash % ctx->max_entries;
+
+ for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
+ if (++wrapped_index == ctx->max_entries)
+ wrapped_index = 0;
+ uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
+ if (ENTRY_PSL_VAL(entry) < psl)
+ // When PSL stops increasing it means there are no further entries
+ // with the same key hash.
+ return 0;
+ if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
+ ENTRY_PSL_VAL(entry) = 0;
+ // Shift each following entry that will benefit from a reduced PSL.
+ for (psl++; psl <= ctx->max_entries; psl++) {
+ if (++wrapped_index == ctx->max_entries)
+ wrapped_index = 0;
+ next_entry = ctx->table + wrapped_index * ctx->entry_size;
+ if (ENTRY_PSL_VAL(next_entry) <= 1) {
+ ctx->nb_entries--;
+ return 1;
+ }
+ memcpy(entry, next_entry, ctx->entry_size);
+ ENTRY_PSL_VAL(entry)--;
+ ENTRY_PSL_VAL(next_entry) = 0;
+ entry = next_entry;
+ }
+ }
+ };
+ return 0;
+}
+
+void ff_hashtable_clear(struct FFHashtableContext *ctx)
+{
+ memset(ctx->table, 0, ctx->entry_size * ctx->max_entries);
+}
+
+void ff_hashtable_freep(struct FFHashtableContext **ctx)
+{
+ if (*ctx) {
+ av_freep(&(*ctx)->table);
+ av_freep(&(*ctx)->swapbuf);
+ }
+ av_freep(ctx);
+}
diff --git a/libavcodec/hashtable.h b/libavcodec/hashtable.h
new file mode 100644
index 0000000000..f81b4bb93f
--- /dev/null
+++ b/libavcodec/hashtable.h
@@ -0,0 +1,94 @@
+/*
+ * Generic hashtable
+ * Copyright (C) 2024 Emma Worley <emma@emma.gg>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_HASHTABLE_H
+#define AVCODEC_HASHTABLE_H
+
+#include <stddef.h>
+
+/* Implements a hash table using Robin Hood open addressing.
+ * See: https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf
+ *
+ * Keys are placed in the table based on their CRC value and are considered
+ * equal when they are bytewise-identical.
+ */
+
+typedef struct FFHashtableContext FFHashtableContext;
+
+/**
+ * Create a fixed-sized Robin Hood hash table.
+ *
+ * @param ctx context to allocate and initialize
+ * @param key_size size of key type in bytes
+ * @param val_size size of value type in bytes
+ * @param max_entries maximum number of key-value pairs to store
+ *
+ * @return zero on success, nonzero on error
+ */
+int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries);
+
+/**
+ * Look up a value from a hash table given a key.
+ *
+ * @param ctx hash table context
+ * @param key pointer to key data
+ * @param val destination pointer for value data
+ *
+ * @return 1 if the key is found, zero if the key is not found
+ */
+int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val);
+
+/**
+ * Store a value in a hash table given a key.
+ *
+ * @param ctx hash table context
+ * @param key pointer to key data
+ * @param val pointer for value data
+ *
+ * @return 1 if the key is written, zero if the key is not written due to the hash table reaching max capacity
+ */
+int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val);
+
+/**
+ * Delete a value from a hash table given a key.
+ *
+ * @param ctx hash table context
+ * @param key pointer to key data
+ *
+ * @return 1 if the key is deleted, zero if the key is not deleted due to not being found
+ */
+int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key);
+
+/**
+ * Delete all values from a hash table.
+ *
+ * @param ctx hash table context
+ */
+void ff_hashtable_clear(struct FFHashtableContext *ctx);
+
+/**
+ * Free a hash table.
+ *
+ * @param ctx hash table context
+ */
+void ff_hashtable_freep(struct FFHashtableContext **ctx);
+
+#endif
diff --git a/libavcodec/tests/hashtable.c b/libavcodec/tests/hashtable.c
new file mode 100644
index 0000000000..02c0ac8afa
--- /dev/null
+++ b/libavcodec/tests/hashtable.c
@@ -0,0 +1,110 @@
+/*
+ * Generic hashtable tests
+ * Copyright (C) 2024 Emma Worley <emma@emma.gg>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdint.h>
+
+#include "libavutil/avassert.h"
+#include "libavcodec/hashtable.h"
+
+int main(void)
+{
+ struct FFHashtableContext *ctx;
+ uint8_t k;
+ uint64_t v;
+
+ // impossibly large allocation should fail gracefully
+ av_assert0(ff_hashtable_alloc(&ctx, -1, -1, -1) < 0);
+
+ // hashtable can store up to 3 uint8_t->uint64_t entries
+ av_assert0(!ff_hashtable_alloc(&ctx, sizeof(k), sizeof(v), 3));
+
+ // unsuccessful deletes return 0
+ k = 1;
+ av_assert0(!ff_hashtable_delete(ctx, &k));
+
+ // unsuccessful gets return 0
+ k = 1;
+ av_assert0(!ff_hashtable_get(ctx, &k, &v));
+
+ // successful sets returns 1
+ k = 1;
+ v = 1;
+ av_assert0(ff_hashtable_set(ctx, &k, &v));
+
+ // get should now contain 1
+ k = 1;
+ v = 0;
+ av_assert0(ff_hashtable_get(ctx, &k, &v));
+ av_assert0(v == 1);
+
+ // updating sets should return 1
+ k = 1;
+ v = 2;
+ av_assert0(ff_hashtable_set(ctx, &k, &v));
+
+ // get should now contain 2
+ k = 1;
+ v = 0;
+ av_assert0(ff_hashtable_get(ctx, &k, &v));
+ av_assert0(v == 2);
+
+ // fill the table
+ k = 2;
+ v = 2;
+ av_assert0(ff_hashtable_set(ctx, &k, &v));
+ k = 3;
+ v = 3;
+ av_assert0(ff_hashtable_set(ctx, &k, &v));
+
+ // inserting sets on a full table should return 0
+ k = 4;
+ v = 4;
+ av_assert0(!ff_hashtable_set(ctx, &k, &v));
+
+ // updating sets on a full table should return 1
+ k = 1;
+ v = 4;
+ av_assert0(ff_hashtable_set(ctx, &k, &v));
+ v = 0;
+ av_assert0(ff_hashtable_get(ctx, &k, &v));
+ av_assert0(v == 4);
+
+ // successful deletes should return 1
+ k = 1;
+ av_assert0(ff_hashtable_delete(ctx, &k));
+
+ // get should now return 0
+ av_assert0(!ff_hashtable_get(ctx, &k, &v));
+
+ // sanity check remaining keys
+ k = 2;
+ v = 0;
+ av_assert0(ff_hashtable_get(ctx, &k, &v));
+ av_assert0(v == 2);
+ k = 3;
+ v = 0;
+ av_assert0(ff_hashtable_get(ctx, &k, &v));
+ av_assert0(v == 3);
+
+ ff_hashtable_freep(&ctx);
+
+ return 0;
+}
--
2.48.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 3+ messages in thread
* [FFmpeg-devel] [PATCH v6 2/3] lavc/dxvenc: migrate DXT1 encoder to lavc hashtable
2025-05-30 23:20 [FFmpeg-devel] [PATCH v6 1/3] lavc/hashtable: create generic robin hood hash table Emma Worley
@ 2025-05-30 23:20 ` Emma Worley
2025-05-30 23:20 ` [FFmpeg-devel] [PATCH v6 3/3] lavc/dxvenc: improve compatibility with Resolume products Emma Worley
1 sibling, 0 replies; 3+ messages in thread
From: Emma Worley @ 2025-05-30 23:20 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Emma Worley
Offers a modest performance gain due to the switch from naive linear
probling to robin hood.
Signed-off-by: Emma Worley <emma@emma.gg>
---
libavcodec/dxv.h | 2 +-
libavcodec/dxvenc.c | 123 ++++++++++++--------------------------------
2 files changed, 35 insertions(+), 90 deletions(-)
diff --git a/libavcodec/dxv.h b/libavcodec/dxv.h
index 71cfddec85..184813e427 100644
--- a/libavcodec/dxv.h
+++ b/libavcodec/dxv.h
@@ -1,6 +1,6 @@
/*
* Resolume DXV common
- * Copyright (C) 2024 Connor Worley <connorbworley@gmail.com>
+ * Copyright (C) 2024 Emma Worley <emma@emma.gg>
*
* This file is part of FFmpeg.
*
diff --git a/libavcodec/dxvenc.c b/libavcodec/dxvenc.c
index 808d8daedb..6eaf363f06 100644
--- a/libavcodec/dxvenc.c
+++ b/libavcodec/dxvenc.c
@@ -1,6 +1,6 @@
/*
* Resolume DXV encoder
- * Copyright (C) 2024 Connor Worley <connorbworley@gmail.com>
+ * Copyright (C) 2024 Emma Worley <emma@emma.gg>
*
* This file is part of FFmpeg.
*
@@ -21,7 +21,7 @@
#include <stdint.h>
-#include "libavutil/crc.h"
+#include "libavcodec/hashtable.h"
#include "libavutil/imgutils.h"
#include "libavutil/mem.h"
#include "libavutil/opt.h"
@@ -39,72 +39,9 @@
* appeared in the decompressed stream. Using a simple hash table (HT)
* significantly speeds up the lookback process while encoding.
*/
-#define LOOKBACK_HT_ELEMS 0x40000
+#define LOOKBACK_HT_ELEMS 0x20202
#define LOOKBACK_WORDS 0x20202
-typedef struct HTEntry {
- uint32_t key;
- uint32_t pos;
-} HTEntry;
-
-static void ht_init(HTEntry *ht)
-{
- for (size_t i = 0; i < LOOKBACK_HT_ELEMS; i++) {
- ht[i].pos = -1;
- }
-}
-
-static uint32_t ht_lookup_and_upsert(HTEntry *ht, const AVCRC *hash_ctx,
- uint32_t key, uint32_t pos)
-{
- uint32_t ret = -1;
- size_t hash = av_crc(hash_ctx, 0, (uint8_t*)&key, 4) % LOOKBACK_HT_ELEMS;
- for (size_t i = hash; i < hash + LOOKBACK_HT_ELEMS; i++) {
- size_t wrapped_index = i % LOOKBACK_HT_ELEMS;
- HTEntry *entry = &ht[wrapped_index];
- if (entry->key == key || entry->pos == -1) {
- ret = entry->pos;
- entry->key = key;
- entry->pos = pos;
- break;
- }
- }
- return ret;
-}
-
-static void ht_delete(HTEntry *ht, const AVCRC *hash_ctx,
- uint32_t key, uint32_t pos)
-{
- HTEntry *removed_entry = NULL;
- size_t removed_hash;
- size_t hash = av_crc(hash_ctx, 0, (uint8_t*)&key, 4) % LOOKBACK_HT_ELEMS;
-
- for (size_t i = hash; i < hash + LOOKBACK_HT_ELEMS; i++) {
- size_t wrapped_index = i % LOOKBACK_HT_ELEMS;
- HTEntry *entry = &ht[wrapped_index];
- if (entry->pos == -1)
- return;
- if (removed_entry) {
- size_t candidate_hash = av_crc(hash_ctx, 0, (uint8_t*)&entry->key, 4) % LOOKBACK_HT_ELEMS;
- if ((wrapped_index > removed_hash && (candidate_hash <= removed_hash || candidate_hash > wrapped_index)) ||
- (wrapped_index < removed_hash && (candidate_hash <= removed_hash && candidate_hash > wrapped_index))) {
- *removed_entry = *entry;
- entry->pos = -1;
- removed_entry = entry;
- removed_hash = wrapped_index;
- }
- } else if (entry->key == key) {
- if (entry->pos <= pos) {
- entry->pos = -1;
- removed_entry = entry;
- removed_hash = wrapped_index;
- } else {
- return;
- }
- }
- }
-}
-
typedef struct DXVEncContext {
AVClass *class;
@@ -121,10 +58,8 @@ typedef struct DXVEncContext {
DXVTextureFormat tex_fmt;
int (*compress_tex)(AVCodecContext *avctx);
- const AVCRC *crc_ctx;
-
- HTEntry color_lookback_ht[LOOKBACK_HT_ELEMS];
- HTEntry lut_lookback_ht[LOOKBACK_HT_ELEMS];
+ FFHashtableContext *color_ht;
+ FFHashtableContext *lut_ht;
} DXVEncContext;
/* Converts an index offset value to a 2-bit opcode and pushes it to a stream.
@@ -159,27 +94,32 @@ static int dxv_compress_dxt1(AVCodecContext *avctx)
DXVEncContext *ctx = avctx->priv_data;
PutByteContext *pbc = &ctx->pbc;
void *value;
- uint32_t color, lut, idx, color_idx, lut_idx, prev_pos, state = 16, pos = 2, op = 0;
+ uint32_t color, lut, idx, color_idx, lut_idx, prev_pos, state = 16, pos = 0, op = 0;
- ht_init(ctx->color_lookback_ht);
- ht_init(ctx->lut_lookback_ht);
+ ff_hashtable_clear(ctx->color_ht);
+ ff_hashtable_clear(ctx->lut_ht);
bytestream2_put_le32(pbc, AV_RL32(ctx->tex_data));
+ ff_hashtable_set(ctx->color_ht, ctx->tex_data, &pos);
+ pos++;
bytestream2_put_le32(pbc, AV_RL32(ctx->tex_data + 4));
-
- ht_lookup_and_upsert(ctx->color_lookback_ht, ctx->crc_ctx, AV_RL32(ctx->tex_data), 0);
- ht_lookup_and_upsert(ctx->lut_lookback_ht, ctx->crc_ctx, AV_RL32(ctx->tex_data + 4), 1);
+ ff_hashtable_set(ctx->lut_ht, ctx->tex_data + 4, &pos);
+ pos++;
while (pos + 2 <= ctx->tex_size / 4) {
idx = 0;
+ color_idx = 0;
+ lut_idx = 0;
color = AV_RL32(ctx->tex_data + pos * 4);
- prev_pos = ht_lookup_and_upsert(ctx->color_lookback_ht, ctx->crc_ctx, color, pos);
- color_idx = prev_pos != -1 ? pos - prev_pos : 0;
+ if (ff_hashtable_get(ctx->color_ht, &color, &prev_pos))
+ color_idx = pos - prev_pos;
+ ff_hashtable_set(ctx->color_ht, &color, &pos);
+
if (pos >= LOOKBACK_WORDS) {
uint32_t old_pos = pos - LOOKBACK_WORDS;
- uint32_t old_color = AV_RL32(ctx->tex_data + old_pos * 4);
- ht_delete(ctx->color_lookback_ht, ctx->crc_ctx, old_color, old_pos);
+ if (ff_hashtable_get(ctx->color_ht, ctx->tex_data + old_pos * 4, &prev_pos) && prev_pos <= old_pos)
+ ff_hashtable_delete(ctx->color_ht, ctx->tex_data + old_pos * 4);
}
pos++;
@@ -188,13 +128,14 @@ static int dxv_compress_dxt1(AVCodecContext *avctx)
idx = color_idx;
} else {
idx = 0;
- prev_pos = ht_lookup_and_upsert(ctx->lut_lookback_ht, ctx->crc_ctx, lut, pos);
- lut_idx = prev_pos != -1 ? pos - prev_pos : 0;
+ if (ff_hashtable_get(ctx->lut_ht, &lut, &prev_pos))
+ lut_idx = pos - prev_pos;
+ ff_hashtable_set(ctx->lut_ht, &lut, &pos);
}
if (pos >= LOOKBACK_WORDS) {
uint32_t old_pos = pos - LOOKBACK_WORDS;
- uint32_t old_lut = AV_RL32(ctx->tex_data + old_pos * 4);
- ht_delete(ctx->lut_lookback_ht, ctx->crc_ctx, old_lut, old_pos);
+ if (ff_hashtable_get(ctx->lut_ht, ctx->tex_data + old_pos * 4, &prev_pos) && prev_pos <= old_pos)
+ ff_hashtable_delete(ctx->lut_ht, ctx->tex_data + old_pos * 4);
}
pos++;
@@ -306,11 +247,12 @@ static av_cold int dxv_init(AVCodecContext *avctx)
return AVERROR(ENOMEM);
}
- ctx->crc_ctx = av_crc_get_table(AV_CRC_32_IEEE);
- if (!ctx->crc_ctx) {
- av_log(avctx, AV_LOG_ERROR, "Could not initialize CRC table.\n");
- return AVERROR_BUG;
- }
+ ret = ff_hashtable_alloc(&ctx->color_ht, sizeof(uint32_t), sizeof(uint32_t), LOOKBACK_HT_ELEMS);
+ if (ret < 0)
+ return ret;
+ ret = ff_hashtable_alloc(&ctx->lut_ht, sizeof(uint32_t), sizeof(uint32_t), LOOKBACK_HT_ELEMS);
+ if (ret < 0)
+ return ret;
return 0;
}
@@ -321,6 +263,9 @@ static av_cold int dxv_close(AVCodecContext *avctx)
av_freep(&ctx->tex_data);
+ ff_hashtable_freep(&ctx->color_ht);
+ ff_hashtable_freep(&ctx->lut_ht);
+
return 0;
}
--
2.48.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 3+ messages in thread
* [FFmpeg-devel] [PATCH v6 3/3] lavc/dxvenc: improve compatibility with Resolume products
2025-05-30 23:20 [FFmpeg-devel] [PATCH v6 1/3] lavc/hashtable: create generic robin hood hash table Emma Worley
2025-05-30 23:20 ` [FFmpeg-devel] [PATCH v6 2/3] lavc/dxvenc: migrate DXT1 encoder to lavc hashtable Emma Worley
@ 2025-05-30 23:20 ` Emma Worley
1 sibling, 0 replies; 3+ messages in thread
From: Emma Worley @ 2025-05-30 23:20 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Emma Worley
Improves compatibility with Resolume products by adding an additional
hashtable for DXT color+LUT combinations, and padding the DXT texture
dimensions to the next largest multiple of 16. Produces identical
packets to Resolume Alley in manual tests.
Signed-off-by: Emma Worley <emma@emma.gg>
---
libavcodec/dxvenc.c | 131 +++++++++++++++++++++++-------------
tests/ref/fate/dxv3enc-dxt1 | 2 +-
2 files changed, 86 insertions(+), 47 deletions(-)
diff --git a/libavcodec/dxvenc.c b/libavcodec/dxvenc.c
index 6eaf363f06..ee6a0a5b36 100644
--- a/libavcodec/dxvenc.c
+++ b/libavcodec/dxvenc.c
@@ -34,6 +34,11 @@
#define DXV_HEADER_LENGTH 12
+/*
+ * Resolume will refuse to display frames that are not padded to 16x16 pixels.
+ */
+#define DXV_ALIGN(x) FFALIGN(x, 16)
+
/*
* DXV uses LZ-like back-references to avoid copying words that have already
* appeared in the decompressed stream. Using a simple hash table (HT)
@@ -60,6 +65,7 @@ typedef struct DXVEncContext {
FFHashtableContext *color_ht;
FFHashtableContext *lut_ht;
+ FFHashtableContext *combo_ht;
} DXVEncContext;
/* Converts an index offset value to a 2-bit opcode and pushes it to a stream.
@@ -94,10 +100,14 @@ static int dxv_compress_dxt1(AVCodecContext *avctx)
DXVEncContext *ctx = avctx->priv_data;
PutByteContext *pbc = &ctx->pbc;
void *value;
- uint32_t color, lut, idx, color_idx, lut_idx, prev_pos, state = 16, pos = 0, op = 0;
+ uint64_t combo;
+ uint32_t color, lut, idx, combo_idx, prev_pos, old_pos, state = 16, pos = 0, op = 0;
ff_hashtable_clear(ctx->color_ht);
ff_hashtable_clear(ctx->lut_ht);
+ ff_hashtable_clear(ctx->combo_ht);
+
+ ff_hashtable_set(ctx->combo_ht, ctx->tex_data, &pos);
bytestream2_put_le32(pbc, AV_RL32(ctx->tex_data));
ff_hashtable_set(ctx->color_ht, ctx->tex_data, &pos);
@@ -107,51 +117,46 @@ static int dxv_compress_dxt1(AVCodecContext *avctx)
pos++;
while (pos + 2 <= ctx->tex_size / 4) {
- idx = 0;
- color_idx = 0;
- lut_idx = 0;
+ combo = AV_RL64(ctx->tex_data + pos * 4);
+ combo_idx = ff_hashtable_get(ctx->combo_ht, &combo, &prev_pos) ? pos - prev_pos : 0;
+ idx = combo_idx;
+ PUSH_OP(2);
+ if (pos >= LOOKBACK_WORDS) {
+ old_pos = pos - LOOKBACK_WORDS;
+ if (ff_hashtable_get(ctx->combo_ht, ctx->tex_data + old_pos * 4, &prev_pos) && prev_pos <= old_pos)
+ ff_hashtable_delete(ctx->combo_ht, ctx->tex_data + old_pos * 4);
+ }
+ ff_hashtable_set(ctx->combo_ht, &combo, &pos);
color = AV_RL32(ctx->tex_data + pos * 4);
- if (ff_hashtable_get(ctx->color_ht, &color, &prev_pos))
- color_idx = pos - prev_pos;
- ff_hashtable_set(ctx->color_ht, &color, &pos);
-
+ if (!combo_idx) {
+ idx = ff_hashtable_get(ctx->color_ht, &color, &prev_pos) ? pos - prev_pos : 0;
+ PUSH_OP(2);
+ if (!idx)
+ bytestream2_put_le32(pbc, color);
+ }
if (pos >= LOOKBACK_WORDS) {
- uint32_t old_pos = pos - LOOKBACK_WORDS;
+ old_pos = pos - LOOKBACK_WORDS;
if (ff_hashtable_get(ctx->color_ht, ctx->tex_data + old_pos * 4, &prev_pos) && prev_pos <= old_pos)
ff_hashtable_delete(ctx->color_ht, ctx->tex_data + old_pos * 4);
}
+ ff_hashtable_set(ctx->color_ht, &color, &pos);
pos++;
lut = AV_RL32(ctx->tex_data + pos * 4);
- if (color_idx && lut == AV_RL32(ctx->tex_data + (pos - color_idx) * 4)) {
- idx = color_idx;
- } else {
- idx = 0;
- if (ff_hashtable_get(ctx->lut_ht, &lut, &prev_pos))
- lut_idx = pos - prev_pos;
- ff_hashtable_set(ctx->lut_ht, &lut, &pos);
+ if (!combo_idx) {
+ idx = ff_hashtable_get(ctx->lut_ht, &lut, &prev_pos) ? pos - prev_pos : 0;
+ PUSH_OP(2);
+ if (!idx)
+ bytestream2_put_le32(pbc, lut);
}
if (pos >= LOOKBACK_WORDS) {
- uint32_t old_pos = pos - LOOKBACK_WORDS;
+ old_pos = pos - LOOKBACK_WORDS;
if (ff_hashtable_get(ctx->lut_ht, ctx->tex_data + old_pos * 4, &prev_pos) && prev_pos <= old_pos)
ff_hashtable_delete(ctx->lut_ht, ctx->tex_data + old_pos * 4);
}
+ ff_hashtable_set(ctx->lut_ht, &lut, &pos);
pos++;
-
- PUSH_OP(2);
-
- if (!idx) {
- idx = color_idx;
- PUSH_OP(2);
- if (!idx)
- bytestream2_put_le32(pbc, color);
-
- idx = lut_idx;
- PUSH_OP(2);
- if (!idx)
- bytestream2_put_le32(pbc, lut);
- }
}
return 0;
@@ -172,12 +177,50 @@ static int dxv_encode(AVCodecContext *avctx, AVPacket *pkt,
return ret;
if (ctx->enc.tex_funct) {
+ uint8_t *safe_data[4] = {frame->data[0], 0, 0, 0};
+ int safe_linesize[4] = {frame->linesize[0], 0, 0, 0};
+
+ if (avctx->width != DXV_ALIGN(avctx->width) || avctx->height != DXV_ALIGN(avctx->height)) {
+ ret = av_image_alloc(
+ safe_data,
+ safe_linesize,
+ DXV_ALIGN(avctx->width),
+ DXV_ALIGN(avctx->height),
+ avctx->pix_fmt,
+ 1);
+ if (ret < 0)
+ return ret;
+
+ av_image_copy2(
+ safe_data,
+ safe_linesize,
+ frame->data,
+ frame->linesize,
+ avctx->pix_fmt,
+ avctx->width,
+ avctx->height);
+
+ if (avctx->width != DXV_ALIGN(avctx->width)) {
+ for (int y = 0; y < avctx->height; y++) {
+ memset(safe_data[0] + y * safe_linesize[0] + frame->linesize[0], 0, safe_linesize[0] - frame->linesize[0]);
+ }
+ }
+ if (avctx->height != DXV_ALIGN(avctx->height)) {
+ for (int y = avctx->height; y < DXV_ALIGN(avctx->height); y++) {
+ memset(safe_data[0] + y * safe_linesize[0], 0, safe_linesize[0]);
+ }
+ }
+ }
+
ctx->enc.tex_data.out = ctx->tex_data;
- ctx->enc.frame_data.in = frame->data[0];
- ctx->enc.stride = frame->linesize[0];
- ctx->enc.width = avctx->width;
- ctx->enc.height = avctx->height;
+ ctx->enc.frame_data.in = safe_data[0];
+ ctx->enc.stride = safe_linesize[0];
+ ctx->enc.width = DXV_ALIGN(avctx->width);
+ ctx->enc.height = DXV_ALIGN(avctx->height);
ff_texturedsp_exec_compress_threads(avctx, &ctx->enc);
+
+ if (safe_data[0] != frame->data[0])
+ av_freep(&safe_data[0]);
} else {
/* unimplemented: YCoCg formats */
return AVERROR_INVALIDDATA;
@@ -216,14 +259,6 @@ static av_cold int dxv_init(AVCodecContext *avctx)
return ret;
}
- if (avctx->width % TEXTURE_BLOCK_W || avctx->height % TEXTURE_BLOCK_H) {
- av_log(avctx,
- AV_LOG_ERROR,
- "Video size %dx%d is not multiple of "AV_STRINGIFY(TEXTURE_BLOCK_W)"x"AV_STRINGIFY(TEXTURE_BLOCK_H)".\n",
- avctx->width, avctx->height);
- return AVERROR_INVALIDDATA;
- }
-
ff_texturedspenc_init(&texdsp);
switch (ctx->tex_fmt) {
@@ -237,10 +272,10 @@ static av_cold int dxv_init(AVCodecContext *avctx)
return AVERROR_INVALIDDATA;
}
ctx->enc.raw_ratio = 16;
- ctx->tex_size = avctx->width / TEXTURE_BLOCK_W *
- avctx->height / TEXTURE_BLOCK_H *
+ ctx->tex_size = DXV_ALIGN(avctx->width) / TEXTURE_BLOCK_W *
+ DXV_ALIGN(avctx->height) / TEXTURE_BLOCK_H *
ctx->enc.tex_ratio;
- ctx->enc.slice_count = av_clip(avctx->thread_count, 1, avctx->height / TEXTURE_BLOCK_H);
+ ctx->enc.slice_count = av_clip(avctx->thread_count, 1, DXV_ALIGN(avctx->height) / TEXTURE_BLOCK_H);
ctx->tex_data = av_malloc(ctx->tex_size);
if (!ctx->tex_data) {
@@ -251,6 +286,9 @@ static av_cold int dxv_init(AVCodecContext *avctx)
if (ret < 0)
return ret;
ret = ff_hashtable_alloc(&ctx->lut_ht, sizeof(uint32_t), sizeof(uint32_t), LOOKBACK_HT_ELEMS);
+ if (ret < 0)
+ return ret;
+ ret = ff_hashtable_alloc(&ctx->combo_ht, sizeof(uint64_t), sizeof(uint32_t), LOOKBACK_HT_ELEMS);
if (ret < 0)
return ret;
@@ -265,6 +303,7 @@ static av_cold int dxv_close(AVCodecContext *avctx)
ff_hashtable_freep(&ctx->color_ht);
ff_hashtable_freep(&ctx->lut_ht);
+ ff_hashtable_freep(&ctx->combo_ht);
return 0;
}
diff --git a/tests/ref/fate/dxv3enc-dxt1 b/tests/ref/fate/dxv3enc-dxt1
index 74849a8031..e09000e181 100644
--- a/tests/ref/fate/dxv3enc-dxt1
+++ b/tests/ref/fate/dxv3enc-dxt1
@@ -3,4 +3,4 @@
#codec_id 0: dxv
#dimensions 0: 1920x1080
#sar 0: 1/1
-0, 0, 0, 1, 76521, 0xed387a5e
+0, 0, 0, 1, 76190, 0x0e6f0326
--
2.48.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2025-05-30 23:20 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2025-05-30 23:20 [FFmpeg-devel] [PATCH v6 1/3] lavc/hashtable: create generic robin hood hash table Emma Worley
2025-05-30 23:20 ` [FFmpeg-devel] [PATCH v6 2/3] lavc/dxvenc: migrate DXT1 encoder to lavc hashtable Emma Worley
2025-05-30 23:20 ` [FFmpeg-devel] [PATCH v6 3/3] lavc/dxvenc: improve compatibility with Resolume products Emma Worley
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git