[FFmpeg-devel] [PATCH v5 1/3] lavc/hashtable: create generic robin hood hash table

* [FFmpeg-devel] [PATCH v5 1/3] lavc/hashtable: create generic robin hood hash table
@ 2025-04-25 16:05 Emma Worley
  2025-04-25 16:05 ` [FFmpeg-devel] [PATCH v5 2/3] lavc/dxvenc: migrate DXT1 encoder to lavc hashtable Emma Worley
                   ` (2 more replies)
  0 siblings, 3 replies; 14+ messages in thread
From: Emma Worley @ 2025-04-25 16:05 UTC (permalink / raw)
  To: ffmpeg-devel; +Cc: Emma Worley

Adds a generic hash table with the DXV encoder as an initial use case.

Signed-off-by: Emma Worley <emma@emma.gg>
---
 libavcodec/Makefile          |   2 +
 libavcodec/hashtable.c       | 214 +++++++++++++++++++++++++++++++++++
 libavcodec/hashtable.h       |  91 +++++++++++++++
 libavcodec/tests/hashtable.c | 110 ++++++++++++++++++
 4 files changed, 417 insertions(+)
 create mode 100644 libavcodec/hashtable.c
 create mode 100644 libavcodec/hashtable.h
 create mode 100644 libavcodec/tests/hashtable.c

diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 7bd1dbec9a..8071c59378 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -42,6 +42,7 @@ OBJS = ac3_parser.o                                                     \
        dv_profile.o                                                     \
        encode.o                                                         \
        get_buffer.o                                                     \
+       hashtable.o                                                      \
        imgconvert.o                                                     \
        jni.o                                                            \
        lcevcdec.o                                                       \
@@ -1321,6 +1322,7 @@ TESTPROGS = avcodec                                                     \
             bitstream_le                                                \
             celp_math                                                   \
             codec_desc                                                  \
+            hashtable                                                   \
             htmlsubtitles                                               \
             jpeg2000dwt                                                 \
             mathops                                                    \
diff --git a/libavcodec/hashtable.c b/libavcodec/hashtable.c
new file mode 100644
index 0000000000..151476176b
--- /dev/null
+++ b/libavcodec/hashtable.c
@@ -0,0 +1,214 @@
+/*
+ * Generic hashtable
+ * Copyright (C) 2025 Emma Worley <emma@emma.gg>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdint.h>
+#include <string.h>
+
+#include "libavutil/crc.h"
+#include "libavutil/error.h"
+#include "libavutil/mem.h"
+#include "hashtable.h"
+
+#define ALIGN _Alignof(size_t)
+
+struct FFHashtableContext {
+    size_t key_size;
+    size_t key_size_aligned;
+    size_t val_size;
+    size_t val_size_aligned;
+    size_t entry_size;
+    size_t max_entries;
+    size_t nb_entries;
+    const AVCRC *crc;
+    uint8_t *table;
+    uint8_t *swapbuf;
+};
+
+/*
+ * Hash table entries are comprised of a probe sequence length (PSL), key, and
+ * value. When the PSL of an entry is zero, it means it is not occupied by a
+ * key/value pair. When the PSL is non-zero, it represents the "distance" of
+ * the entry from its "home" location plus one, where the "home" location is
+ * hash(key) % max_entries.
+ */
+
+#define ENTRY_PSL_VAL(entry) (*(size_t*)(entry))
+#define ENTRY_KEY_PTR(entry) ((entry) + FFALIGN(sizeof(size_t), ALIGN))
+#define ENTRY_VAL_PTR(entry) (ENTRY_KEY_PTR(entry) + ctx->key_size_aligned)
+
+#define KEYS_EQUAL(k1, k2) (!memcmp((k1), (k2), ctx->key_size))
+
+int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries)
+{
+    struct FFHashtableContext *res = av_malloc(sizeof(struct FFHashtableContext));
+    if (!res)
+        return AVERROR(ENOMEM);
+    res->key_size = key_size;
+    res->key_size_aligned = FFALIGN(key_size, ALIGN);
+    res->val_size = val_size;
+    res->val_size_aligned = FFALIGN(val_size, ALIGN);
+    res->entry_size = FFALIGN(sizeof(size_t), ALIGN)
+                    + res->key_size_aligned
+                    + res->val_size_aligned;
+    res->max_entries = max_entries;
+    res->nb_entries = 0;
+    res->crc = av_crc_get_table(AV_CRC_32_IEEE);
+    if (!res->crc) {
+        ff_hashtable_freep(&res);
+        return AVERROR_BUG;
+    }
+    res->table = av_calloc(res->max_entries, res->entry_size);
+    if (!res->table) {
+        ff_hashtable_freep(&res);
+        return AVERROR(ENOMEM);
+    }
+
+    res->swapbuf = av_calloc(2, res->key_size_aligned + res->val_size_aligned);
+    if (!res->swapbuf) {
+        ff_hashtable_freep(&res);
+        return AVERROR(ENOMEM);
+    }
+    *ctx = res;
+    return 0;
+}
+
+static size_t hash_key(const struct FFHashtableContext *ctx, const void *key)
+{
+    return av_crc(ctx->crc, 0, key, ctx->key_size) % ctx->max_entries;
+}
+
+int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val)
+{
+    if (!ctx->nb_entries)
+        return 0;
+
+    size_t hash = hash_key(ctx, key);
+
+    for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
+        size_t wrapped_index = (hash + psl) % ctx->max_entries;
+        uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
+        if (ENTRY_PSL_VAL(entry) < psl)
+            // When PSL stops increasing it means there are no further entries
+            // with the same key hash.
+            return 0;
+        if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
+            memcpy(val, ENTRY_VAL_PTR(entry), ctx->val_size);
+            return 1;
+        }
+    }
+    return 0;
+}
+
+int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val)
+{
+    int swapping = 0;
+    size_t psl = 1;
+    size_t hash = hash_key(ctx, key);
+    size_t wrapped_index = hash % ctx->max_entries;
+    uint8_t *set = ctx->swapbuf;
+    uint8_t *tmp = ctx->swapbuf + ctx->key_size_aligned + ctx->val_size_aligned;
+
+    memcpy(set, key, ctx->key_size);
+    memcpy(set + ctx->key_size_aligned, val, ctx->val_size);
+
+    for (size_t i = 0; i < ctx->max_entries; i++) {
+        if (++wrapped_index == ctx->max_entries)
+            wrapped_index = 0;
+        uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
+        if (!ENTRY_PSL_VAL(entry) || (!swapping && KEYS_EQUAL(ENTRY_KEY_PTR(entry), set))) {
+            if (!ENTRY_PSL_VAL(entry))
+                ctx->nb_entries++;
+            ENTRY_PSL_VAL(entry) = psl;
+            memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size);
+            return 1;
+        }
+        if (ENTRY_PSL_VAL(entry) < psl) {
+            // When PSL stops increasing it means there are no further entries
+            // with the same key hash. We can only hope to find an unoccupied
+            // entry.
+            if (ctx->nb_entries == ctx->max_entries)
+                 // The table is full so inserts are impossible.
+                return 0;
+            // Robin Hood hash tables "steal from the rich" by minimizing the
+            // PSL of the inserted entry.
+            swapping = 1;
+            // set needs to swap with entry
+            memcpy(tmp, ENTRY_KEY_PTR(entry), ctx->key_size_aligned + ctx->val_size_aligned);
+            memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size_aligned);
+            FFSWAP(uint8_t*, set, tmp);
+            FFSWAP(size_t, psl, ENTRY_PSL_VAL(entry));
+        }
+        psl++;
+    }
+    return 0;
+}
+
+int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key)
+{
+    if (!ctx->nb_entries)
+        return 0;
+
+    uint8_t *next_entry;
+    size_t hash = hash_key(ctx, key);
+    size_t wrapped_index = hash % ctx->max_entries;
+
+    for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
+        if (++wrapped_index == ctx->max_entries)
+            wrapped_index = 0;
+        uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
+        if (ENTRY_PSL_VAL(entry) < psl)
+            // When PSL stops increasing it means there are no further entries
+            // with the same key hash.
+            return 0;
+        if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
+            ENTRY_PSL_VAL(entry) = 0;
+            // Shift each following entry that will benefit from a reduced PSL.
+            for (psl++; psl <= ctx->max_entries; psl++) {
+                if (++wrapped_index == ctx->max_entries)
+                    wrapped_index = 0;
+                next_entry = ctx->table + wrapped_index * ctx->entry_size;
+                if (ENTRY_PSL_VAL(next_entry) <= 1) {
+                    ctx->nb_entries--;
+                    return 1;
+                }
+                memcpy(entry, next_entry, ctx->entry_size);
+                ENTRY_PSL_VAL(entry)--;
+                ENTRY_PSL_VAL(next_entry) = 0;
+                entry = next_entry;
+            }
+        }
+    };
+    return 0;
+}
+
+void ff_hashtable_clear(struct FFHashtableContext *ctx)
+{
+    memset(ctx->table, 0, ctx->entry_size * ctx->max_entries);
+}
+
+void ff_hashtable_freep(struct FFHashtableContext **ctx)
+{
+    if (*ctx) {
+        av_freep(&(*ctx)->table);
+        av_freep(&(*ctx)->swapbuf);
+    }
+    av_freep(ctx);
+}
diff --git a/libavcodec/hashtable.h b/libavcodec/hashtable.h
new file mode 100644
index 0000000000..a26eb30cec
--- /dev/null
+++ b/libavcodec/hashtable.h
@@ -0,0 +1,91 @@
+/*
+ * Generic hashtable
+ * Copyright (C) 2024 Connor Worley <connorbworley@gmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_HASHTABLE_H
+#define AVCODEC_HASHTABLE_H
+
+#include <stddef.h>
+
+/* Implements a hash table using Robin Hood open addressing.
+ * See: https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf
+ */
+
+typedef struct FFHashtableContext FFHashtableContext;
+
+/**
+ * Create a fixed-sized Robin Hood hash table.
+ *
+ * @param ctx         context to allocate and initialize
+ * @param key_size    size of key type in bytes
+ * @param val_size    size of value type in bytes
+ * @param max_entries maximum number of key-value pairs to store
+ *
+ * @return zero on success, nonzero on error
+ */
+int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries);
+
+/**
+ * Look up a value from a hash table given a key.
+ *
+ * @param ctx hash table context
+ * @param key pointer to key data
+ * @param val destination pointer for value data
+ *
+ * @return 1 if the key is found, zero if the key is not found
+ */
+int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val);
+
+/**
+ * Store a value in a hash table given a key.
+ *
+ * @param ctx hash table context
+ * @param key pointer to key data
+ * @param val pointer for value data
+ *
+ * @return 1 if the key is written, zero if the key is not written due to the hash table reaching max capacity
+ */
+int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val);
+
+/**
+ * Delete a value from a hash table given a key.
+ *
+ * @param ctx hash table context
+ * @param key pointer to key data
+ *
+ * @return 1 if the key is deleted, zero if the key is not deleted due to not being found
+ */
+int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key);
+
+/**
+ * Delete all values from a hash table.
+ *
+ * @param ctx hash table context
+ */
+void ff_hashtable_clear(struct FFHashtableContext *ctx);
+
+/**
+ * Free a hash table.
+ *
+ * @param ctx hash table context
+ */
+void ff_hashtable_freep(struct FFHashtableContext **ctx);
+
+#endif
diff --git a/libavcodec/tests/hashtable.c b/libavcodec/tests/hashtable.c
new file mode 100644
index 0000000000..367baf4f96
--- /dev/null
+++ b/libavcodec/tests/hashtable.c
@@ -0,0 +1,110 @@
+/*
+ * Generic hashtable tests
+ * Copyright (C) 2024 Connor Worley <connorbworley@gmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdint.h>
+
+#include "libavutil/avassert.h"
+#include "libavcodec/hashtable.h"
+
+int main(void)
+{
+    struct FFHashtableContext *ctx;
+    uint8_t k;
+    uint64_t v;
+
+    // impossibly large allocation should fail gracefully
+    av_assert0(ff_hashtable_alloc(&ctx, -1, -1, -1) < 0);
+
+    // hashtable can store up to 3 uint8_t->uint64_t entries
+    av_assert0(!ff_hashtable_alloc(&ctx, sizeof(k), sizeof(v), 3));
+
+    // unsuccessful deletes return 0
+    k = 1;
+    av_assert0(!ff_hashtable_delete(ctx, &k));
+
+    // unsuccessful gets return 0
+    k = 1;
+    av_assert0(!ff_hashtable_get(ctx, &k, &v));
+
+    // successful sets returns 1
+    k = 1;
+    v = 1;
+    av_assert0(ff_hashtable_set(ctx, &k, &v));
+
+    // get should now contain 1
+    k = 1;
+    v = 0;
+    av_assert0(ff_hashtable_get(ctx, &k, &v));
+    av_assert0(v == 1);
+
+    // updating sets should return 1
+    k = 1;
+    v = 2;
+    av_assert0(ff_hashtable_set(ctx, &k, &v));
+
+    // get should now contain 2
+    k = 1;
+    v = 0;
+    av_assert0(ff_hashtable_get(ctx, &k, &v));
+    av_assert0(v == 2);
+
+    // fill the table
+    k = 2;
+    v = 2;
+    av_assert0(ff_hashtable_set(ctx, &k, &v));
+    k = 3;
+    v = 3;
+    av_assert0(ff_hashtable_set(ctx, &k, &v));
+
+    // inserting sets on a full table should return 0
+    k = 4;
+    v = 4;
+    av_assert0(!ff_hashtable_set(ctx, &k, &v));
+
+    // updating sets on a full table should return 1
+    k = 1;
+    v = 4;
+    av_assert0(ff_hashtable_set(ctx, &k, &v));
+    v = 0;
+    av_assert0(ff_hashtable_get(ctx, &k, &v));
+    av_assert0(v == 4);
+
+    // successful deletes should return 1
+    k = 1;
+    av_assert0(ff_hashtable_delete(ctx, &k));
+
+    // get should now return 0
+    av_assert0(!ff_hashtable_get(ctx, &k, &v));
+
+    // sanity check remaining keys
+    k = 2;
+    v = 0;
+    av_assert0(ff_hashtable_get(ctx, &k, &v));
+    av_assert0(v == 2);
+    k = 3;
+    v = 0;
+    av_assert0(ff_hashtable_get(ctx, &k, &v));
+    av_assert0(v == 3);
+
+    ff_hashtable_freep(&ctx);
+
+    return 0;
+}
-- 
2.48.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

^ permalink raw reply	[flat|nested] 14+ messages in thread