* [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
@ 2025-06-02 2:38 Andreas Rheinhardt
2025-06-02 3:22 ` softworkz .
2025-06-02 22:33 ` softworkz .
0 siblings, 2 replies; 19+ messages in thread
From: Andreas Rheinhardt @ 2025-06-02 2:38 UTC (permalink / raw)
To: FFmpeg development discussions and patches
[-- Attachment #1: Type: text/plain, Size: 207 bytes --]
This is a WIP patchset to further improve the resource manager. The
configure bits are the one that need more work: How do we detect zlib
(and the required library to link to) on the host system?
- Andreas
[-- Attachment #2: 0001-ffbuild-bin2c-Use-zlib-directly-instead-of-gzip.patch --]
[-- Type: text/x-patch, Size: 12792 bytes --]
From 30e67850175e5dd6dffa7488a032bbbe14067d10 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Sun, 1 Jun 2025 22:38:28 +0200
Subject: [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
Perform zlib compression directly in bin2c.c instead of
using gzip for compression, followed by bin2c to
put the file in the binary. This has the advantage of
avoiding intermediate files and also saves a few bytes
(the gzip header is longer than the zlib header).
It also avoids the hypothetical problem of the build system
supporting gzip with the zlib version used at runtime
not supporting gzip.
But the real advantages will become apparent in future commits.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
.gitignore | 1 -
ffbuild/bin2c.c | 154 ++++++++++++++++++++++++++++++---
ffbuild/common.mak | 44 ++--------
fftools/resources/.gitignore | 3 -
fftools/resources/Makefile | 2 +-
fftools/resources/resman.c | 6 +-
libavfilter/cuda/load_helper.c | 2 +-
7 files changed, 151 insertions(+), 61 deletions(-)
diff --git a/.gitignore b/.gitignore
index 59c89da5e0..901b47e76b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,7 +25,6 @@
*.metallib.c
*.ptx
*.ptx.c
-*.ptx.gz
*_g
\#*
.\#*
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index 91f2d81e5b..a3e49f0a11 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -20,48 +20,174 @@
* DEALINGS IN THE SOFTWARE.
*/
+#include "config.h"
+
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
#include <string.h>
#include <stdio.h>
+#include "libavutil/attributes.h"
+#include "libavutil/macros.h"
+
+#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
+#include <zlib.h>
+
+#define MAX_BUF_SIZE UINT32_MAX
+
+static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *compressed_sizep,
+ FILE *input)
+{
+ z_stream zstream = { 0 };
+ int ret, zret = deflateInit(&zstream, 9);
+
+ if (zret != Z_OK)
+ return -1;
+
+ uint32_t buffer_size = 0, compressed_size = 0;
+ unsigned char *compressed_data = NULL;
+ int flush = Z_NO_FLUSH;
+
+ do {
+ unsigned char tmp[4096];
+ size_t read = fread(tmp, 1, sizeof(tmp), input);
+ if (read < sizeof(tmp))
+ flush = Z_FINISH;
+
+ zstream.next_in = tmp;
+ zstream.avail_in = read;
+
+ do {
+ if (compressed_size >= buffer_size) {
+ if (buffer_size == MAX_BUF_SIZE) {
+ ret = -1;
+ goto fail;
+ }
+
+ buffer_size = buffer_size ? FFMIN(compressed_size * 2ll, MAX_BUF_SIZE) : 4096;
+ void *tmp_ptr = realloc(compressed_data, buffer_size);
+ if (!tmp_ptr) {
+ ret = -1;
+ goto fail;
+ }
+ compressed_data = tmp_ptr;
+ }
+
+ zstream.next_out = compressed_data + compressed_size;
+ zstream.avail_out = buffer_size - compressed_size;
+
+ zret = deflate(&zstream, flush);
+ compressed_size = buffer_size - zstream.avail_out;
+ if (zret == Z_STREAM_END)
+ break;
+ if (zret != Z_OK) {
+ ret = -1;
+ goto fail;
+ }
+ } while (zstream.avail_in > 0 || zstream.avail_out == 0);
+ } while (flush != Z_FINISH);
+
+ deflateEnd(&zstream);
+ *compressed_datap = compressed_data;
+ *compressed_sizep = compressed_size;
+
+ return 0;
+fail:
+ free(compressed_data);
+ deflateEnd(&zstream);
+
+ return ret;
+}
+
+static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep)
+{
+ unsigned char *compressed_data;
+ uint32_t compressed_size;
+
+ int err = read_file_and_compress(&compressed_data, &compressed_size, input);
+ if (err)
+ return err;
+
+ *compressed_sizep = compressed_size;
+
+ for (unsigned i = 0; i < compressed_size; ++i)
+ fprintf(output, "0x%02x, ", compressed_data[i]);
+
+ free(compressed_data);
+
+ return 0;
+}
+#endif
+
int main(int argc, char **argv)
{
const char *name;
FILE *input, *output;
unsigned int length = 0;
unsigned char data;
+ av_unused int compression = 0;
+ int arg_idx = 1;
- if (argc < 3 || argc > 4)
+ if (argc < 3)
return 1;
- input = fopen(argv[1], "rb");
+ if (!strcmp(argv[arg_idx], "--compress")) {
+#if !CONFIG_PTX_COMPRESSION && !CONFIG_RESOURCE_COMPRESSION
+ fprintf(stderr, "Compression unsupported in this configuration. "
+ "This is a bug. Please report it.\n");
+ return -1;
+#endif
+ compression = 1;
+ ++arg_idx;
+ }
+
+ if (argc - arg_idx > 3)
+ return 1;
+
+ char *input_name = argv[arg_idx++];
+ input = fopen(input_name, "rb");
if (!input)
return -1;
- output = fopen(argv[2], "wb");
+ output = fopen(argv[arg_idx++], "wb");
if (!output) {
fclose(input);
return -1;
}
- if (argc == 4) {
- name = argv[3];
+ if (arg_idx < argc) {
+ name = argv[arg_idx++];
} else {
- size_t arglen = strlen(argv[1]);
- name = argv[1];
+ size_t arglen = strlen(input_name);
+ name = input_name;
for (int i = 0; i < arglen; i++) {
- if (argv[1][i] == '.')
- argv[1][i] = '_';
- else if (argv[1][i] == '/')
- name = &argv[1][i+1];
+ if (input_name[i] == '.')
+ input_name[i] = '_';
+ else if (input_name[i] == '/')
+ name = &input_name[i+1];
}
}
fprintf(output, "const unsigned char ff_%s_data[] = { ", name);
- while (fread(&data, 1, 1, input) > 0) {
- fprintf(output, "0x%02x, ", data);
- length++;
+#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
+ if (compression) {
+ int err = handle_compressed_file(input, output, &length);
+ if (err) {
+ fclose(input);
+ fclose(output);
+ return err;
+ }
+ } else
+#endif
+ {
+ while (fread(&data, 1, 1, input) > 0) {
+ fprintf(output, "0x%02x, ", data);
+ length++;
+ }
}
fprintf(output, "0x00 };\n");
diff --git a/ffbuild/common.mak b/ffbuild/common.mak
index ddf48923ea..b830f9862b 100644
--- a/ffbuild/common.mak
+++ b/ffbuild/common.mak
@@ -113,7 +113,7 @@ COMPILE_LASX = $(call COMPILE,CC,LASXFLAGS)
$(Q)echo '#include "$*.h"' >$@
$(BIN2CEXE): ffbuild/bin2c_host.o
- $(HOSTLD) $(HOSTLDFLAGS) $(HOSTLD_O) $^ $(HOSTEXTRALIBS)
+ $(HOSTLD) $(HOSTLDFLAGS) $(HOSTLD_O) $^ $(HOSTEXTRALIBS) $(if $(CONFIG_PTX_COMPRESSION)$(CONFIG_RESOURCE_COMPRESSION),-lz)
%.metal.air: %.metal
$(METALCC) $< -o $@
@@ -127,17 +127,8 @@ $(BIN2CEXE): ffbuild/bin2c_host.o
%.ptx: %.cu $(SRC_PATH)/compat/cuda/cuda_runtime.h
$(COMPILE_NVCC)
-ifdef CONFIG_PTX_COMPRESSION
-%.ptx.gz: TAG = GZIP
-%.ptx.gz: %.ptx
- $(M)gzip -nc9 $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) >$@
-
-%.ptx.c: %.ptx.gz $(BIN2CEXE)
- $(BIN2C) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
-else
%.ptx.c: %.ptx $(BIN2CEXE)
- $(BIN2C) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
-endif
+ $(BIN2C) $(if $(CONFIG_PTX_COMPRESSION),--compress) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
# 1) Preprocess CSS to a minified version
%.css.min: TAG = SED
@@ -148,36 +139,13 @@ endif
| sed 's/^ //; s/ $$//' \
> $@
-ifdef CONFIG_RESOURCE_COMPRESSION
-
-# 2) Gzip the minified CSS
-%.css.min.gz: TAG = GZIP
-%.css.min.gz: %.css.min
- $(M)gzip -nc9 $< > $@
-
-# 3) Convert the gzipped CSS to a .c array
-%.css.c: %.css.min.gz $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
-
-# 4) Gzip the HTML file (no minification needed)
-%.html.gz: TAG = GZIP
-%.html.gz: %.html
- $(M)gzip -nc9 $< > $@
-
-# 5) Convert the gzipped HTML to a .c array
-%.html.c: %.html.gz $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
-
-else # NO COMPRESSION
-
# 2) Convert the minified CSS to a .c array
%.css.c: %.css.min $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
+ $(BIN2C) $(if $(CONFIG_RESOURCE_COMPRESSION),--compress) $< $@ $(subst .,_,$(basename $(notdir $@)))
# 3) Convert the plain HTML to a .c array
%.html.c: %.html $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
-endif
+ $(BIN2C) $(if $(CONFIG_RESOURCE_COMPRESSION),--compress) $< $@ $(subst .,_,$(basename $(notdir $@)))
clean::
$(RM) $(BIN2CEXE) $(CLEANSUFFIXES:%=ffbuild/%)
@@ -232,7 +200,7 @@ PTXOBJS = $(filter %.ptx.o,$(OBJS))
RESOURCEOBJS = $(filter %.css.o %.html.o,$(OBJS))
$(HOBJS): CCFLAGS += $(CFLAGS_HEADERS)
checkheaders: $(HOBJS)
-.SECONDARY: $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=.gz) $(PTXOBJS:.o=) $(RESOURCEOBJS:.o=.c) $(RESOURCEOBJS:%.css.o=%.css.min) $(RESOURCEOBJS:%.css.o=%.css.min.gz) $(RESOURCEOBJS:%.html.o=%.html.gz) $(RESOURCEOBJS:.o=)
+.SECONDARY: $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=) $(RESOURCEOBJS:.o=.c) $(RESOURCEOBJS:%.css.o=%.css.min) $(RESOURCEOBJS:.o=)
alltools: $(TOOLS)
@@ -252,7 +220,7 @@ $(TOOLOBJS): | tools
OUTDIRS := $(OUTDIRS) $(dir $(OBJS) $(HOBJS) $(HOSTOBJS) $(SHLIBOBJS) $(STLIBOBJS) $(TESTOBJS))
-CLEANSUFFIXES = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.gz *.ptx.c *.ver *.version *.html.gz *.html.c *.css.gz *.css.c *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb
+CLEANSUFFIXES = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.c *.ver *.version *.html.c *.css.c *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb
LIBSUFFIXES = *.a *.lib *.so *.so.* *.dylib *.dll *.def *.dll.a
define RULES
diff --git a/fftools/resources/.gitignore b/fftools/resources/.gitignore
index bda2c59a1c..65febdf13b 100644
--- a/fftools/resources/.gitignore
+++ b/fftools/resources/.gitignore
@@ -1,6 +1,3 @@
*.html.c
*.css.c
-*.html.gz
-*.css.gz
*.min
-*.min.gz
diff --git a/fftools/resources/Makefile b/fftools/resources/Makefile
index 8579a52678..aeb631f206 100644
--- a/fftools/resources/Makefile
+++ b/fftools/resources/Makefile
@@ -5,7 +5,7 @@ vpath %.html $(SRC_PATH)
vpath %.css $(SRC_PATH)
# Uncomment to prevent deletion during build
-#.PRECIOUS: %.css.c %.css.min %.css.gz %.css.min.gz %.html.gz %.html.c
+#.PRECIOUS: %.css.c %.css.min %.html.c
OBJS-resman += \
fftools/resources/resman.o \
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index aa53e96bf4..fdf73c36d5 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -65,7 +65,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
#if CONFIG_RESOURCE_COMPRESSION
-static int decompress_gzip(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
+static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
{
z_stream strm;
unsigned chunk = 65534;
@@ -83,7 +83,7 @@ static int decompress_gzip(ResourceManagerContext *ctx, uint8_t *in, unsigned in
}
// 15 + 16 tells zlib to detect GZIP or zlib automatically
- ret = inflateInit2(&strm, 15 + 16);
+ ret = inflateInit(&strm);
if (ret != Z_OK) {
av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg);
av_free(buf);
@@ -156,7 +156,7 @@ char *ff_resman_get_string(FFResourceId resource_id)
char *out = NULL;
size_t out_len;
- int ret = decompress_gzip(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
+ int ret = decompress_zlib(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index b049ec7130..7c62793077 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -43,7 +43,7 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
uint64_t buf_size;
int ret;
- if (inflateInit2(&stream, 32 + 15) != Z_OK) {
+ if (inflateInit(&stream) != Z_OK) {
av_log(avctx, AV_LOG_ERROR, "Error during zlib initialisation: %s\n", stream.msg);
return AVERROR(ENOSYS);
}
--
2.45.2
[-- Attachment #3: 0002-fftools-resources-resman-Don-t-mix-FFmpeg-and-zlib-e.patch --]
[-- Type: text/x-patch, Size: 1222 bytes --]
From c7f297f99bc68abcc535f29fcced8a81e668728f Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 02:46:58 +0200
Subject: [PATCH WIP 02/10] fftools/resources/resman: Don't mix FFmpeg and zlib
errors
Also remove some always-false checks.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index fdf73c36d5..f910535750 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -100,7 +100,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in
av_log(ctx, AV_LOG_ERROR, "Inflate failed: %d, %s\n", ret, strm.msg);
inflateEnd(&strm);
av_free(buf);
- return (ret == Z_STREAM_END) ? Z_OK : ((ret == Z_OK) ? Z_BUF_ERROR : ret);
+ return AVERROR_EXTERNAL;
}
if (strm.avail_out == 0) {
@@ -113,7 +113,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in
inflateEnd(&strm);
*out = (char *)buf;
- return Z_OK;
+ return 0;
}
#endif
--
2.45.2
[-- Attachment #4: 0003-fftools-resources-resman-Remove-unused-function-para.patch --]
[-- Type: text/x-patch, Size: 1931 bytes --]
From c4cdbee4002c1109d576a6e140f07b2bac3f9b81 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 02:53:24 +0200
Subject: [PATCH WIP 03/10] fftools/resources/resman: Remove unused function
parameter
Also constify in.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index f910535750..99d312bad8 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -65,7 +65,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
#if CONFIG_RESOURCE_COMPRESSION
-static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
+static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out)
{
z_stream strm;
unsigned chunk = 65534;
@@ -108,8 +108,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in
av_log(ctx, AV_LOG_WARNING, "Decompression buffer may be too small\n");
}
- *out_len = chunk - strm.avail_out;
- buf[*out_len] = 0; // Ensure null termination
+ buf[chunk - strm.avail_out] = 0; // Ensure null termination
inflateEnd(&strm);
*out = (char *)buf;
@@ -154,9 +153,8 @@ char *ff_resman_get_string(FFResourceId resource_id)
#if CONFIG_RESOURCE_COMPRESSION
char *out = NULL;
- size_t out_len;
- int ret = decompress_zlib(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
+ int ret = decompress_zlib(ctx, resource_definition.data, *resource_definition.data_len, &out);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
--
2.45.2
[-- Attachment #5: 0004-ffbuild-bin2c-Avoid-reallocations-when-uncompressing.patch --]
[-- Type: text/x-patch, Size: 8361 bytes --]
From 38436c69fc818b989594fb9912d4e99d8c4f3d1b Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Sun, 1 Jun 2025 23:19:29 +0200
Subject: [PATCH WIP 04/10] ffbuild/bin2c: Avoid reallocations when
uncompressing code,resources
This can be easily achieved by storing the uncompressed size
together with the actual zlib data. It also allows to use
zlib's uncompress() utility function.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
ffbuild/bin2c.c | 26 ++++++++++++---
fftools/resources/resman.c | 44 +++++--------------------
libavfilter/cuda/load_helper.c | 60 ++++++++--------------------------
3 files changed, 43 insertions(+), 87 deletions(-)
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index a3e49f0a11..7bcb126c7e 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -38,7 +38,7 @@
#define MAX_BUF_SIZE UINT32_MAX
static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *compressed_sizep,
- FILE *input)
+ uint32_t *uncompressed_sizep, FILE *input)
{
z_stream zstream = { 0 };
int ret, zret = deflateInit(&zstream, 9);
@@ -46,7 +46,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co
if (zret != Z_OK)
return -1;
- uint32_t buffer_size = 0, compressed_size = 0;
+ uint32_t buffer_size = 0, compressed_size = 0, uncompressed_size = 0;
unsigned char *compressed_data = NULL;
int flush = Z_NO_FLUSH;
@@ -56,6 +56,12 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co
if (read < sizeof(tmp))
flush = Z_FINISH;
+ uncompressed_size += read;
+ if (uncompressed_size < read) { // overflow
+ ret = -1;
+ goto fail;
+ }
+
zstream.next_in = tmp;
zstream.avail_in = read;
@@ -92,6 +98,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co
deflateEnd(&zstream);
*compressed_datap = compressed_data;
*compressed_sizep = compressed_size;
+ *uncompressed_sizep = uncompressed_size;
return 0;
fail:
@@ -101,17 +108,28 @@ fail:
return ret;
}
+/// Write a 32bit integer according to the target's endianness.
+static void write_u32(FILE *output, uint32_t num)
+{
+ for (int i = 0; i < 4; ++i)
+ fprintf(output, "0x%02x, ",
+ (num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff);
+}
+
static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep)
{
unsigned char *compressed_data;
- uint32_t compressed_size;
+ uint32_t compressed_size, uncompressed_size;
- int err = read_file_and_compress(&compressed_data, &compressed_size, input);
+ int err = read_file_and_compress(&compressed_data, &compressed_size,
+ &uncompressed_size, input);
if (err)
return err;
*compressed_sizep = compressed_size;
+ write_u32(output, uncompressed_size);
+
for (unsigned i = 0; i < compressed_size; ++i)
fprintf(output, "0x%02x, ", compressed_data[i]);
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 99d312bad8..698529aca1 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -33,9 +33,8 @@
#include "resman.h"
#include "libavutil/avassert.h"
-#include "libavutil/pixdesc.h"
+#include "libavutil/intreadwrite.h"
#include "libavutil/dict.h"
-#include "libavutil/common.h"
extern const unsigned char ff_graph_html_data[];
extern const unsigned int ff_graph_html_len;
@@ -67,50 +66,23 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out)
{
- z_stream strm;
- unsigned chunk = 65534;
- int ret;
- uint8_t *buf;
-
- *out = NULL;
- memset(&strm, 0, sizeof(strm));
-
// Allocate output buffer with extra byte for null termination
- buf = (uint8_t *)av_mallocz(chunk + 1);
+ uint32_t uncompressed_size = AV_RN32(in);
+ uint8_t *buf = av_malloc(uncompressed_size + 1);
if (!buf) {
av_log(ctx, AV_LOG_ERROR, "Failed to allocate decompression buffer\n");
return AVERROR(ENOMEM);
}
-
- // 15 + 16 tells zlib to detect GZIP or zlib automatically
- ret = inflateInit(&strm);
- if (ret != Z_OK) {
- av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg);
- av_free(buf);
- return AVERROR(ENOSYS);
- }
-
- strm.avail_in = in_len;
- strm.next_in = in;
- strm.avail_out = chunk;
- strm.next_out = buf;
-
- ret = inflate(&strm, Z_FINISH);
- if (ret != Z_OK && ret != Z_STREAM_END) {
- av_log(ctx, AV_LOG_ERROR, "Inflate failed: %d, %s\n", ret, strm.msg);
- inflateEnd(&strm);
+ uLongf buf_size = uncompressed_size;
+ int ret = uncompress(buf, &buf_size, in + 4, in_len);
+ if (ret != Z_OK || uncompressed_size != buf_size) {
+ av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret);
av_free(buf);
return AVERROR_EXTERNAL;
}
- if (strm.avail_out == 0) {
- // TODO: Error or loop decoding?
- av_log(ctx, AV_LOG_WARNING, "Decompression buffer may be too small\n");
- }
-
- buf[chunk - strm.avail_out] = 0; // Ensure null termination
+ buf[uncompressed_size] = 0; // Ensure null termination
- inflateEnd(&strm);
*out = (char *)buf;
return 0;
}
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index 7c62793077..0288ea49bb 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -21,11 +21,11 @@
#include "libavutil/hwcontext.h"
#include "libavutil/hwcontext_cuda_internal.h"
#include "libavutil/cuda_check.h"
+#include "libavutil/intreadwrite.h"
#include "libavutil/mem.h"
#if CONFIG_PTX_COMPRESSION
#include <zlib.h>
-#define CHUNK_SIZE 1024 * 64
#endif
#include "load_helper.h"
@@ -38,57 +38,23 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
CudaFunctions *cu = hwctx->internal->cuda_dl;
#if CONFIG_PTX_COMPRESSION
- z_stream stream = { 0 };
- uint8_t *buf, *tmp;
- uint64_t buf_size;
- int ret;
-
- if (inflateInit(&stream) != Z_OK) {
- av_log(avctx, AV_LOG_ERROR, "Error during zlib initialisation: %s\n", stream.msg);
- return AVERROR(ENOSYS);
- }
-
- buf_size = CHUNK_SIZE * 4;
- buf = av_realloc(NULL, buf_size);
- if (!buf) {
- inflateEnd(&stream);
+ uint32_t uncompressed_size = AV_RN32(data);
+ uint8_t *buf = av_realloc(NULL, uncompressed_size + 1);
+ if (!buf)
return AVERROR(ENOMEM);
- }
-
- stream.next_in = data;
- stream.avail_in = length;
-
- do {
- stream.avail_out = buf_size - stream.total_out;
- stream.next_out = buf + stream.total_out;
-
- ret = inflate(&stream, Z_FINISH);
- if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) {
- av_log(avctx, AV_LOG_ERROR, "zlib inflate error(%d): %s\n", ret, stream.msg);
- inflateEnd(&stream);
- av_free(buf);
- return AVERROR(EINVAL);
- }
-
- if (stream.avail_out == 0) {
- buf_size += CHUNK_SIZE;
- tmp = av_realloc(buf, buf_size);
- if (!tmp) {
- inflateEnd(&stream);
- av_free(buf);
- return AVERROR(ENOMEM);
- }
- buf = tmp;
- }
- } while (ret != Z_STREAM_END);
+ uLongf buf_size = uncompressed_size;
+ int ret = uncompress(buf, &buf_size, data + 4, length);
+ if (ret != Z_OK || uncompressed_size != buf_size) {
+ av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret);
+ ret = AVERROR_EXTERNAL;
+ goto fail;
+ }
// NULL-terminate string
- // there is guaranteed to be space for this, due to condition in loop
- buf[stream.total_out] = 0;
-
- inflateEnd(&stream);
+ buf[uncompressed_size] = 0;
ret = CHECK_CU(cu->cuModuleLoadData(cu_module, buf));
+fail:
av_free(buf);
return ret;
#else
--
2.45.2
[-- Attachment #6: 0005-ffbuild-bin2c-Store-compressed-size-in-band.patch --]
[-- Type: text/x-patch, Size: 13490 bytes --]
From 89c5a836ebac6b9073fc377351d57ebd141c3271 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Sun, 1 Jun 2025 23:37:54 +0200
Subject: [PATCH WIP 05/10] ffbuild/bin2c: Store compressed size in-band
This is simpler than using a separate constant.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
ffbuild/bin2c.c | 10 +++-------
fftools/resources/resman.c | 14 +++++---------
fftools/resources/resman.h | 2 --
libavfilter/cuda/load_helper.c | 4 ++--
libavfilter/cuda/load_helper.h | 2 +-
libavfilter/vf_bilateral_cuda.c | 3 +--
libavfilter/vf_bwdif_cuda.c | 3 +--
libavfilter/vf_chromakey_cuda.c | 3 +--
libavfilter/vf_colorspace_cuda.c | 4 +---
libavfilter/vf_overlay_cuda.c | 3 +--
libavfilter/vf_scale_cuda.c | 3 +--
libavfilter/vf_thumbnail_cuda.c | 3 +--
libavfilter/vf_yadif_cuda.c | 3 +--
13 files changed, 19 insertions(+), 38 deletions(-)
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index 7bcb126c7e..fcff9d4ab7 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -116,7 +116,7 @@ static void write_u32(FILE *output, uint32_t num)
(num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff);
}
-static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep)
+static int handle_compressed_file(FILE *input, FILE *output)
{
unsigned char *compressed_data;
uint32_t compressed_size, uncompressed_size;
@@ -126,9 +126,8 @@ static int handle_compressed_file(FILE *input, FILE *output, unsigned *compresse
if (err)
return err;
- *compressed_sizep = compressed_size;
-
write_u32(output, uncompressed_size);
+ write_u32(output, compressed_size);
for (unsigned i = 0; i < compressed_size; ++i)
fprintf(output, "0x%02x, ", compressed_data[i]);
@@ -143,7 +142,6 @@ int main(int argc, char **argv)
{
const char *name;
FILE *input, *output;
- unsigned int length = 0;
unsigned char data;
av_unused int compression = 0;
int arg_idx = 1;
@@ -193,7 +191,7 @@ int main(int argc, char **argv)
#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
if (compression) {
- int err = handle_compressed_file(input, output, &length);
+ int err = handle_compressed_file(input, output);
if (err) {
fclose(input);
fclose(output);
@@ -204,12 +202,10 @@ int main(int argc, char **argv)
{
while (fread(&data, 1, 1, input) > 0) {
fprintf(output, "0x%02x, ", data);
- length++;
}
}
fprintf(output, "0x00 };\n");
- fprintf(output, "const unsigned int ff_%s_len = %u;\n", name, length);
fclose(output);
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 698529aca1..fc00465775 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -37,14 +37,11 @@
#include "libavutil/dict.h"
extern const unsigned char ff_graph_html_data[];
-extern const unsigned int ff_graph_html_len;
-
extern const unsigned char ff_graph_css_data[];
-extern const unsigned ff_graph_css_len;
static const FFResourceDefinition resource_definitions[] = {
- [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", &ff_graph_css_data[0], &ff_graph_css_len },
- [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", &ff_graph_html_data[0], &ff_graph_html_len },
+ [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", ff_graph_css_data },
+ [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", ff_graph_html_data },
};
@@ -64,7 +61,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
#if CONFIG_RESOURCE_COMPRESSION
-static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out)
+static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out)
{
// Allocate output buffer with extra byte for null termination
uint32_t uncompressed_size = AV_RN32(in);
@@ -74,7 +71,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsig
return AVERROR(ENOMEM);
}
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, in + 4, in_len);
+ int ret = uncompress(buf, &buf_size, in + 8, AV_RN32(in + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret);
av_free(buf);
@@ -126,8 +123,7 @@ char *ff_resman_get_string(FFResourceId resource_id)
char *out = NULL;
- int ret = decompress_zlib(ctx, resource_definition.data, *resource_definition.data_len, &out);
-
+ int ret = decompress_zlib(ctx, resource_definition.data, &out);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
goto end;
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index 6485db5091..ad57a10204 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -39,8 +39,6 @@ typedef struct FFResourceDefinition {
const char *name;
const unsigned char *data;
- const unsigned *data_len;
-
} FFResourceDefinition;
void ff_resman_uninit(void);
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index 0288ea49bb..e47f0421ef 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -33,7 +33,7 @@
#define CHECK_CU(x) FF_CUDA_CHECK_DL(avctx, cu, x)
int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_module,
- const unsigned char *data, const unsigned int length)
+ const unsigned char *data)
{
CudaFunctions *cu = hwctx->internal->cuda_dl;
@@ -44,7 +44,7 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
return AVERROR(ENOMEM);
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, data + 4, length);
+ int ret = uncompress(buf, &buf_size, data + 8, AV_RN32(data + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret);
ret = AVERROR_EXTERNAL;
diff --git a/libavfilter/cuda/load_helper.h b/libavfilter/cuda/load_helper.h
index 455bf36a23..d25aa0ae27 100644
--- a/libavfilter/cuda/load_helper.h
+++ b/libavfilter/cuda/load_helper.h
@@ -23,6 +23,6 @@
* Loads a CUDA module and applies any decompression, if necessary.
*/
int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_module,
- const unsigned char *data, const unsigned int length);
+ const unsigned char *data);
#endif /* AVFILTER_CUDA_LOAD_HELPER_H */
diff --git a/libavfilter/vf_bilateral_cuda.c b/libavfilter/vf_bilateral_cuda.c
index 7115fa9e05..50ddaad213 100644
--- a/libavfilter/vf_bilateral_cuda.c
+++ b/libavfilter/vf_bilateral_cuda.c
@@ -217,14 +217,13 @@ static av_cold int cuda_bilateral_load_functions(AVFilterContext *ctx)
int ret;
extern const unsigned char ff_vf_bilateral_cuda_ptx_data[];
- extern const unsigned int ff_vf_bilateral_cuda_ptx_len;
ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx));
if (ret < 0)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_bilateral_cuda_ptx_data, ff_vf_bilateral_cuda_ptx_len);
+ ff_vf_bilateral_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_bwdif_cuda.c b/libavfilter/vf_bwdif_cuda.c
index cc954ab1d3..82c46886f3 100644
--- a/libavfilter/vf_bwdif_cuda.c
+++ b/libavfilter/vf_bwdif_cuda.c
@@ -29,7 +29,6 @@
#include "cuda/load_helper.h"
extern const unsigned char ff_vf_bwdif_cuda_ptx_data[];
-extern const unsigned int ff_vf_bwdif_cuda_ptx_len;
typedef struct DeintCUDAContext {
YADIFContext yadif;
@@ -305,7 +304,7 @@ static int config_output(AVFilterLink *link)
if (ret < 0)
goto exit;
- ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_bwdif_cuda_ptx_data, ff_vf_bwdif_cuda_ptx_len);
+ ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_bwdif_cuda_ptx_data);
if (ret < 0)
goto exit;
diff --git a/libavfilter/vf_chromakey_cuda.c b/libavfilter/vf_chromakey_cuda.c
index 43f50c5a9a..e606da01ac 100644
--- a/libavfilter/vf_chromakey_cuda.c
+++ b/libavfilter/vf_chromakey_cuda.c
@@ -220,14 +220,13 @@ static av_cold int cudachromakey_load_functions(AVFilterContext *ctx)
int ret;
extern const unsigned char ff_vf_chromakey_cuda_ptx_data[];
- extern const unsigned int ff_vf_chromakey_cuda_ptx_len;
ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx));
if (ret < 0)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_chromakey_cuda_ptx_data, ff_vf_chromakey_cuda_ptx_len);
+ ff_vf_chromakey_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_colorspace_cuda.c b/libavfilter/vf_colorspace_cuda.c
index 54d6228cd1..2d922430e4 100644
--- a/libavfilter/vf_colorspace_cuda.c
+++ b/libavfilter/vf_colorspace_cuda.c
@@ -198,15 +198,13 @@ static av_cold int cudacolorspace_load_functions(AVFilterContext* ctx)
int ret;
extern const unsigned char ff_vf_colorspace_cuda_ptx_data[];
- extern const unsigned int ff_vf_colorspace_cuda_ptx_len;
ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx));
if (ret < 0)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_colorspace_cuda_ptx_data,
- ff_vf_colorspace_cuda_ptx_len);
+ ff_vf_colorspace_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_overlay_cuda.c b/libavfilter/vf_overlay_cuda.c
index 4708b34d30..77da5b5e01 100644
--- a/libavfilter/vf_overlay_cuda.c
+++ b/libavfilter/vf_overlay_cuda.c
@@ -415,7 +415,6 @@ static int overlay_cuda_activate(AVFilterContext *avctx)
static int overlay_cuda_config_output(AVFilterLink *outlink)
{
extern const unsigned char ff_vf_overlay_cuda_ptx_data[];
- extern const unsigned int ff_vf_overlay_cuda_ptx_len;
int err;
FilterLink *outl = ff_filter_link(outlink);
@@ -494,7 +493,7 @@ static int overlay_cuda_config_output(AVFilterLink *outlink)
return err;
}
- err = ff_cuda_load_module(ctx, ctx->hwctx, &ctx->cu_module, ff_vf_overlay_cuda_ptx_data, ff_vf_overlay_cuda_ptx_len);
+ err = ff_cuda_load_module(ctx, ctx->hwctx, &ctx->cu_module, ff_vf_overlay_cuda_ptx_data);
if (err < 0) {
CHECK_CU(cu->cuCtxPopCurrent(&dummy));
return err;
diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c
index 44eef207ca..6e238c7c73 100644
--- a/libavfilter/vf_scale_cuda.c
+++ b/libavfilter/vf_scale_cuda.c
@@ -290,7 +290,6 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx)
const char *function_infix = "";
extern const unsigned char ff_vf_scale_cuda_ptx_data[];
- extern const unsigned int ff_vf_scale_cuda_ptx_len;
switch(s->interp_algo) {
case INTERP_ALGO_NEAREST:
@@ -324,7 +323,7 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_scale_cuda_ptx_data, ff_vf_scale_cuda_ptx_len);
+ ff_vf_scale_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_thumbnail_cuda.c b/libavfilter/vf_thumbnail_cuda.c
index 121274de11..bea07f7041 100644
--- a/libavfilter/vf_thumbnail_cuda.c
+++ b/libavfilter/vf_thumbnail_cuda.c
@@ -368,7 +368,6 @@ static int config_props(AVFilterLink *inlink)
int ret;
extern const unsigned char ff_vf_thumbnail_cuda_ptx_data[];
- extern const unsigned int ff_vf_thumbnail_cuda_ptx_len;
s->hwctx = device_hwctx;
s->cu_stream = s->hwctx->stream;
@@ -377,7 +376,7 @@ static int config_props(AVFilterLink *inlink)
if (ret < 0)
return ret;
- ret = ff_cuda_load_module(ctx, device_hwctx, &s->cu_module, ff_vf_thumbnail_cuda_ptx_data, ff_vf_thumbnail_cuda_ptx_len);
+ ret = ff_cuda_load_module(ctx, device_hwctx, &s->cu_module, ff_vf_thumbnail_cuda_ptx_data);
if (ret < 0)
return ret;
diff --git a/libavfilter/vf_yadif_cuda.c b/libavfilter/vf_yadif_cuda.c
index 50ac61ad8a..cd9a5ba2e7 100644
--- a/libavfilter/vf_yadif_cuda.c
+++ b/libavfilter/vf_yadif_cuda.c
@@ -29,7 +29,6 @@
#include "cuda/load_helper.h"
extern const unsigned char ff_vf_yadif_cuda_ptx_data[];
-extern const unsigned int ff_vf_yadif_cuda_ptx_len;
typedef struct DeintCUDAContext {
YADIFContext yadif;
@@ -291,7 +290,7 @@ static int config_output(AVFilterLink *link)
if (ret < 0)
goto exit;
- ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_yadif_cuda_ptx_data, ff_vf_yadif_cuda_ptx_len);
+ ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_yadif_cuda_ptx_data);
if (ret < 0)
goto exit;
--
2.45.2
[-- Attachment #7: 0006-avfilter-cuda-load_helper-Use-aligned-reads.patch --]
[-- Type: text/x-patch, Size: 4063 bytes --]
From ced36923900a289ca0c74bdfc9a34b35154f4003 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 00:08:31 +0200
Subject: [PATCH WIP 06/10] avfilter/cuda/load_helper: Use aligned reads
Possible if one aligns the buffer.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
ffbuild/bin2c.c | 10 ++++++----
fftools/resources/resman.c | 4 ++--
libavfilter/cuda/load_helper.c | 4 ++--
3 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index fcff9d4ab7..1f1bce3a97 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -116,11 +116,14 @@ static void write_u32(FILE *output, uint32_t num)
(num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff);
}
-static int handle_compressed_file(FILE *input, FILE *output)
+static int handle_compressed_file(FILE *input, FILE *output, const char *name)
{
unsigned char *compressed_data;
uint32_t compressed_size, uncompressed_size;
+ fprintf(output, "#include \"libavutil/mem_internal.h\"\n");
+ fprintf(output, "DECLARE_ALIGNED_4(const unsigned char, ff_%s_data)[] = { ", name);
+
int err = read_file_and_compress(&compressed_data, &compressed_size,
&uncompressed_size, input);
if (err)
@@ -187,11 +190,9 @@ int main(int argc, char **argv)
}
}
- fprintf(output, "const unsigned char ff_%s_data[] = { ", name);
-
#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
if (compression) {
- int err = handle_compressed_file(input, output);
+ int err = handle_compressed_file(input, output, name);
if (err) {
fclose(input);
fclose(output);
@@ -200,6 +201,7 @@ int main(int argc, char **argv)
} else
#endif
{
+ fprintf(output, "const unsigned char ff_%s_data[] = { ", name);
while (fread(&data, 1, 1, input) > 0) {
fprintf(output, "0x%02x, ", data);
}
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index fc00465775..0094ecd4c5 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -64,14 +64,14 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out)
{
// Allocate output buffer with extra byte for null termination
- uint32_t uncompressed_size = AV_RN32(in);
+ uint32_t uncompressed_size = AV_RN32A(in);
uint8_t *buf = av_malloc(uncompressed_size + 1);
if (!buf) {
av_log(ctx, AV_LOG_ERROR, "Failed to allocate decompression buffer\n");
return AVERROR(ENOMEM);
}
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, in + 8, AV_RN32(in + 4));
+ int ret = uncompress(buf, &buf_size, in + 8, AV_RN32A(in + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret);
av_free(buf);
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index e47f0421ef..79ec37c7cd 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -38,13 +38,13 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
CudaFunctions *cu = hwctx->internal->cuda_dl;
#if CONFIG_PTX_COMPRESSION
- uint32_t uncompressed_size = AV_RN32(data);
+ uint32_t uncompressed_size = AV_RN32A(data);
uint8_t *buf = av_realloc(NULL, uncompressed_size + 1);
if (!buf)
return AVERROR(ENOMEM);
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, data + 8, AV_RN32(data + 4));
+ int ret = uncompress(buf, &buf_size, data + 8, AV_RN32A(data + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret);
ret = AVERROR_EXTERNAL;
--
2.45.2
[-- Attachment #8: 0007-fftools-resources-resman-Use-pointer-to-resource-def.patch --]
[-- Type: text/x-patch, Size: 3377 bytes --]
From dbb59f19565db0cb9c69296487bd7432d53b634b Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 03:09:45 +0200
Subject: [PATCH WIP 07/10] fftools/resources/resman: Use pointer to resource
definition
It does not really matter on its own given that FFResourceDefinition
is so small; it is mostly in preparation for the next commit.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 0094ecd4c5..1de98475c3 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -98,23 +98,23 @@ void ff_resman_uninit(void)
char *ff_resman_get_string(FFResourceId resource_id)
{
ResourceManagerContext *ctx = &resman_ctx;
- FFResourceDefinition resource_definition = { 0 };
+ const FFResourceDefinition *resource_definition = NULL;
AVDictionaryEntry *dic_entry;
char *res = NULL;
for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
- FFResourceDefinition def = resource_definitions[i];
- if (def.resource_id == resource_id) {
+ const FFResourceDefinition *def = &resource_definitions[i];
+ if (def->resource_id == resource_id) {
resource_definition = def;
break;
}
}
- av_assert1(resource_definition.name);
+ av_assert1(resource_definition);
ff_mutex_lock(&mutex);
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition.name, NULL, 0);
+ dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
if (!dic_entry) {
int dict_ret;
@@ -123,13 +123,13 @@ char *ff_resman_get_string(FFResourceId resource_id)
char *out = NULL;
- int ret = decompress_zlib(ctx, resource_definition.data, &out);
+ int ret = decompress_zlib(ctx, resource_definition->data, &out);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
goto end;
}
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition.name, out, 0);
+ dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, out, 0);
if (dict_ret < 0) {
av_log(ctx, AV_LOG_ERROR, "Failed to store decompressed resource in dictionary: %d\n", dict_ret);
av_freep(&out);
@@ -139,14 +139,14 @@ char *ff_resman_get_string(FFResourceId resource_id)
av_freep(&out);
#else
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition.name, (const char *)resource_definition.data, 0);
+ dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, (const char *)resource_definition->data, 0);
if (dict_ret < 0) {
av_log(ctx, AV_LOG_ERROR, "Failed to store resource in dictionary: %d\n", dict_ret);
goto end;
}
#endif
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition.name, NULL, 0);
+ dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
if (!dic_entry) {
av_log(ctx, AV_LOG_ERROR, "Failed to retrieve resource from dictionary after storing it\n");
--
2.45.2
[-- Attachment #9: 0008-fftools-resources-resman-Constify-ret-of-ff_resman_g.patch --]
[-- Type: text/x-patch, Size: 1571 bytes --]
From 19f4c569b687e4568513630f15f281774ce650e4 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 03:14:36 +0200
Subject: [PATCH WIP 08/10] fftools/resources/resman: Constify ret of
ff_resman_get_string()
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 4 ++--
fftools/resources/resman.h | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 1de98475c3..aa99e07f50 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -95,12 +95,12 @@ void ff_resman_uninit(void)
}
-char *ff_resman_get_string(FFResourceId resource_id)
+const char *ff_resman_get_string(FFResourceId resource_id)
{
ResourceManagerContext *ctx = &resman_ctx;
const FFResourceDefinition *resource_definition = NULL;
AVDictionaryEntry *dic_entry;
- char *res = NULL;
+ const char *res = NULL;
for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
const FFResourceDefinition *def = &resource_definitions[i];
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index ad57a10204..d41c89ce37 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -43,6 +43,6 @@ typedef struct FFResourceDefinition {
void ff_resman_uninit(void);
-char *ff_resman_get_string(FFResourceId resource_id);
+const char *ff_resman_get_string(FFResourceId resource_id);
#endif /* FFTOOLS_RESOURCES_RESMAN_H */
--
2.45.2
[-- Attachment #10: 0009-fftools-resources-resman-Avoid-AVDictionary.patch --]
[-- Type: text/x-patch, Size: 5281 bytes --]
From 66a22cb4bd701183cf06272d315a70ab6d6221df Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 03:37:43 +0200
Subject: [PATCH WIP 09/10] fftools/resources/resman: Avoid AVDictionary
Using it incurs an unnecessary duplication (temporary when
compression is enabled). It furthermore incurs checks
for av_dict_set()/av_dict_get(). Using an array of pointers
is simpler.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 64 +++++++++++---------------------------
fftools/resources/resman.h | 1 -
2 files changed, 18 insertions(+), 47 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index aa99e07f50..4497d65f3d 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -34,24 +34,23 @@
#include "resman.h"
#include "libavutil/avassert.h"
#include "libavutil/intreadwrite.h"
-#include "libavutil/dict.h"
extern const unsigned char ff_graph_html_data[];
extern const unsigned char ff_graph_css_data[];
static const FFResourceDefinition resource_definitions[] = {
- [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", ff_graph_css_data },
- [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", ff_graph_html_data },
+ [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, ff_graph_css_data },
+ [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, ff_graph_html_data },
};
-
+#if CONFIG_RESOURCE_COMPRESSION
static const AVClass resman_class = {
.class_name = "ResourceManager",
};
typedef struct ResourceManagerContext {
const AVClass *class;
- AVDictionary *resource_dic;
+ char *resources[FF_ARRAY_ELEMS(resource_definitions)];
} ResourceManagerContext;
static AVMutex mutex = AV_MUTEX_INITIALIZER;
@@ -59,8 +58,6 @@ static AVMutex mutex = AV_MUTEX_INITIALIZER;
static ResourceManagerContext resman_ctx = { .class = &resman_class };
-#if CONFIG_RESOURCE_COMPRESSION
-
static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out)
{
// Allocate output buffer with extra byte for null termination
@@ -87,76 +84,51 @@ static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char
void ff_resman_uninit(void)
{
+#if CONFIG_RESOURCE_COMPRESSION
ff_mutex_lock(&mutex);
- av_dict_free(&resman_ctx.resource_dic);
+ for (size_t i = 0; i < FF_ARRAY_ELEMS(resman_ctx.resources); ++i)
+ av_freep(&resman_ctx.resources[i]);
ff_mutex_unlock(&mutex);
+#endif
}
const char *ff_resman_get_string(FFResourceId resource_id)
{
- ResourceManagerContext *ctx = &resman_ctx;
const FFResourceDefinition *resource_definition = NULL;
- AVDictionaryEntry *dic_entry;
const char *res = NULL;
+ av_unused unsigned idx;
for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
const FFResourceDefinition *def = &resource_definitions[i];
if (def->resource_id == resource_id) {
resource_definition = def;
+ idx = i;
break;
}
}
av_assert1(resource_definition);
- ff_mutex_lock(&mutex);
-
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
-
- if (!dic_entry) {
- int dict_ret;
-
#if CONFIG_RESOURCE_COMPRESSION
+ ff_mutex_lock(&mutex);
- char *out = NULL;
+ ResourceManagerContext *ctx = &resman_ctx;
- int ret = decompress_zlib(ctx, resource_definition->data, &out);
+ if (!ctx->resources[idx]) {
+ int ret = decompress_zlib(ctx, resource_definition->data, &ctx->resources[idx]);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
goto end;
}
-
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, out, 0);
- if (dict_ret < 0) {
- av_log(ctx, AV_LOG_ERROR, "Failed to store decompressed resource in dictionary: %d\n", dict_ret);
- av_freep(&out);
- goto end;
- }
-
- av_freep(&out);
-#else
-
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, (const char *)resource_definition->data, 0);
- if (dict_ret < 0) {
- av_log(ctx, AV_LOG_ERROR, "Failed to store resource in dictionary: %d\n", dict_ret);
- goto end;
- }
-
-#endif
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
-
- if (!dic_entry) {
- av_log(ctx, AV_LOG_ERROR, "Failed to retrieve resource from dictionary after storing it\n");
- goto end;
- }
}
-
- res = dic_entry->value;
-
+ res = ctx->resources[idx];
end:
ff_mutex_unlock(&mutex);
+#else
+ res = resource_definition->data;
+#endif
return res;
}
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index d41c89ce37..ee22e960d3 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -36,7 +36,6 @@ typedef enum {
typedef struct FFResourceDefinition {
FFResourceId resource_id;
- const char *name;
const unsigned char *data;
} FFResourceDefinition;
--
2.45.2
[-- Attachment #11: 0010-fftools-resources-resman-Improve-headers.patch --]
[-- Type: text/x-patch, Size: 1476 bytes --]
From c4e1a52e6170f1d88bfef26fb4cfdaff470c961a Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 03:48:50 +0200
Subject: [PATCH WIP 10/10] fftools/resources/resman: Improve headers
Don't rely on implicit inclusions of lavu/thread.h and lavu/mem.h.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 5 ++++-
fftools/resources/resman.h | 8 --------
2 files changed, 4 insertions(+), 9 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 4497d65f3d..9b8f6e406a 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -25,10 +25,13 @@
#include "config.h"
-#include <string.h>
+#include <stddef.h>
#if CONFIG_RESOURCE_COMPRESSION
#include <zlib.h>
+#include "libavutil/intreadwrite.h"
+#include "libavutil/mem.h"
+#include "libavutil/thread.h"
#endif
#include "resman.h"
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index ee22e960d3..20e6b1037c 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -21,14 +21,6 @@
#ifndef FFTOOLS_RESOURCES_RESMAN_H
#define FFTOOLS_RESOURCES_RESMAN_H
-#include <stdint.h>
-
-#include "config.h"
-#include "fftools/ffmpeg.h"
-#include "libavutil/avutil.h"
-#include "libavutil/bprint.h"
-#include "fftools/textformat/avtextformat.h"
-
typedef enum {
FF_RESOURCE_GRAPH_CSS,
FF_RESOURCE_GRAPH_HTML,
--
2.45.2
[-- Attachment #12: Type: text/plain, Size: 251 bytes --]
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-02 2:38 [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip Andreas Rheinhardt
@ 2025-06-02 3:22 ` softworkz .
2025-06-02 22:33 ` softworkz .
1 sibling, 0 replies; 19+ messages in thread
From: softworkz . @ 2025-06-02 3:22 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
> Rheinhardt
> Sent: Montag, 2. Juni 2025 04:39
> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> Subject: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly
> instead of gzip
>
> This is a WIP patchset to further improve the resource manager. The
> configure bits are the one that need more work
>
> - Andreas
I like it. This obsoletes my patchset
https://patchwork.ffmpeg.org/project/ffmpeg/list/?series=14651
..which is fine.
> How do we detect zlib
> (and the required library to link to) on the host system?
I suppose you've seen ramiro's patchset and discussion already,
but in case not:
https://patchwork.ffmpeg.org/project/ffmpeg/list/?series=14620
(not sure whether this gives an answer to your question, though)
Best,
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-02 2:38 [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip Andreas Rheinhardt
2025-06-02 3:22 ` softworkz .
@ 2025-06-02 22:33 ` softworkz .
2025-06-03 14:33 ` Andreas Rheinhardt
1 sibling, 1 reply; 19+ messages in thread
From: softworkz . @ 2025-06-02 22:33 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
> Rheinhardt
> Sent: Montag, 2. Juni 2025 04:39
> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> Subject: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly
> instead of gzip
>
> This is a WIP patchset to further improve the resource manager. The
> configure bits are the one that need more work: How do we detect zlib
> (and the required library to link to) on the host system?
>
Hi,
I ran CI builds for the whole Patchset
https://github.com/ffstaging/FFmpeg/pull/91
Mac build fails (all others are fine)
https://dev.azure.com/githubsync/ffmpeg/_build/results?buildId=91751&view=logs
Undefined symbols for architecture x86_64:
Undefined symbols for architecture x86_64:
"_ff_vf_yadif_videotoolbox_metallib_len", referenced from:
_do_init in libavfilter.a[397](vf_yadif_videotoolbox.o)
ld: symbol(s) not found for architecture x86_64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
make: *** [ffprobe_g] Error 1
"_ff_vf_yadif_videotoolbox_metallib_len", referenced from:
_do_init in libavfilter.a[397](vf_yadif_videotoolbox.o)
ld: symbol(s) not found for architecture x86_64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
make: *** [ffplay_g] Error 1
Best regards
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-02 22:33 ` softworkz .
@ 2025-06-03 14:33 ` Andreas Rheinhardt
2025-06-03 14:47 ` softworkz .
2025-06-11 3:05 ` softworkz .
0 siblings, 2 replies; 19+ messages in thread
From: Andreas Rheinhardt @ 2025-06-03 14:33 UTC (permalink / raw)
To: ffmpeg-devel
[-- Attachment #1: Type: text/plain, Size: 1934 bytes --]
softworkz .:
>
>
>> -----Original Message-----
>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
>> Rheinhardt
>> Sent: Montag, 2. Juni 2025 04:39
>> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
>> Subject: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly
>> instead of gzip
>>
>> This is a WIP patchset to further improve the resource manager. The
>> configure bits are the one that need more work: How do we detect zlib
>> (and the required library to link to) on the host system?
>>
>
> Hi,
>
> I ran CI builds for the whole Patchset
> https://github.com/ffstaging/FFmpeg/pull/91
>
>
> Mac build fails (all others are fine)
> https://dev.azure.com/githubsync/ffmpeg/_build/results?buildId=91751&view=logs
>
>
>
> Undefined symbols for architecture x86_64:
> Undefined symbols for architecture x86_64:
> "_ff_vf_yadif_videotoolbox_metallib_len", referenced from:
> _do_init in libavfilter.a[397](vf_yadif_videotoolbox.o)
> ld: symbol(s) not found for architecture x86_64
> clang: error: linker command failed with exit code 1 (use -v to see invocation)
> make: *** [ffprobe_g] Error 1
> "_ff_vf_yadif_videotoolbox_metallib_len", referenced from:
> _do_init in libavfilter.a[397](vf_yadif_videotoolbox.o)
> ld: symbol(s) not found for architecture x86_64
> clang: error: linker command failed with exit code 1 (use -v to see invocation)
> make: *** [ffplay_g] Error 1
>
Thanks for the test. I overlooked that this was also used by metal.
Updated version attached and also available at
https://github.com/mkver/FFmpeg/tree/bin2c. Btw: The non-bin2c patches
are actually logically independent from the bin2c patches and don't need
to wait for them (there will be rebase conflicts when changing the
order, but fixing them would be on me).
The metal bits are untested, but should work.
I have not worked on the configure bits.
- Andreas
[-- Attachment #2: v2-0001-ffbuild-bin2c-Use-zlib-directly-instead-of-gzip.patch --]
[-- Type: text/x-patch, Size: 12817 bytes --]
From b8cfe2dbece9c94b277dd30c212ef6d67599c1fa Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Sun, 1 Jun 2025 22:38:28 +0200
Subject: [PATCH WIP v2 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
Perform zlib compression directly in bin2c.c instead of
using gzip for compression, followed by bin2c to
put the file in the binary. This has the advantage of
avoiding intermediate files and also saves a few bytes
(the gzip header is longer than the zlib header).
It also avoids the hypothetical problem of the build system
supporting gzip with the zlib version used at runtime
not supporting gzip.
But the real advantages will become apparent in future commits.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
.gitignore | 1 -
ffbuild/bin2c.c | 154 ++++++++++++++++++++++++++++++---
ffbuild/common.mak | 44 ++--------
fftools/resources/.gitignore | 3 -
fftools/resources/Makefile | 2 +-
fftools/resources/resman.c | 6 +-
libavfilter/cuda/load_helper.c | 2 +-
7 files changed, 151 insertions(+), 61 deletions(-)
diff --git a/.gitignore b/.gitignore
index 59c89da5e0..901b47e76b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,7 +25,6 @@
*.metallib.c
*.ptx
*.ptx.c
-*.ptx.gz
*_g
\#*
.\#*
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index 91f2d81e5b..21bd5ecb75 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -20,48 +20,174 @@
* DEALINGS IN THE SOFTWARE.
*/
+#include "config.h"
+
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
#include <string.h>
#include <stdio.h>
+#include "libavutil/attributes.h"
+#include "libavutil/macros.h"
+
+#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
+#include <zlib.h>
+
+#define MAX_BUF_SIZE UINT32_MAX
+
+static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *compressed_sizep,
+ FILE *input)
+{
+ z_stream zstream = { 0 };
+ int ret, zret = deflateInit(&zstream, 9);
+
+ if (zret != Z_OK)
+ return -1;
+
+ uint32_t buffer_size = 0, compressed_size = 0;
+ unsigned char *compressed_data = NULL;
+ int flush = Z_NO_FLUSH;
+
+ do {
+ unsigned char tmp[4096];
+ size_t read = fread(tmp, 1, sizeof(tmp), input);
+ if (read < sizeof(tmp))
+ flush = Z_FINISH;
+
+ zstream.next_in = tmp;
+ zstream.avail_in = read;
+
+ do {
+ if (compressed_size >= buffer_size) {
+ if (buffer_size == MAX_BUF_SIZE) {
+ ret = -1;
+ goto fail;
+ }
+
+ buffer_size = buffer_size ? FFMIN(compressed_size * 2ll, MAX_BUF_SIZE) : 4096;
+ void *tmp_ptr = realloc(compressed_data, buffer_size);
+ if (!tmp_ptr) {
+ ret = -1;
+ goto fail;
+ }
+ compressed_data = tmp_ptr;
+ }
+
+ zstream.next_out = compressed_data + compressed_size;
+ zstream.avail_out = buffer_size - compressed_size;
+
+ zret = deflate(&zstream, flush);
+ compressed_size = buffer_size - zstream.avail_out;
+ if (zret == Z_STREAM_END)
+ break;
+ if (zret != Z_OK) {
+ ret = -1;
+ goto fail;
+ }
+ } while (zstream.avail_in > 0 || zstream.avail_out == 0);
+ } while (flush != Z_FINISH);
+
+ deflateEnd(&zstream);
+ *compressed_datap = compressed_data;
+ *compressed_sizep = compressed_size;
+
+ return 0;
+fail:
+ free(compressed_data);
+ deflateEnd(&zstream);
+
+ return ret;
+}
+
+static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep)
+{
+ unsigned char *compressed_data;
+ uint32_t compressed_size;
+
+ int err = read_file_and_compress(&compressed_data, &compressed_size, input);
+ if (err)
+ return err;
+
+ *compressed_sizep = compressed_size;
+
+ for (unsigned i = 0; i < compressed_size; ++i)
+ fprintf(output, "0x%02x, ", compressed_data[i]);
+
+ free(compressed_data);
+
+ return 0;
+}
+#endif
+
int main(int argc, char **argv)
{
const char *name;
FILE *input, *output;
unsigned int length = 0;
unsigned char data;
+ av_unused int compression = 0;
+ int arg_idx = 1;
- if (argc < 3 || argc > 4)
+ if (argc < 3)
return 1;
- input = fopen(argv[1], "rb");
+ if (!strcmp(argv[arg_idx], "--compress")) {
+#if !CONFIG_PTX_COMPRESSION && !CONFIG_RESOURCE_COMPRESSION
+ fprintf(stderr, "Compression unsupported in this configuration. "
+ "This is a bug. Please report it.\n");
+ return -1;
+#endif
+ compression = 1;
+ ++arg_idx;
+ }
+
+ if (argc - arg_idx < 2 || argc - arg_idx > 3)
+ return 1;
+
+ char *input_name = argv[arg_idx++];
+ input = fopen(input_name, "rb");
if (!input)
return -1;
- output = fopen(argv[2], "wb");
+ output = fopen(argv[arg_idx++], "wb");
if (!output) {
fclose(input);
return -1;
}
- if (argc == 4) {
- name = argv[3];
+ if (arg_idx < argc) {
+ name = argv[arg_idx++];
} else {
- size_t arglen = strlen(argv[1]);
- name = argv[1];
+ size_t arglen = strlen(input_name);
+ name = input_name;
for (int i = 0; i < arglen; i++) {
- if (argv[1][i] == '.')
- argv[1][i] = '_';
- else if (argv[1][i] == '/')
- name = &argv[1][i+1];
+ if (input_name[i] == '.')
+ input_name[i] = '_';
+ else if (input_name[i] == '/')
+ name = &input_name[i+1];
}
}
fprintf(output, "const unsigned char ff_%s_data[] = { ", name);
- while (fread(&data, 1, 1, input) > 0) {
- fprintf(output, "0x%02x, ", data);
- length++;
+#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
+ if (compression) {
+ int err = handle_compressed_file(input, output, &length);
+ if (err) {
+ fclose(input);
+ fclose(output);
+ return err;
+ }
+ } else
+#endif
+ {
+ while (fread(&data, 1, 1, input) > 0) {
+ fprintf(output, "0x%02x, ", data);
+ length++;
+ }
}
fprintf(output, "0x00 };\n");
diff --git a/ffbuild/common.mak b/ffbuild/common.mak
index ddf48923ea..b830f9862b 100644
--- a/ffbuild/common.mak
+++ b/ffbuild/common.mak
@@ -113,7 +113,7 @@ COMPILE_LASX = $(call COMPILE,CC,LASXFLAGS)
$(Q)echo '#include "$*.h"' >$@
$(BIN2CEXE): ffbuild/bin2c_host.o
- $(HOSTLD) $(HOSTLDFLAGS) $(HOSTLD_O) $^ $(HOSTEXTRALIBS)
+ $(HOSTLD) $(HOSTLDFLAGS) $(HOSTLD_O) $^ $(HOSTEXTRALIBS) $(if $(CONFIG_PTX_COMPRESSION)$(CONFIG_RESOURCE_COMPRESSION),-lz)
%.metal.air: %.metal
$(METALCC) $< -o $@
@@ -127,17 +127,8 @@ $(BIN2CEXE): ffbuild/bin2c_host.o
%.ptx: %.cu $(SRC_PATH)/compat/cuda/cuda_runtime.h
$(COMPILE_NVCC)
-ifdef CONFIG_PTX_COMPRESSION
-%.ptx.gz: TAG = GZIP
-%.ptx.gz: %.ptx
- $(M)gzip -nc9 $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) >$@
-
-%.ptx.c: %.ptx.gz $(BIN2CEXE)
- $(BIN2C) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
-else
%.ptx.c: %.ptx $(BIN2CEXE)
- $(BIN2C) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
-endif
+ $(BIN2C) $(if $(CONFIG_PTX_COMPRESSION),--compress) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
# 1) Preprocess CSS to a minified version
%.css.min: TAG = SED
@@ -148,36 +139,13 @@ endif
| sed 's/^ //; s/ $$//' \
> $@
-ifdef CONFIG_RESOURCE_COMPRESSION
-
-# 2) Gzip the minified CSS
-%.css.min.gz: TAG = GZIP
-%.css.min.gz: %.css.min
- $(M)gzip -nc9 $< > $@
-
-# 3) Convert the gzipped CSS to a .c array
-%.css.c: %.css.min.gz $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
-
-# 4) Gzip the HTML file (no minification needed)
-%.html.gz: TAG = GZIP
-%.html.gz: %.html
- $(M)gzip -nc9 $< > $@
-
-# 5) Convert the gzipped HTML to a .c array
-%.html.c: %.html.gz $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
-
-else # NO COMPRESSION
-
# 2) Convert the minified CSS to a .c array
%.css.c: %.css.min $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
+ $(BIN2C) $(if $(CONFIG_RESOURCE_COMPRESSION),--compress) $< $@ $(subst .,_,$(basename $(notdir $@)))
# 3) Convert the plain HTML to a .c array
%.html.c: %.html $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
-endif
+ $(BIN2C) $(if $(CONFIG_RESOURCE_COMPRESSION),--compress) $< $@ $(subst .,_,$(basename $(notdir $@)))
clean::
$(RM) $(BIN2CEXE) $(CLEANSUFFIXES:%=ffbuild/%)
@@ -232,7 +200,7 @@ PTXOBJS = $(filter %.ptx.o,$(OBJS))
RESOURCEOBJS = $(filter %.css.o %.html.o,$(OBJS))
$(HOBJS): CCFLAGS += $(CFLAGS_HEADERS)
checkheaders: $(HOBJS)
-.SECONDARY: $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=.gz) $(PTXOBJS:.o=) $(RESOURCEOBJS:.o=.c) $(RESOURCEOBJS:%.css.o=%.css.min) $(RESOURCEOBJS:%.css.o=%.css.min.gz) $(RESOURCEOBJS:%.html.o=%.html.gz) $(RESOURCEOBJS:.o=)
+.SECONDARY: $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=) $(RESOURCEOBJS:.o=.c) $(RESOURCEOBJS:%.css.o=%.css.min) $(RESOURCEOBJS:.o=)
alltools: $(TOOLS)
@@ -252,7 +220,7 @@ $(TOOLOBJS): | tools
OUTDIRS := $(OUTDIRS) $(dir $(OBJS) $(HOBJS) $(HOSTOBJS) $(SHLIBOBJS) $(STLIBOBJS) $(TESTOBJS))
-CLEANSUFFIXES = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.gz *.ptx.c *.ver *.version *.html.gz *.html.c *.css.gz *.css.c *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb
+CLEANSUFFIXES = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.c *.ver *.version *.html.c *.css.c *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb
LIBSUFFIXES = *.a *.lib *.so *.so.* *.dylib *.dll *.def *.dll.a
define RULES
diff --git a/fftools/resources/.gitignore b/fftools/resources/.gitignore
index bda2c59a1c..65febdf13b 100644
--- a/fftools/resources/.gitignore
+++ b/fftools/resources/.gitignore
@@ -1,6 +1,3 @@
*.html.c
*.css.c
-*.html.gz
-*.css.gz
*.min
-*.min.gz
diff --git a/fftools/resources/Makefile b/fftools/resources/Makefile
index 8579a52678..aeb631f206 100644
--- a/fftools/resources/Makefile
+++ b/fftools/resources/Makefile
@@ -5,7 +5,7 @@ vpath %.html $(SRC_PATH)
vpath %.css $(SRC_PATH)
# Uncomment to prevent deletion during build
-#.PRECIOUS: %.css.c %.css.min %.css.gz %.css.min.gz %.html.gz %.html.c
+#.PRECIOUS: %.css.c %.css.min %.html.c
OBJS-resman += \
fftools/resources/resman.o \
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index aa53e96bf4..fdf73c36d5 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -65,7 +65,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
#if CONFIG_RESOURCE_COMPRESSION
-static int decompress_gzip(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
+static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
{
z_stream strm;
unsigned chunk = 65534;
@@ -83,7 +83,7 @@ static int decompress_gzip(ResourceManagerContext *ctx, uint8_t *in, unsigned in
}
// 15 + 16 tells zlib to detect GZIP or zlib automatically
- ret = inflateInit2(&strm, 15 + 16);
+ ret = inflateInit(&strm);
if (ret != Z_OK) {
av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg);
av_free(buf);
@@ -156,7 +156,7 @@ char *ff_resman_get_string(FFResourceId resource_id)
char *out = NULL;
size_t out_len;
- int ret = decompress_gzip(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
+ int ret = decompress_zlib(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index b049ec7130..7c62793077 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -43,7 +43,7 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
uint64_t buf_size;
int ret;
- if (inflateInit2(&stream, 32 + 15) != Z_OK) {
+ if (inflateInit(&stream) != Z_OK) {
av_log(avctx, AV_LOG_ERROR, "Error during zlib initialisation: %s\n", stream.msg);
return AVERROR(ENOSYS);
}
--
2.45.2
[-- Attachment #3: v2-0002-fftools-resources-resman-Don-t-mix-FFmpeg-and-zli.patch --]
[-- Type: text/x-patch, Size: 1225 bytes --]
From 2adb0b7eeaefd50a2f7a300b118f34bafe7bfa9f Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 02:46:58 +0200
Subject: [PATCH WIP v2 02/10] fftools/resources/resman: Don't mix FFmpeg and
zlib errors
Also remove some always-false checks.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index fdf73c36d5..f910535750 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -100,7 +100,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in
av_log(ctx, AV_LOG_ERROR, "Inflate failed: %d, %s\n", ret, strm.msg);
inflateEnd(&strm);
av_free(buf);
- return (ret == Z_STREAM_END) ? Z_OK : ((ret == Z_OK) ? Z_BUF_ERROR : ret);
+ return AVERROR_EXTERNAL;
}
if (strm.avail_out == 0) {
@@ -113,7 +113,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in
inflateEnd(&strm);
*out = (char *)buf;
- return Z_OK;
+ return 0;
}
#endif
--
2.45.2
[-- Attachment #4: v2-0003-fftools-resources-resman-Remove-unused-function-p.patch --]
[-- Type: text/x-patch, Size: 1934 bytes --]
From 264ce990145b9bb7d2cf831ecf48807264dc6bc3 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 02:53:24 +0200
Subject: [PATCH WIP v2 03/10] fftools/resources/resman: Remove unused function
parameter
Also constify in.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index f910535750..99d312bad8 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -65,7 +65,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
#if CONFIG_RESOURCE_COMPRESSION
-static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
+static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out)
{
z_stream strm;
unsigned chunk = 65534;
@@ -108,8 +108,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in
av_log(ctx, AV_LOG_WARNING, "Decompression buffer may be too small\n");
}
- *out_len = chunk - strm.avail_out;
- buf[*out_len] = 0; // Ensure null termination
+ buf[chunk - strm.avail_out] = 0; // Ensure null termination
inflateEnd(&strm);
*out = (char *)buf;
@@ -154,9 +153,8 @@ char *ff_resman_get_string(FFResourceId resource_id)
#if CONFIG_RESOURCE_COMPRESSION
char *out = NULL;
- size_t out_len;
- int ret = decompress_zlib(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
+ int ret = decompress_zlib(ctx, resource_definition.data, *resource_definition.data_len, &out);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
--
2.45.2
[-- Attachment #5: v2-0004-ffbuild-bin2c-Avoid-reallocations-when-uncompress.patch --]
[-- Type: text/x-patch, Size: 8364 bytes --]
From 1d49944d7e1d89d8e2d2f8472b08b69ffa2c1f26 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Sun, 1 Jun 2025 23:19:29 +0200
Subject: [PATCH WIP v2 04/10] ffbuild/bin2c: Avoid reallocations when
uncompressing code,resources
This can be easily achieved by storing the uncompressed size
together with the actual zlib data. It also allows to use
zlib's uncompress() utility function.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
ffbuild/bin2c.c | 26 ++++++++++++---
fftools/resources/resman.c | 44 +++++--------------------
libavfilter/cuda/load_helper.c | 60 ++++++++--------------------------
3 files changed, 43 insertions(+), 87 deletions(-)
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index 21bd5ecb75..b4581f9872 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -38,7 +38,7 @@
#define MAX_BUF_SIZE UINT32_MAX
static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *compressed_sizep,
- FILE *input)
+ uint32_t *uncompressed_sizep, FILE *input)
{
z_stream zstream = { 0 };
int ret, zret = deflateInit(&zstream, 9);
@@ -46,7 +46,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co
if (zret != Z_OK)
return -1;
- uint32_t buffer_size = 0, compressed_size = 0;
+ uint32_t buffer_size = 0, compressed_size = 0, uncompressed_size = 0;
unsigned char *compressed_data = NULL;
int flush = Z_NO_FLUSH;
@@ -56,6 +56,12 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co
if (read < sizeof(tmp))
flush = Z_FINISH;
+ uncompressed_size += read;
+ if (uncompressed_size < read) { // overflow
+ ret = -1;
+ goto fail;
+ }
+
zstream.next_in = tmp;
zstream.avail_in = read;
@@ -92,6 +98,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co
deflateEnd(&zstream);
*compressed_datap = compressed_data;
*compressed_sizep = compressed_size;
+ *uncompressed_sizep = uncompressed_size;
return 0;
fail:
@@ -101,17 +108,28 @@ fail:
return ret;
}
+/// Write a 32bit integer according to the target's endianness.
+static void write_u32(FILE *output, uint32_t num)
+{
+ for (int i = 0; i < 4; ++i)
+ fprintf(output, "0x%02x, ",
+ (num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff);
+}
+
static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep)
{
unsigned char *compressed_data;
- uint32_t compressed_size;
+ uint32_t compressed_size, uncompressed_size;
- int err = read_file_and_compress(&compressed_data, &compressed_size, input);
+ int err = read_file_and_compress(&compressed_data, &compressed_size,
+ &uncompressed_size, input);
if (err)
return err;
*compressed_sizep = compressed_size;
+ write_u32(output, uncompressed_size);
+
for (unsigned i = 0; i < compressed_size; ++i)
fprintf(output, "0x%02x, ", compressed_data[i]);
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 99d312bad8..698529aca1 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -33,9 +33,8 @@
#include "resman.h"
#include "libavutil/avassert.h"
-#include "libavutil/pixdesc.h"
+#include "libavutil/intreadwrite.h"
#include "libavutil/dict.h"
-#include "libavutil/common.h"
extern const unsigned char ff_graph_html_data[];
extern const unsigned int ff_graph_html_len;
@@ -67,50 +66,23 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out)
{
- z_stream strm;
- unsigned chunk = 65534;
- int ret;
- uint8_t *buf;
-
- *out = NULL;
- memset(&strm, 0, sizeof(strm));
-
// Allocate output buffer with extra byte for null termination
- buf = (uint8_t *)av_mallocz(chunk + 1);
+ uint32_t uncompressed_size = AV_RN32(in);
+ uint8_t *buf = av_malloc(uncompressed_size + 1);
if (!buf) {
av_log(ctx, AV_LOG_ERROR, "Failed to allocate decompression buffer\n");
return AVERROR(ENOMEM);
}
-
- // 15 + 16 tells zlib to detect GZIP or zlib automatically
- ret = inflateInit(&strm);
- if (ret != Z_OK) {
- av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg);
- av_free(buf);
- return AVERROR(ENOSYS);
- }
-
- strm.avail_in = in_len;
- strm.next_in = in;
- strm.avail_out = chunk;
- strm.next_out = buf;
-
- ret = inflate(&strm, Z_FINISH);
- if (ret != Z_OK && ret != Z_STREAM_END) {
- av_log(ctx, AV_LOG_ERROR, "Inflate failed: %d, %s\n", ret, strm.msg);
- inflateEnd(&strm);
+ uLongf buf_size = uncompressed_size;
+ int ret = uncompress(buf, &buf_size, in + 4, in_len);
+ if (ret != Z_OK || uncompressed_size != buf_size) {
+ av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret);
av_free(buf);
return AVERROR_EXTERNAL;
}
- if (strm.avail_out == 0) {
- // TODO: Error or loop decoding?
- av_log(ctx, AV_LOG_WARNING, "Decompression buffer may be too small\n");
- }
-
- buf[chunk - strm.avail_out] = 0; // Ensure null termination
+ buf[uncompressed_size] = 0; // Ensure null termination
- inflateEnd(&strm);
*out = (char *)buf;
return 0;
}
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index 7c62793077..0288ea49bb 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -21,11 +21,11 @@
#include "libavutil/hwcontext.h"
#include "libavutil/hwcontext_cuda_internal.h"
#include "libavutil/cuda_check.h"
+#include "libavutil/intreadwrite.h"
#include "libavutil/mem.h"
#if CONFIG_PTX_COMPRESSION
#include <zlib.h>
-#define CHUNK_SIZE 1024 * 64
#endif
#include "load_helper.h"
@@ -38,57 +38,23 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
CudaFunctions *cu = hwctx->internal->cuda_dl;
#if CONFIG_PTX_COMPRESSION
- z_stream stream = { 0 };
- uint8_t *buf, *tmp;
- uint64_t buf_size;
- int ret;
-
- if (inflateInit(&stream) != Z_OK) {
- av_log(avctx, AV_LOG_ERROR, "Error during zlib initialisation: %s\n", stream.msg);
- return AVERROR(ENOSYS);
- }
-
- buf_size = CHUNK_SIZE * 4;
- buf = av_realloc(NULL, buf_size);
- if (!buf) {
- inflateEnd(&stream);
+ uint32_t uncompressed_size = AV_RN32(data);
+ uint8_t *buf = av_realloc(NULL, uncompressed_size + 1);
+ if (!buf)
return AVERROR(ENOMEM);
- }
-
- stream.next_in = data;
- stream.avail_in = length;
-
- do {
- stream.avail_out = buf_size - stream.total_out;
- stream.next_out = buf + stream.total_out;
-
- ret = inflate(&stream, Z_FINISH);
- if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) {
- av_log(avctx, AV_LOG_ERROR, "zlib inflate error(%d): %s\n", ret, stream.msg);
- inflateEnd(&stream);
- av_free(buf);
- return AVERROR(EINVAL);
- }
-
- if (stream.avail_out == 0) {
- buf_size += CHUNK_SIZE;
- tmp = av_realloc(buf, buf_size);
- if (!tmp) {
- inflateEnd(&stream);
- av_free(buf);
- return AVERROR(ENOMEM);
- }
- buf = tmp;
- }
- } while (ret != Z_STREAM_END);
+ uLongf buf_size = uncompressed_size;
+ int ret = uncompress(buf, &buf_size, data + 4, length);
+ if (ret != Z_OK || uncompressed_size != buf_size) {
+ av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret);
+ ret = AVERROR_EXTERNAL;
+ goto fail;
+ }
// NULL-terminate string
- // there is guaranteed to be space for this, due to condition in loop
- buf[stream.total_out] = 0;
-
- inflateEnd(&stream);
+ buf[uncompressed_size] = 0;
ret = CHECK_CU(cu->cuModuleLoadData(cu_module, buf));
+fail:
av_free(buf);
return ret;
#else
--
2.45.2
[-- Attachment #6: v2-0005-ffbuild-bin2c-Store-compressed-size-in-band.patch --]
[-- Type: text/x-patch, Size: 16058 bytes --]
From a24b6e897fc5049e38ca4082033984869625965d Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Sun, 1 Jun 2025 23:37:54 +0200
Subject: [PATCH WIP v2 05/10] ffbuild/bin2c: Store compressed size in-band
There are three types of users of bin2c:
a) Users that want to embed a C-string. The resources for
the fftools as well as cuda modules can be of this type
(if compression is disabled).
b) If compression is not disabled, the same users want to store
compressed data.
c) Users that just want to store arbitrary binary data.
This is currently only the videotoolbox yadif filter
for its metal shaders.
Up until now, bin2c unconditionally created a length field
for the data. Users of type a) didn't need it (and in fact actually
didn't use it); for users of type b), putting the size
of the compressed data in-band (i.e. before the actual compressed data)
is simpler. Users of type c) could currently use either an in-band
size field or an external size field.
This commit changes the cases a) and b) to no longer output
an external length field; in case b) the size is stored in-band instead
(and the data is no longer zero-terminated).
Nothing changes for c), as there is no real benefit in doing so
and I wanted to retain the ability of using external sizes
just in case.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
ffbuild/bin2c.c | 30 +++++++++++++++++++-----------
ffbuild/common.mak | 2 +-
fftools/resources/resman.c | 14 +++++---------
fftools/resources/resman.h | 2 --
libavfilter/cuda/load_helper.c | 4 ++--
libavfilter/cuda/load_helper.h | 2 +-
libavfilter/vf_bilateral_cuda.c | 3 +--
libavfilter/vf_bwdif_cuda.c | 3 +--
libavfilter/vf_chromakey_cuda.c | 3 +--
libavfilter/vf_colorspace_cuda.c | 4 +---
libavfilter/vf_overlay_cuda.c | 3 +--
libavfilter/vf_scale_cuda.c | 3 +--
libavfilter/vf_thumbnail_cuda.c | 3 +--
libavfilter/vf_yadif_cuda.c | 3 +--
14 files changed, 36 insertions(+), 43 deletions(-)
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index b4581f9872..e8643ca792 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -29,7 +29,6 @@
#include <string.h>
#include <stdio.h>
-#include "libavutil/attributes.h"
#include "libavutil/macros.h"
#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
@@ -116,7 +115,7 @@ static void write_u32(FILE *output, uint32_t num)
(num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff);
}
-static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep)
+static int handle_compressed_file(FILE *input, FILE *output)
{
unsigned char *compressed_data;
uint32_t compressed_size, uncompressed_size;
@@ -126,9 +125,8 @@ static int handle_compressed_file(FILE *input, FILE *output, unsigned *compresse
if (err)
return err;
- *compressed_sizep = compressed_size;
-
write_u32(output, uncompressed_size);
+ write_u32(output, compressed_size);
for (unsigned i = 0; i < compressed_size; ++i)
fprintf(output, "0x%02x, ", compressed_data[i]);
@@ -143,10 +141,9 @@ int main(int argc, char **argv)
{
const char *name;
FILE *input, *output;
- unsigned int length = 0;
unsigned char data;
- av_unused int compression = 0;
- int arg_idx = 1;
+ int compression = 0;
+ int arg_idx = 1, external_size = 0;
if (argc < 3)
return 1;
@@ -160,6 +157,14 @@ int main(int argc, char **argv)
compression = 1;
++arg_idx;
}
+ if (!strcmp(argv[arg_idx], "--external-size")) {
+ if (compression) {
+ fprintf(stderr, "Compression and external size make no sense.\n");
+ return -1;
+ }
+ external_size = 1;
+ ++arg_idx;
+ }
if (argc - arg_idx < 2 || argc - arg_idx > 3)
return 1;
@@ -193,24 +198,27 @@ int main(int argc, char **argv)
#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
if (compression) {
- int err = handle_compressed_file(input, output, &length);
+ int err = handle_compressed_file(input, output);
if (err) {
fclose(input);
fclose(output);
return err;
}
+ fprintf(output, "};\n");
} else
#endif
{
+ unsigned int length = 0;
+
while (fread(&data, 1, 1, input) > 0) {
fprintf(output, "0x%02x, ", data);
length++;
}
+ fprintf(output, "0x00 };\n");
+ if (external_size)
+ fprintf(output, "const unsigned int ff_%s_len = %u;\n", name, length);
}
- fprintf(output, "0x00 };\n");
- fprintf(output, "const unsigned int ff_%s_len = %u;\n", name, length);
-
fclose(output);
if (ferror(input) || !feof(input)) {
diff --git a/ffbuild/common.mak b/ffbuild/common.mak
index b830f9862b..021bb521d8 100644
--- a/ffbuild/common.mak
+++ b/ffbuild/common.mak
@@ -122,7 +122,7 @@ $(BIN2CEXE): ffbuild/bin2c_host.o
$(METALLIB) --split-module-without-linking $< -o $@
%.metallib.c: %.metallib $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
+ $(BIN2C) --external-size $< $@ $(subst .,_,$(basename $(notdir $@)))
%.ptx: %.cu $(SRC_PATH)/compat/cuda/cuda_runtime.h
$(COMPILE_NVCC)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 698529aca1..fc00465775 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -37,14 +37,11 @@
#include "libavutil/dict.h"
extern const unsigned char ff_graph_html_data[];
-extern const unsigned int ff_graph_html_len;
-
extern const unsigned char ff_graph_css_data[];
-extern const unsigned ff_graph_css_len;
static const FFResourceDefinition resource_definitions[] = {
- [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", &ff_graph_css_data[0], &ff_graph_css_len },
- [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", &ff_graph_html_data[0], &ff_graph_html_len },
+ [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", ff_graph_css_data },
+ [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", ff_graph_html_data },
};
@@ -64,7 +61,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
#if CONFIG_RESOURCE_COMPRESSION
-static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out)
+static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out)
{
// Allocate output buffer with extra byte for null termination
uint32_t uncompressed_size = AV_RN32(in);
@@ -74,7 +71,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsig
return AVERROR(ENOMEM);
}
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, in + 4, in_len);
+ int ret = uncompress(buf, &buf_size, in + 8, AV_RN32(in + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret);
av_free(buf);
@@ -126,8 +123,7 @@ char *ff_resman_get_string(FFResourceId resource_id)
char *out = NULL;
- int ret = decompress_zlib(ctx, resource_definition.data, *resource_definition.data_len, &out);
-
+ int ret = decompress_zlib(ctx, resource_definition.data, &out);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
goto end;
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index 6485db5091..ad57a10204 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -39,8 +39,6 @@ typedef struct FFResourceDefinition {
const char *name;
const unsigned char *data;
- const unsigned *data_len;
-
} FFResourceDefinition;
void ff_resman_uninit(void);
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index 0288ea49bb..e47f0421ef 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -33,7 +33,7 @@
#define CHECK_CU(x) FF_CUDA_CHECK_DL(avctx, cu, x)
int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_module,
- const unsigned char *data, const unsigned int length)
+ const unsigned char *data)
{
CudaFunctions *cu = hwctx->internal->cuda_dl;
@@ -44,7 +44,7 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
return AVERROR(ENOMEM);
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, data + 4, length);
+ int ret = uncompress(buf, &buf_size, data + 8, AV_RN32(data + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret);
ret = AVERROR_EXTERNAL;
diff --git a/libavfilter/cuda/load_helper.h b/libavfilter/cuda/load_helper.h
index 455bf36a23..d25aa0ae27 100644
--- a/libavfilter/cuda/load_helper.h
+++ b/libavfilter/cuda/load_helper.h
@@ -23,6 +23,6 @@
* Loads a CUDA module and applies any decompression, if necessary.
*/
int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_module,
- const unsigned char *data, const unsigned int length);
+ const unsigned char *data);
#endif /* AVFILTER_CUDA_LOAD_HELPER_H */
diff --git a/libavfilter/vf_bilateral_cuda.c b/libavfilter/vf_bilateral_cuda.c
index 7115fa9e05..50ddaad213 100644
--- a/libavfilter/vf_bilateral_cuda.c
+++ b/libavfilter/vf_bilateral_cuda.c
@@ -217,14 +217,13 @@ static av_cold int cuda_bilateral_load_functions(AVFilterContext *ctx)
int ret;
extern const unsigned char ff_vf_bilateral_cuda_ptx_data[];
- extern const unsigned int ff_vf_bilateral_cuda_ptx_len;
ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx));
if (ret < 0)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_bilateral_cuda_ptx_data, ff_vf_bilateral_cuda_ptx_len);
+ ff_vf_bilateral_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_bwdif_cuda.c b/libavfilter/vf_bwdif_cuda.c
index cc954ab1d3..82c46886f3 100644
--- a/libavfilter/vf_bwdif_cuda.c
+++ b/libavfilter/vf_bwdif_cuda.c
@@ -29,7 +29,6 @@
#include "cuda/load_helper.h"
extern const unsigned char ff_vf_bwdif_cuda_ptx_data[];
-extern const unsigned int ff_vf_bwdif_cuda_ptx_len;
typedef struct DeintCUDAContext {
YADIFContext yadif;
@@ -305,7 +304,7 @@ static int config_output(AVFilterLink *link)
if (ret < 0)
goto exit;
- ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_bwdif_cuda_ptx_data, ff_vf_bwdif_cuda_ptx_len);
+ ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_bwdif_cuda_ptx_data);
if (ret < 0)
goto exit;
diff --git a/libavfilter/vf_chromakey_cuda.c b/libavfilter/vf_chromakey_cuda.c
index 43f50c5a9a..e606da01ac 100644
--- a/libavfilter/vf_chromakey_cuda.c
+++ b/libavfilter/vf_chromakey_cuda.c
@@ -220,14 +220,13 @@ static av_cold int cudachromakey_load_functions(AVFilterContext *ctx)
int ret;
extern const unsigned char ff_vf_chromakey_cuda_ptx_data[];
- extern const unsigned int ff_vf_chromakey_cuda_ptx_len;
ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx));
if (ret < 0)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_chromakey_cuda_ptx_data, ff_vf_chromakey_cuda_ptx_len);
+ ff_vf_chromakey_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_colorspace_cuda.c b/libavfilter/vf_colorspace_cuda.c
index 54d6228cd1..2d922430e4 100644
--- a/libavfilter/vf_colorspace_cuda.c
+++ b/libavfilter/vf_colorspace_cuda.c
@@ -198,15 +198,13 @@ static av_cold int cudacolorspace_load_functions(AVFilterContext* ctx)
int ret;
extern const unsigned char ff_vf_colorspace_cuda_ptx_data[];
- extern const unsigned int ff_vf_colorspace_cuda_ptx_len;
ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx));
if (ret < 0)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_colorspace_cuda_ptx_data,
- ff_vf_colorspace_cuda_ptx_len);
+ ff_vf_colorspace_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_overlay_cuda.c b/libavfilter/vf_overlay_cuda.c
index 4708b34d30..77da5b5e01 100644
--- a/libavfilter/vf_overlay_cuda.c
+++ b/libavfilter/vf_overlay_cuda.c
@@ -415,7 +415,6 @@ static int overlay_cuda_activate(AVFilterContext *avctx)
static int overlay_cuda_config_output(AVFilterLink *outlink)
{
extern const unsigned char ff_vf_overlay_cuda_ptx_data[];
- extern const unsigned int ff_vf_overlay_cuda_ptx_len;
int err;
FilterLink *outl = ff_filter_link(outlink);
@@ -494,7 +493,7 @@ static int overlay_cuda_config_output(AVFilterLink *outlink)
return err;
}
- err = ff_cuda_load_module(ctx, ctx->hwctx, &ctx->cu_module, ff_vf_overlay_cuda_ptx_data, ff_vf_overlay_cuda_ptx_len);
+ err = ff_cuda_load_module(ctx, ctx->hwctx, &ctx->cu_module, ff_vf_overlay_cuda_ptx_data);
if (err < 0) {
CHECK_CU(cu->cuCtxPopCurrent(&dummy));
return err;
diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c
index 44eef207ca..6e238c7c73 100644
--- a/libavfilter/vf_scale_cuda.c
+++ b/libavfilter/vf_scale_cuda.c
@@ -290,7 +290,6 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx)
const char *function_infix = "";
extern const unsigned char ff_vf_scale_cuda_ptx_data[];
- extern const unsigned int ff_vf_scale_cuda_ptx_len;
switch(s->interp_algo) {
case INTERP_ALGO_NEAREST:
@@ -324,7 +323,7 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_scale_cuda_ptx_data, ff_vf_scale_cuda_ptx_len);
+ ff_vf_scale_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_thumbnail_cuda.c b/libavfilter/vf_thumbnail_cuda.c
index 121274de11..bea07f7041 100644
--- a/libavfilter/vf_thumbnail_cuda.c
+++ b/libavfilter/vf_thumbnail_cuda.c
@@ -368,7 +368,6 @@ static int config_props(AVFilterLink *inlink)
int ret;
extern const unsigned char ff_vf_thumbnail_cuda_ptx_data[];
- extern const unsigned int ff_vf_thumbnail_cuda_ptx_len;
s->hwctx = device_hwctx;
s->cu_stream = s->hwctx->stream;
@@ -377,7 +376,7 @@ static int config_props(AVFilterLink *inlink)
if (ret < 0)
return ret;
- ret = ff_cuda_load_module(ctx, device_hwctx, &s->cu_module, ff_vf_thumbnail_cuda_ptx_data, ff_vf_thumbnail_cuda_ptx_len);
+ ret = ff_cuda_load_module(ctx, device_hwctx, &s->cu_module, ff_vf_thumbnail_cuda_ptx_data);
if (ret < 0)
return ret;
diff --git a/libavfilter/vf_yadif_cuda.c b/libavfilter/vf_yadif_cuda.c
index 50ac61ad8a..cd9a5ba2e7 100644
--- a/libavfilter/vf_yadif_cuda.c
+++ b/libavfilter/vf_yadif_cuda.c
@@ -29,7 +29,6 @@
#include "cuda/load_helper.h"
extern const unsigned char ff_vf_yadif_cuda_ptx_data[];
-extern const unsigned int ff_vf_yadif_cuda_ptx_len;
typedef struct DeintCUDAContext {
YADIFContext yadif;
@@ -291,7 +290,7 @@ static int config_output(AVFilterLink *link)
if (ret < 0)
goto exit;
- ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_yadif_cuda_ptx_data, ff_vf_yadif_cuda_ptx_len);
+ ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_yadif_cuda_ptx_data);
if (ret < 0)
goto exit;
--
2.45.2
[-- Attachment #7: v2-0006-ffbuild-bin2c-Allow-to-use-aligned-reads.patch --]
[-- Type: text/x-patch, Size: 4091 bytes --]
From e3d2be1ec0a2b7bb4beebc4c2a3cbede4bbbf436 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 00:08:31 +0200
Subject: [PATCH WIP v2 06/10] ffbuild/bin2c: Allow to use aligned reads
Possible if one aligns the buffer.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
ffbuild/bin2c.c | 10 ++++++----
fftools/resources/resman.c | 4 ++--
libavfilter/cuda/load_helper.c | 4 ++--
3 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index e8643ca792..c380a0ac43 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -115,11 +115,14 @@ static void write_u32(FILE *output, uint32_t num)
(num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff);
}
-static int handle_compressed_file(FILE *input, FILE *output)
+static int handle_compressed_file(FILE *input, FILE *output, const char *name)
{
unsigned char *compressed_data;
uint32_t compressed_size, uncompressed_size;
+ fprintf(output, "#include \"libavutil/mem_internal.h\"\n");
+ fprintf(output, "DECLARE_ALIGNED_4(const unsigned char, ff_%s_data)[] = { ", name);
+
int err = read_file_and_compress(&compressed_data, &compressed_size,
&uncompressed_size, input);
if (err)
@@ -194,11 +197,9 @@ int main(int argc, char **argv)
}
}
- fprintf(output, "const unsigned char ff_%s_data[] = { ", name);
-
#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
if (compression) {
- int err = handle_compressed_file(input, output);
+ int err = handle_compressed_file(input, output, name);
if (err) {
fclose(input);
fclose(output);
@@ -210,6 +211,7 @@ int main(int argc, char **argv)
{
unsigned int length = 0;
+ fprintf(output, "const unsigned char ff_%s_data[] = { ", name);
while (fread(&data, 1, 1, input) > 0) {
fprintf(output, "0x%02x, ", data);
length++;
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index fc00465775..0094ecd4c5 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -64,14 +64,14 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out)
{
// Allocate output buffer with extra byte for null termination
- uint32_t uncompressed_size = AV_RN32(in);
+ uint32_t uncompressed_size = AV_RN32A(in);
uint8_t *buf = av_malloc(uncompressed_size + 1);
if (!buf) {
av_log(ctx, AV_LOG_ERROR, "Failed to allocate decompression buffer\n");
return AVERROR(ENOMEM);
}
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, in + 8, AV_RN32(in + 4));
+ int ret = uncompress(buf, &buf_size, in + 8, AV_RN32A(in + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret);
av_free(buf);
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index e47f0421ef..79ec37c7cd 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -38,13 +38,13 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
CudaFunctions *cu = hwctx->internal->cuda_dl;
#if CONFIG_PTX_COMPRESSION
- uint32_t uncompressed_size = AV_RN32(data);
+ uint32_t uncompressed_size = AV_RN32A(data);
uint8_t *buf = av_realloc(NULL, uncompressed_size + 1);
if (!buf)
return AVERROR(ENOMEM);
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, data + 8, AV_RN32(data + 4));
+ int ret = uncompress(buf, &buf_size, data + 8, AV_RN32A(data + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret);
ret = AVERROR_EXTERNAL;
--
2.45.2
[-- Attachment #8: v2-0007-fftools-resources-resman-Use-pointer-to-resource-.patch --]
[-- Type: text/x-patch, Size: 3380 bytes --]
From 0708e8aa396166bb98b5754de4876f1b129b9e97 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 03:09:45 +0200
Subject: [PATCH WIP v2 07/10] fftools/resources/resman: Use pointer to
resource definition
It does not really matter on its own given that FFResourceDefinition
is so small; it is mostly in preparation for the next commit.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 0094ecd4c5..1de98475c3 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -98,23 +98,23 @@ void ff_resman_uninit(void)
char *ff_resman_get_string(FFResourceId resource_id)
{
ResourceManagerContext *ctx = &resman_ctx;
- FFResourceDefinition resource_definition = { 0 };
+ const FFResourceDefinition *resource_definition = NULL;
AVDictionaryEntry *dic_entry;
char *res = NULL;
for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
- FFResourceDefinition def = resource_definitions[i];
- if (def.resource_id == resource_id) {
+ const FFResourceDefinition *def = &resource_definitions[i];
+ if (def->resource_id == resource_id) {
resource_definition = def;
break;
}
}
- av_assert1(resource_definition.name);
+ av_assert1(resource_definition);
ff_mutex_lock(&mutex);
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition.name, NULL, 0);
+ dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
if (!dic_entry) {
int dict_ret;
@@ -123,13 +123,13 @@ char *ff_resman_get_string(FFResourceId resource_id)
char *out = NULL;
- int ret = decompress_zlib(ctx, resource_definition.data, &out);
+ int ret = decompress_zlib(ctx, resource_definition->data, &out);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
goto end;
}
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition.name, out, 0);
+ dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, out, 0);
if (dict_ret < 0) {
av_log(ctx, AV_LOG_ERROR, "Failed to store decompressed resource in dictionary: %d\n", dict_ret);
av_freep(&out);
@@ -139,14 +139,14 @@ char *ff_resman_get_string(FFResourceId resource_id)
av_freep(&out);
#else
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition.name, (const char *)resource_definition.data, 0);
+ dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, (const char *)resource_definition->data, 0);
if (dict_ret < 0) {
av_log(ctx, AV_LOG_ERROR, "Failed to store resource in dictionary: %d\n", dict_ret);
goto end;
}
#endif
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition.name, NULL, 0);
+ dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
if (!dic_entry) {
av_log(ctx, AV_LOG_ERROR, "Failed to retrieve resource from dictionary after storing it\n");
--
2.45.2
[-- Attachment #9: v2-0008-fftools-resources-resman-Constify-ret-of-ff_resma.patch --]
[-- Type: text/x-patch, Size: 1574 bytes --]
From 975a08f9b988cfc628d1941ae8fec5e6b46ba949 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 03:14:36 +0200
Subject: [PATCH WIP v2 08/10] fftools/resources/resman: Constify ret of
ff_resman_get_string()
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 4 ++--
fftools/resources/resman.h | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 1de98475c3..aa99e07f50 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -95,12 +95,12 @@ void ff_resman_uninit(void)
}
-char *ff_resman_get_string(FFResourceId resource_id)
+const char *ff_resman_get_string(FFResourceId resource_id)
{
ResourceManagerContext *ctx = &resman_ctx;
const FFResourceDefinition *resource_definition = NULL;
AVDictionaryEntry *dic_entry;
- char *res = NULL;
+ const char *res = NULL;
for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
const FFResourceDefinition *def = &resource_definitions[i];
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index ad57a10204..d41c89ce37 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -43,6 +43,6 @@ typedef struct FFResourceDefinition {
void ff_resman_uninit(void);
-char *ff_resman_get_string(FFResourceId resource_id);
+const char *ff_resman_get_string(FFResourceId resource_id);
#endif /* FFTOOLS_RESOURCES_RESMAN_H */
--
2.45.2
[-- Attachment #10: v2-0009-fftools-resources-resman-Avoid-AVDictionary.patch --]
[-- Type: text/x-patch, Size: 5284 bytes --]
From 46afbe31fd5156b6d84da607a6304dfa767147e9 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 03:37:43 +0200
Subject: [PATCH WIP v2 09/10] fftools/resources/resman: Avoid AVDictionary
Using it incurs an unnecessary duplication (temporary when
compression is enabled). It furthermore incurs checks
for av_dict_set()/av_dict_get(). Using an array of pointers
is simpler.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 64 +++++++++++---------------------------
fftools/resources/resman.h | 1 -
2 files changed, 18 insertions(+), 47 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index aa99e07f50..4497d65f3d 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -34,24 +34,23 @@
#include "resman.h"
#include "libavutil/avassert.h"
#include "libavutil/intreadwrite.h"
-#include "libavutil/dict.h"
extern const unsigned char ff_graph_html_data[];
extern const unsigned char ff_graph_css_data[];
static const FFResourceDefinition resource_definitions[] = {
- [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", ff_graph_css_data },
- [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", ff_graph_html_data },
+ [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, ff_graph_css_data },
+ [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, ff_graph_html_data },
};
-
+#if CONFIG_RESOURCE_COMPRESSION
static const AVClass resman_class = {
.class_name = "ResourceManager",
};
typedef struct ResourceManagerContext {
const AVClass *class;
- AVDictionary *resource_dic;
+ char *resources[FF_ARRAY_ELEMS(resource_definitions)];
} ResourceManagerContext;
static AVMutex mutex = AV_MUTEX_INITIALIZER;
@@ -59,8 +58,6 @@ static AVMutex mutex = AV_MUTEX_INITIALIZER;
static ResourceManagerContext resman_ctx = { .class = &resman_class };
-#if CONFIG_RESOURCE_COMPRESSION
-
static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out)
{
// Allocate output buffer with extra byte for null termination
@@ -87,76 +84,51 @@ static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char
void ff_resman_uninit(void)
{
+#if CONFIG_RESOURCE_COMPRESSION
ff_mutex_lock(&mutex);
- av_dict_free(&resman_ctx.resource_dic);
+ for (size_t i = 0; i < FF_ARRAY_ELEMS(resman_ctx.resources); ++i)
+ av_freep(&resman_ctx.resources[i]);
ff_mutex_unlock(&mutex);
+#endif
}
const char *ff_resman_get_string(FFResourceId resource_id)
{
- ResourceManagerContext *ctx = &resman_ctx;
const FFResourceDefinition *resource_definition = NULL;
- AVDictionaryEntry *dic_entry;
const char *res = NULL;
+ av_unused unsigned idx;
for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
const FFResourceDefinition *def = &resource_definitions[i];
if (def->resource_id == resource_id) {
resource_definition = def;
+ idx = i;
break;
}
}
av_assert1(resource_definition);
- ff_mutex_lock(&mutex);
-
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
-
- if (!dic_entry) {
- int dict_ret;
-
#if CONFIG_RESOURCE_COMPRESSION
+ ff_mutex_lock(&mutex);
- char *out = NULL;
+ ResourceManagerContext *ctx = &resman_ctx;
- int ret = decompress_zlib(ctx, resource_definition->data, &out);
+ if (!ctx->resources[idx]) {
+ int ret = decompress_zlib(ctx, resource_definition->data, &ctx->resources[idx]);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
goto end;
}
-
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, out, 0);
- if (dict_ret < 0) {
- av_log(ctx, AV_LOG_ERROR, "Failed to store decompressed resource in dictionary: %d\n", dict_ret);
- av_freep(&out);
- goto end;
- }
-
- av_freep(&out);
-#else
-
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, (const char *)resource_definition->data, 0);
- if (dict_ret < 0) {
- av_log(ctx, AV_LOG_ERROR, "Failed to store resource in dictionary: %d\n", dict_ret);
- goto end;
- }
-
-#endif
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
-
- if (!dic_entry) {
- av_log(ctx, AV_LOG_ERROR, "Failed to retrieve resource from dictionary after storing it\n");
- goto end;
- }
}
-
- res = dic_entry->value;
-
+ res = ctx->resources[idx];
end:
ff_mutex_unlock(&mutex);
+#else
+ res = resource_definition->data;
+#endif
return res;
}
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index d41c89ce37..ee22e960d3 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -36,7 +36,6 @@ typedef enum {
typedef struct FFResourceDefinition {
FFResourceId resource_id;
- const char *name;
const unsigned char *data;
} FFResourceDefinition;
--
2.45.2
[-- Attachment #11: v2-0010-fftools-resources-resman-Improve-headers.patch --]
[-- Type: text/x-patch, Size: 1479 bytes --]
From d9c700993c281d1037e0283d187920e6df0f9885 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Mon, 2 Jun 2025 03:48:50 +0200
Subject: [PATCH WIP v2 10/10] fftools/resources/resman: Improve headers
Don't rely on implicit inclusions of lavu/thread.h and lavu/mem.h.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
fftools/resources/resman.c | 5 ++++-
fftools/resources/resman.h | 8 --------
2 files changed, 4 insertions(+), 9 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 4497d65f3d..9b8f6e406a 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -25,10 +25,13 @@
#include "config.h"
-#include <string.h>
+#include <stddef.h>
#if CONFIG_RESOURCE_COMPRESSION
#include <zlib.h>
+#include "libavutil/intreadwrite.h"
+#include "libavutil/mem.h"
+#include "libavutil/thread.h"
#endif
#include "resman.h"
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index ee22e960d3..20e6b1037c 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -21,14 +21,6 @@
#ifndef FFTOOLS_RESOURCES_RESMAN_H
#define FFTOOLS_RESOURCES_RESMAN_H
-#include <stdint.h>
-
-#include "config.h"
-#include "fftools/ffmpeg.h"
-#include "libavutil/avutil.h"
-#include "libavutil/bprint.h"
-#include "fftools/textformat/avtextformat.h"
-
typedef enum {
FF_RESOURCE_GRAPH_CSS,
FF_RESOURCE_GRAPH_HTML,
--
2.45.2
[-- Attachment #12: Type: text/plain, Size: 251 bytes --]
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-03 14:33 ` Andreas Rheinhardt
@ 2025-06-03 14:47 ` softworkz .
2025-06-11 3:05 ` softworkz .
1 sibling, 0 replies; 19+ messages in thread
From: softworkz . @ 2025-06-03 14:47 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
> Rheinhardt
> Sent: Dienstag, 3. Juni 2025 16:34
> To: ffmpeg-devel@ffmpeg.org
> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly
> instead of gzip
>
> softworkz .:
> >
> >
> >> -----Original Message-----
> >> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
> >> Rheinhardt
> >> Sent: Montag, 2. Juni 2025 04:39
> >> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> >> Subject: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly
> >> instead of gzip
> >>
> >> This is a WIP patchset to further improve the resource manager. The
> >> configure bits are the one that need more work: How do we detect zlib
> >> (and the required library to link to) on the host system?
> >>
> >
> > Hi,
> >
> > I ran CI builds for the whole Patchset
> > https://github.com/ffstaging/FFmpeg/pull/91
> >
> >
> > Mac build fails (all others are fine)
> >
> https://dev.azure.com/githubsync/ffmpeg/_build/results?buildId=91751&view=logs
> >
> >
> >
> > Undefined symbols for architecture x86_64:
> > Undefined symbols for architecture x86_64:
> > "_ff_vf_yadif_videotoolbox_metallib_len", referenced from:
> > _do_init in libavfilter.a[397](vf_yadif_videotoolbox.o)
> > ld: symbol(s) not found for architecture x86_64
> > clang: error: linker command failed with exit code 1 (use -v to see
> invocation)
> > make: *** [ffprobe_g] Error 1
> > "_ff_vf_yadif_videotoolbox_metallib_len", referenced from:
> > _do_init in libavfilter.a[397](vf_yadif_videotoolbox.o)
> > ld: symbol(s) not found for architecture x86_64
> > clang: error: linker command failed with exit code 1 (use -v to see
> invocation)
> > make: *** [ffplay_g] Error 1
> >
>
> Thanks for the test. I overlooked that this was also used by metal.
> Updated version attached and also available at
> https://github.com/mkver/FFmpeg/tree/bin2c.
Thanks, I re-pushed and PRed it here:
https://github.com/ffstaging/FFmpeg/pull/92
(will take up to 1h until all builds are through)
> Btw: The non-bin2c patches
> are actually logically independent from the bin2c patches and don't need
> to wait for them (there will be rebase conflicts when changing the
> order, but fixing them would be on me).
From my side, there's no urgency in this,
it can wait until you got it all together.
> The metal bits are untested, but should work.
I don't have a mac, so I can't test locally. Not sure whether FATE is
including some metal tests? Then I could try enabling HW tests on
the builder..
> I have not worked on the configure bits.
I'm totally clueless in that area, unfortunately 😊
Best
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-03 14:33 ` Andreas Rheinhardt
2025-06-03 14:47 ` softworkz .
@ 2025-06-11 3:05 ` softworkz .
2025-06-16 18:51 ` Andreas Rheinhardt
1 sibling, 1 reply; 19+ messages in thread
From: softworkz . @ 2025-06-11 3:05 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Andreas Rheinhardt
> Sent: Dienstag, 3. Juni 2025 16:34
> To: ffmpeg-devel@ffmpeg.org
> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
> directly instead of gzip
>
Hi Andreas,
thinking about the direction of this patchset, I came to this idea:
How about ditching zlib altogether for this and replace it with some
compact compression code right away?
Something like these might be suitable:
https://github.com/ariya/FastLZ/blob/master/fastlz.c
https://github.com/richgel999/miniz/blob/master/miniz.c
(licenses should be compatible for including the code)
This would solve both problems:
- zlib detection during build
- zlib being unavailable at runtime when linked dynamically
How do you think about this?
I think it wouldn't matter when the compression is slightly less effective..
Best regards
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-11 3:05 ` softworkz .
@ 2025-06-16 18:51 ` Andreas Rheinhardt
2025-06-16 19:15 ` softworkz .
0 siblings, 1 reply; 19+ messages in thread
From: Andreas Rheinhardt @ 2025-06-16 18:51 UTC (permalink / raw)
To: ffmpeg-devel
softworkz .:
>
>
>> -----Original Message-----
>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
>> Andreas Rheinhardt
>> Sent: Dienstag, 3. Juni 2025 16:34
>> To: ffmpeg-devel@ffmpeg.org
>> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
>> directly instead of gzip
>>
>
> Hi Andreas,
>
> thinking about the direction of this patchset, I came to this idea:
>
> How about ditching zlib altogether for this and replace it with some
> compact compression code right away?
>
> Something like these might be suitable:
>
> https://github.com/ariya/FastLZ/blob/master/fastlz.c
> https://github.com/richgel999/miniz/blob/master/miniz.c
> (licenses should be compatible for including the code)
>
>
> This would solve both problems:
>
> - zlib detection during build
> - zlib being unavailable at runtime when linked dynamically
>
>
> How do you think about this?
>
> I think it wouldn't matter when the compression is slightly less effective..
>
The compressed objects will be included in many binaries (and I guess
that most users will never ever use them...). They should therefore be
as small as possible.
But the approach of simply including the code for compression is a good
one. It solves the zlib-detection-during-build problem and the .text
size of the encoding code does not matter given that bin2c only exists
on the dev machine. I'll implement this.
- Andreas
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-16 18:51 ` Andreas Rheinhardt
@ 2025-06-16 19:15 ` softworkz .
2025-06-16 19:24 ` Andreas Rheinhardt
0 siblings, 1 reply; 19+ messages in thread
From: softworkz . @ 2025-06-16 19:15 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Andreas Rheinhardt
> Sent: Monday, June 16, 2025 8:52 PM
> To: ffmpeg-devel@ffmpeg.org
> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
> directly instead of gzip
>
> softworkz .:
> >
> >
> >> -----Original Message-----
> >> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> >> Andreas Rheinhardt
> >> Sent: Dienstag, 3. Juni 2025 16:34
> >> To: ffmpeg-devel@ffmpeg.org
> >> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
> >> directly instead of gzip
> >>
> >
> > Hi Andreas,
> >
> > thinking about the direction of this patchset, I came to this idea:
> >
> > How about ditching zlib altogether for this and replace it with some
> > compact compression code right away?
> >
> > Something like these might be suitable:
> >
> > https://github.com/ariya/FastLZ/blob/master/fastlz.c
> > https://github.com/richgel999/miniz/blob/master/miniz.c
> > (licenses should be compatible for including the code)
> >
> >
> > This would solve both problems:
> >
> > - zlib detection during build
> > - zlib being unavailable at runtime when linked dynamically
> >
> >
> > How do you think about this?
> >
> > I think it wouldn't matter when the compression is slightly less effective..
> >
Hi Andreas,
> The compressed objects will be included in many binaries (and I guess that
> most users will never ever use them...). They should therefore be as small as
> possible.
What I meant is that when it would be like 5 or 10% less in compression ratio,
the benefits would still outweigh. But I suppose you'll get pretty close to zlib
anyway.
> But the approach of simply including the code for compression is a good one.
> It solves the zlib-detection-during-build problem and the .text size of the
> encoding code does not matter given that bin2c only exists on the dev
> machine. I'll implement this.
And - as I just realized after suggesting it - there are even more benefits:
- the dual code paths for handling compressed/non-compressed resources
Will become obsolete
- same for the conditional pattern rules in common.mak
- and finally, the configure parameters --disable-ptx-compression and
--disable-resource-compression can be dropped as well
All in all it will be a big win on several sides. Sounds like a good plan 😊
Thanks,
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-16 19:15 ` softworkz .
@ 2025-06-16 19:24 ` Andreas Rheinhardt
2025-06-16 20:18 ` softworkz .
0 siblings, 1 reply; 19+ messages in thread
From: Andreas Rheinhardt @ 2025-06-16 19:24 UTC (permalink / raw)
To: ffmpeg-devel
softworkz .:
>
>> -----Original Message-----
>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
>> Andreas Rheinhardt
>> Sent: Monday, June 16, 2025 8:52 PM
>> To: ffmpeg-devel@ffmpeg.org
>> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
>> directly instead of gzip
>>
>> softworkz .:
>>>
>>>
>>>> -----Original Message-----
>>>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
>>>> Andreas Rheinhardt
>>>> Sent: Dienstag, 3. Juni 2025 16:34
>>>> To: ffmpeg-devel@ffmpeg.org
>>>> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
>>>> directly instead of gzip
>>>>
>>>
>>> Hi Andreas,
>>>
>>> thinking about the direction of this patchset, I came to this idea:
>>>
>>> How about ditching zlib altogether for this and replace it with some
>>> compact compression code right away?
>>>
>>> Something like these might be suitable:
>>>
>>> https://github.com/ariya/FastLZ/blob/master/fastlz.c
>>> https://github.com/richgel999/miniz/blob/master/miniz.c
>>> (licenses should be compatible for including the code)
>>>
>>>
>>> This would solve both problems:
>>>
>>> - zlib detection during build
>>> - zlib being unavailable at runtime when linked dynamically
>>>
>>>
>>> How do you think about this?
>>>
>>> I think it wouldn't matter when the compression is slightly less effective..
>>>
>
> Hi Andreas,
>
>> The compressed objects will be included in many binaries (and I guess that
>> most users will never ever use them...). They should therefore be as small as
>> possible.
>
> What I meant is that when it would be like 5 or 10% less in compression ratio,
> the benefits would still outweigh. But I suppose you'll get pretty close to zlib
> anyway.
I'll try your tiny compression libaries as well as zlib and brotli and
report back.
>
>> But the approach of simply including the code for compression is a good one.
>> It solves the zlib-detection-during-build problem and the .text size of the
>> encoding code does not matter given that bin2c only exists on the dev
>> machine. I'll implement this.
>
> And - as I just realized after suggesting it - there are even more benefits:
>
> - the dual code paths for handling compressed/non-compressed resources
> Will become obsolete
> - same for the conditional pattern rules in common.mak
> - and finally, the configure parameters --disable-ptx-compression and
> --disable-resource-compression can be dropped as well
I do not intend to remove these options; in fact, we will always have to
keep the codepath for creating uncompressed objects (because otherwise
we would make zlib a hard requirement on the target system).
>
> All in all it will be a big win on several sides. Sounds like a good plan 😊
>
> Thanks,
> sw
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-16 19:24 ` Andreas Rheinhardt
@ 2025-06-16 20:18 ` softworkz .
2025-06-16 22:15 ` Andreas Rheinhardt
` (2 more replies)
0 siblings, 3 replies; 19+ messages in thread
From: softworkz . @ 2025-06-16 20:18 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Andreas Rheinhardt
> Sent: Monday, June 16, 2025 9:24 PM
> To: ffmpeg-devel@ffmpeg.org
> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
> directly instead of gzip
>
> softworkz .:
> >
> >> -----Original Message-----
> >> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> >> Andreas Rheinhardt
> >> Sent: Monday, June 16, 2025 8:52 PM
> >> To: ffmpeg-devel@ffmpeg.org
> >> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
> >> directly instead of gzip
> >>
> >> softworkz .:
> >>>
> >>>
> >>>> -----Original Message-----
> >>>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> >>>> Andreas Rheinhardt
> >>>> Sent: Dienstag, 3. Juni 2025 16:34
> >>>> To: ffmpeg-devel@ffmpeg.org
> >>>> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use
> >>>> zlib directly instead of gzip
> >>>>
> >>>
> >>> Hi Andreas,
> >>>
> >>> thinking about the direction of this patchset, I came to this idea:
> >>>
> >>> How about ditching zlib altogether for this and replace it with some
> >>> compact compression code right away?
> >>>
> >>> Something like these might be suitable:
> >>>
> >>> https://github.com/ariya/FastLZ/blob/master/fastlz.c
> >>> https://github.com/richgel999/miniz/blob/master/miniz.c
> >>> (licenses should be compatible for including the code)
> >>>
> >>>
> >>> This would solve both problems:
> >>>
> >>> - zlib detection during build
> >>> - zlib being unavailable at runtime when linked dynamically
> >>>
> >>>
> >>> How do you think about this?
> >>>
> >>> I think it wouldn't matter when the compression is slightly less effective..
> >>>
> >
> > Hi Andreas,
> >
> >> The compressed objects will be included in many binaries (and I guess
> >> that most users will never ever use them...). They should therefore
> >> be as small as possible.
> >
> > What I meant is that when it would be like 5 or 10% less in
> > compression ratio, the benefits would still outweigh. But I suppose
> > you'll get pretty close to zlib anyway.
>
> I'll try your tiny compression libaries as well as zlib and brotli and report back.
Awesome!
> >> But the approach of simply including the code for compression is a good
> one.
> >> It solves the zlib-detection-during-build problem and the .text size
> >> of the encoding code does not matter given that bin2c only exists on
> >> the dev machine. I'll implement this.
> >
> > And - as I just realized after suggesting it - there are even more benefits:
> >
> > - the dual code paths for handling compressed/non-compressed resources
> > Will become obsolete
> > - same for the conditional pattern rules in common.mak
> > - and finally, the configure parameters --disable-ptx-compression and
> > --disable-resource-compression can be dropped as well
>
> I do not intend to remove these options; in fact, we will always have to keep
> the codepath for creating uncompressed objects (because otherwise we
> would make zlib a hard requirement on the target system).
But why that? If we include the decompression code (that's why I've been
searching for the most compact implementations available with compatible
licenses), then we won't have any zlib dependency anymore - neither at build-
nor at run-time.
Thanks,
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-16 20:18 ` softworkz .
@ 2025-06-16 22:15 ` Andreas Rheinhardt
2025-06-17 9:29 ` Nicolas George
2025-06-17 14:41 ` James Almer
2 siblings, 0 replies; 19+ messages in thread
From: Andreas Rheinhardt @ 2025-06-16 22:15 UTC (permalink / raw)
To: ffmpeg-devel
softworkz .:
>
>
>> -----Original Message-----
>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
>> Andreas Rheinhardt
>> Sent: Monday, June 16, 2025 9:24 PM
>> To: ffmpeg-devel@ffmpeg.org
>> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
>> directly instead of gzip
>>
>> softworkz .:
>>>
>>>> -----Original Message-----
>>>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
>>>> Andreas Rheinhardt
>>>> Sent: Monday, June 16, 2025 8:52 PM
>>>> To: ffmpeg-devel@ffmpeg.org
>>>> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
>>>> directly instead of gzip
>>>>
>>>> softworkz .:
>>>>>
>>>>>
>>>>>> -----Original Message-----
>>>>>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
>>>>>> Andreas Rheinhardt
>>>>>> Sent: Dienstag, 3. Juni 2025 16:34
>>>>>> To: ffmpeg-devel@ffmpeg.org
>>>>>> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use
>>>>>> zlib directly instead of gzip
>>>>>>
>>>>>
>>>>> Hi Andreas,
>>>>>
>>>>> thinking about the direction of this patchset, I came to this idea:
>>>>>
>>>>> How about ditching zlib altogether for this and replace it with some
>>>>> compact compression code right away?
>>>>>
>>>>> Something like these might be suitable:
>>>>>
>>>>> https://github.com/ariya/FastLZ/blob/master/fastlz.c
>>>>> https://github.com/richgel999/miniz/blob/master/miniz.c
>>>>> (licenses should be compatible for including the code)
>>>>>
>>>>>
>>>>> This would solve both problems:
>>>>>
>>>>> - zlib detection during build
>>>>> - zlib being unavailable at runtime when linked dynamically
>>>>>
>>>>>
>>>>> How do you think about this?
>>>>>
>>>>> I think it wouldn't matter when the compression is slightly less effective..
>>>>>
>>>
>>> Hi Andreas,
>>>
>>>> The compressed objects will be included in many binaries (and I guess
>>>> that most users will never ever use them...). They should therefore
>>>> be as small as possible.
>>>
>>> What I meant is that when it would be like 5 or 10% less in
>>> compression ratio, the benefits would still outweigh. But I suppose
>>> you'll get pretty close to zlib anyway.
>>
>> I'll try your tiny compression libaries as well as zlib and brotli and report back.
>
> Awesome!
>
>>>> But the approach of simply including the code for compression is a good
>> one.
>>>> It solves the zlib-detection-during-build problem and the .text size
>>>> of the encoding code does not matter given that bin2c only exists on
>>>> the dev machine. I'll implement this.
>>>
>>> And - as I just realized after suggesting it - there are even more benefits:
>>>
>>> - the dual code paths for handling compressed/non-compressed resources
>>> Will become obsolete
>>> - same for the conditional pattern rules in common.mak
>>> - and finally, the configure parameters --disable-ptx-compression and
>>> --disable-resource-compression can be dropped as well
>>
>> I do not intend to remove these options; in fact, we will always have to keep
>> the codepath for creating uncompressed objects (because otherwise we
>> would make zlib a hard requirement on the target system).
>
> But why that? If we include the decompression code (that's why I've been
> searching for the most compact implementations available with compatible
> licenses), then we won't have any zlib dependency anymore - neither at build-
> nor at run-time.
>
zlib availability is not an onerous requirement (and in our case not
even a hard requirement); several components already have such a
depedency (e.g. Matroska allows zlib compression and files using it are
common, png (both decoder and encoder) requires zlib, the http(s)
protocol requires it for zlib handling).
If we include the decompression code, then this code exists in every
tool (and in libavfilter). If said code also performs worse
(compression-wise) than zlib/brotli, then this is a big no for me.
- Andreas
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-16 20:18 ` softworkz .
2025-06-16 22:15 ` Andreas Rheinhardt
@ 2025-06-17 9:29 ` Nicolas George
2025-06-17 13:23 ` softworkz .
2025-06-17 14:41 ` James Almer
2 siblings, 1 reply; 19+ messages in thread
From: Nicolas George @ 2025-06-17 9:29 UTC (permalink / raw)
To: FFmpeg development discussions and patches
softworkz . (HE12025-06-16):
> But why that? If we include the decompression code (that's why I've been
> searching for the most compact implementations available with compatible
> licenses), then we won't have any zlib dependency anymore - neither at build-
> nor at run-time.
Wait. What you want to do is not offer the option to link with a
different implementation of decompression feature but import the code of
a certain library into our source tree?
We do not do that.
If somebody would write an implementation of zlib specifically for
ffmpeg, using av_malloc() and av_log() and all our utility API, and if
that implementation had some kind of benefit over any existing
implementation, then we would be happy to adopt it.
But dumping somebody else's code into our source tree, increasing our
maintenance burden, just to avoid linking with system zlib…
WTF? Only somebody who does not understand what FFmpeg stands for would
suggest that.
--
Nicolas George
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-17 9:29 ` Nicolas George
@ 2025-06-17 13:23 ` softworkz .
2025-06-17 14:09 ` Nicolas George
0 siblings, 1 reply; 19+ messages in thread
From: softworkz . @ 2025-06-17 13:23 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Nicolas George
> Sent: Tuesday, June 17, 2025 11:30 AM
> To: FFmpeg development discussions and patches <ffmpeg-
> devel@ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
> directly instead of gzip
>
> softworkz . (HE12025-06-16):
> > But why that? If we include the decompression code (that's why I've
> > been searching for the most compact implementations available with
> > compatible licenses), then we won't have any zlib dependency anymore -
> > neither at build- nor at run-time.
>
> Wait. What you want to do is not offer the option to link with a different
> implementation of decompression feature but import the code of a certain
> library into our source tree?
>
> We do not do that.
>
> If somebody would write an implementation of zlib specifically for ffmpeg,
> using av_malloc() and av_log() and all our utility API, and if that
> implementation had some kind of benefit over any existing implementation,
> then we would be happy to adopt it.
It's not about replacing zlib in general. The scope is having our own compression
code for ptx and resource compression only. I gave a number of examples for
how this could possibly be achieved with a small amount of code - which of
course would need to be adapted to match our style and procedures.
The compression side would be in bin2c and only the decompression code
would ship (e.g. in avutil). Only if that part would be really small, the benefits
would weigh out the size added by the decomp code.
As long as we don't know the compression ratios achievable with such low-code
compression routines, this is merely an idea which might not work out at all
but at least worth exploring IMO.
Anyway, I'm not surprised. Making wrong assumptions and then ranting about,
is one of your most typical patterns.
Best regards,
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-17 13:23 ` softworkz .
@ 2025-06-17 14:09 ` Nicolas George
2025-06-17 14:35 ` softworkz .
0 siblings, 1 reply; 19+ messages in thread
From: Nicolas George @ 2025-06-17 14:09 UTC (permalink / raw)
To: FFmpeg development discussions and patches
softworkz . (HE12025-06-17):
> It's not about replacing zlib in general. The scope is having our own compression
> code for ptx and resource compression only.
This is what I understood. So let me tell it once again more concisely:
If you can write a zlib replacement specifically for FFmpeg with some
technical benefit over other implementations, do so.
I you propose to take somebody else's code that was not written for
FFmpeg, go away.
--
Nicolas George
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-17 14:09 ` Nicolas George
@ 2025-06-17 14:35 ` softworkz .
2025-06-17 14:40 ` Nicolas George
0 siblings, 1 reply; 19+ messages in thread
From: softworkz . @ 2025-06-17 14:35 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Nicolas George
> Sent: Tuesday, June 17, 2025 4:09 PM
> To: FFmpeg development discussions and patches <ffmpeg-
> devel@ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
> directly instead of gzip
>
> softworkz . (HE12025-06-17):
> > It's not about replacing zlib in general. The scope is having our own
> > compression code for ptx and resource compression only.
>
> This is what I understood.
But obviously you didn't:
> If you can write a zlib replacement specifically for FFmpeg with some technical
> benefit over other implementations, do so.
I didn't talk about a zlib replacement. It is about any kind of compression code,
not needing to be zlib compatible at all.
> go away.
Likewise!
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-17 14:35 ` softworkz .
@ 2025-06-17 14:40 ` Nicolas George
0 siblings, 0 replies; 19+ messages in thread
From: Nicolas George @ 2025-06-17 14:40 UTC (permalink / raw)
To: FFmpeg development discussions and patches
softworkz . (HE12025-06-17):
> I didn't talk about a zlib replacement. It is about any kind of
> compression code, not needing to be zlib compatible at all.
I know you did not talk about it. I am trying to get you to understand
the only case that would be acceptable.
Once again, in uppercase in case it helps you read it:
WE DO NOT DUMP OTHER PEOPLE'S CODE NOT WRITTEN FOR FFMPEG INTO OUR TREE.
--
Nicolas George
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-16 20:18 ` softworkz .
2025-06-16 22:15 ` Andreas Rheinhardt
2025-06-17 9:29 ` Nicolas George
@ 2025-06-17 14:41 ` James Almer
2025-06-17 14:51 ` softworkz .
2025-06-18 7:09 ` Nicolas George
2 siblings, 2 replies; 19+ messages in thread
From: James Almer @ 2025-06-17 14:41 UTC (permalink / raw)
To: ffmpeg-devel
[-- Attachment #1.1.1: Type: text/plain, Size: 477 bytes --]
On 6/16/2025 5:18 PM, softworkz . wrote:
> But why that? If we include the decompression code (that's why I've been
> searching for the most compact implementations available with compatible
> licenses), then we won't have any zlib dependency anymore - neither at build-
> nor at run-time.
We don't add some random project's code into our own tree to remove an
external dependency. But you can adapt it to behave as a proper lavu
module, or write one from scratch.
[-- Attachment #1.2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]
[-- Attachment #2: Type: text/plain, Size: 251 bytes --]
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-17 14:41 ` James Almer
@ 2025-06-17 14:51 ` softworkz .
2025-06-18 7:09 ` Nicolas George
1 sibling, 0 replies; 19+ messages in thread
From: softworkz . @ 2025-06-17 14:51 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> James Almer
> Sent: Tuesday, June 17, 2025 4:41 PM
> To: ffmpeg-devel@ffmpeg.org
> Subject: Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib
> directly instead of gzip
>
> On 6/16/2025 5:18 PM, softworkz . wrote:
> > But why that? If we include the decompression code (that's why I've
> > been searching for the most compact implementations available with
> > compatible licenses), then we won't have any zlib dependency anymore -
> > neither at build- nor at run-time.
> We don't add some random project's code into our own tree to remove an
> external dependency. But you can adapt it to behave as a proper lavu module,
> or write one from scratch.
Hi James,
yea, the former is what I meant.
I was a bit unclear when saying "we won't have any zlib dependency anymore",
I rather meant to say "we won't have any zlib dependency anymore for ptx and
resource compression". Targeting those specific cases only might allow for a much
simpler compression code.
Thanks,
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
2025-06-17 14:41 ` James Almer
2025-06-17 14:51 ` softworkz .
@ 2025-06-18 7:09 ` Nicolas George
1 sibling, 0 replies; 19+ messages in thread
From: Nicolas George @ 2025-06-18 7:09 UTC (permalink / raw)
To: FFmpeg development discussions and patches
James Almer (HE12025-06-17):
> But you can adapt it to behave as a proper lavu module,
> or write one from scratch.
Do you know of a precedent of something like that happening: somebody
taking external code and properly turning it into FFmpeg code?
Doing the cosmetic change so that the foreign code looks like FFmpeg
code is easy. Understanding the foreign code enough to maintain as part
of our project is a much harder task. We should not let somebody
undertake it just because they were able to achieve the easy task.
Regards,
--
Nicolas George
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 19+ messages in thread
end of thread, other threads:[~2025-06-18 7:10 UTC | newest]
Thread overview: 19+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2025-06-02 2:38 [FFmpeg-devel] [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip Andreas Rheinhardt
2025-06-02 3:22 ` softworkz .
2025-06-02 22:33 ` softworkz .
2025-06-03 14:33 ` Andreas Rheinhardt
2025-06-03 14:47 ` softworkz .
2025-06-11 3:05 ` softworkz .
2025-06-16 18:51 ` Andreas Rheinhardt
2025-06-16 19:15 ` softworkz .
2025-06-16 19:24 ` Andreas Rheinhardt
2025-06-16 20:18 ` softworkz .
2025-06-16 22:15 ` Andreas Rheinhardt
2025-06-17 9:29 ` Nicolas George
2025-06-17 13:23 ` softworkz .
2025-06-17 14:09 ` Nicolas George
2025-06-17 14:35 ` softworkz .
2025-06-17 14:40 ` Nicolas George
2025-06-17 14:41 ` James Almer
2025-06-17 14:51 ` softworkz .
2025-06-18 7:09 ` Nicolas George
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git