From b8cfe2dbece9c94b277dd30c212ef6d67599c1fa Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt Date: Sun, 1 Jun 2025 22:38:28 +0200 Subject: [PATCH WIP v2 01/10] ffbuild/bin2c: Use zlib directly instead of gzip Perform zlib compression directly in bin2c.c instead of using gzip for compression, followed by bin2c to put the file in the binary. This has the advantage of avoiding intermediate files and also saves a few bytes (the gzip header is longer than the zlib header). It also avoids the hypothetical problem of the build system supporting gzip with the zlib version used at runtime not supporting gzip. But the real advantages will become apparent in future commits. Signed-off-by: Andreas Rheinhardt --- .gitignore | 1 - ffbuild/bin2c.c | 154 ++++++++++++++++++++++++++++++--- ffbuild/common.mak | 44 ++-------- fftools/resources/.gitignore | 3 - fftools/resources/Makefile | 2 +- fftools/resources/resman.c | 6 +- libavfilter/cuda/load_helper.c | 2 +- 7 files changed, 151 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index 59c89da5e0..901b47e76b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ *.metallib.c *.ptx *.ptx.c -*.ptx.gz *_g \#* .\#* diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c index 91f2d81e5b..21bd5ecb75 100644 --- a/ffbuild/bin2c.c +++ b/ffbuild/bin2c.c @@ -20,48 +20,174 @@ * DEALINGS IN THE SOFTWARE. */ +#include "config.h" + +#include +#include +#include +#include #include #include +#include "libavutil/attributes.h" +#include "libavutil/macros.h" + +#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION +#include + +#define MAX_BUF_SIZE UINT32_MAX + +static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *compressed_sizep, + FILE *input) +{ + z_stream zstream = { 0 }; + int ret, zret = deflateInit(&zstream, 9); + + if (zret != Z_OK) + return -1; + + uint32_t buffer_size = 0, compressed_size = 0; + unsigned char *compressed_data = NULL; + int flush = Z_NO_FLUSH; + + do { + unsigned char tmp[4096]; + size_t read = fread(tmp, 1, sizeof(tmp), input); + if (read < sizeof(tmp)) + flush = Z_FINISH; + + zstream.next_in = tmp; + zstream.avail_in = read; + + do { + if (compressed_size >= buffer_size) { + if (buffer_size == MAX_BUF_SIZE) { + ret = -1; + goto fail; + } + + buffer_size = buffer_size ? FFMIN(compressed_size * 2ll, MAX_BUF_SIZE) : 4096; + void *tmp_ptr = realloc(compressed_data, buffer_size); + if (!tmp_ptr) { + ret = -1; + goto fail; + } + compressed_data = tmp_ptr; + } + + zstream.next_out = compressed_data + compressed_size; + zstream.avail_out = buffer_size - compressed_size; + + zret = deflate(&zstream, flush); + compressed_size = buffer_size - zstream.avail_out; + if (zret == Z_STREAM_END) + break; + if (zret != Z_OK) { + ret = -1; + goto fail; + } + } while (zstream.avail_in > 0 || zstream.avail_out == 0); + } while (flush != Z_FINISH); + + deflateEnd(&zstream); + *compressed_datap = compressed_data; + *compressed_sizep = compressed_size; + + return 0; +fail: + free(compressed_data); + deflateEnd(&zstream); + + return ret; +} + +static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep) +{ + unsigned char *compressed_data; + uint32_t compressed_size; + + int err = read_file_and_compress(&compressed_data, &compressed_size, input); + if (err) + return err; + + *compressed_sizep = compressed_size; + + for (unsigned i = 0; i < compressed_size; ++i) + fprintf(output, "0x%02x, ", compressed_data[i]); + + free(compressed_data); + + return 0; +} +#endif + int main(int argc, char **argv) { const char *name; FILE *input, *output; unsigned int length = 0; unsigned char data; + av_unused int compression = 0; + int arg_idx = 1; - if (argc < 3 || argc > 4) + if (argc < 3) return 1; - input = fopen(argv[1], "rb"); + if (!strcmp(argv[arg_idx], "--compress")) { +#if !CONFIG_PTX_COMPRESSION && !CONFIG_RESOURCE_COMPRESSION + fprintf(stderr, "Compression unsupported in this configuration. " + "This is a bug. Please report it.\n"); + return -1; +#endif + compression = 1; + ++arg_idx; + } + + if (argc - arg_idx < 2 || argc - arg_idx > 3) + return 1; + + char *input_name = argv[arg_idx++]; + input = fopen(input_name, "rb"); if (!input) return -1; - output = fopen(argv[2], "wb"); + output = fopen(argv[arg_idx++], "wb"); if (!output) { fclose(input); return -1; } - if (argc == 4) { - name = argv[3]; + if (arg_idx < argc) { + name = argv[arg_idx++]; } else { - size_t arglen = strlen(argv[1]); - name = argv[1]; + size_t arglen = strlen(input_name); + name = input_name; for (int i = 0; i < arglen; i++) { - if (argv[1][i] == '.') - argv[1][i] = '_'; - else if (argv[1][i] == '/') - name = &argv[1][i+1]; + if (input_name[i] == '.') + input_name[i] = '_'; + else if (input_name[i] == '/') + name = &input_name[i+1]; } } fprintf(output, "const unsigned char ff_%s_data[] = { ", name); - while (fread(&data, 1, 1, input) > 0) { - fprintf(output, "0x%02x, ", data); - length++; +#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION + if (compression) { + int err = handle_compressed_file(input, output, &length); + if (err) { + fclose(input); + fclose(output); + return err; + } + } else +#endif + { + while (fread(&data, 1, 1, input) > 0) { + fprintf(output, "0x%02x, ", data); + length++; + } } fprintf(output, "0x00 };\n"); diff --git a/ffbuild/common.mak b/ffbuild/common.mak index ddf48923ea..b830f9862b 100644 --- a/ffbuild/common.mak +++ b/ffbuild/common.mak @@ -113,7 +113,7 @@ COMPILE_LASX = $(call COMPILE,CC,LASXFLAGS) $(Q)echo '#include "$*.h"' >$@ $(BIN2CEXE): ffbuild/bin2c_host.o - $(HOSTLD) $(HOSTLDFLAGS) $(HOSTLD_O) $^ $(HOSTEXTRALIBS) + $(HOSTLD) $(HOSTLDFLAGS) $(HOSTLD_O) $^ $(HOSTEXTRALIBS) $(if $(CONFIG_PTX_COMPRESSION)$(CONFIG_RESOURCE_COMPRESSION),-lz) %.metal.air: %.metal $(METALCC) $< -o $@ @@ -127,17 +127,8 @@ $(BIN2CEXE): ffbuild/bin2c_host.o %.ptx: %.cu $(SRC_PATH)/compat/cuda/cuda_runtime.h $(COMPILE_NVCC) -ifdef CONFIG_PTX_COMPRESSION -%.ptx.gz: TAG = GZIP -%.ptx.gz: %.ptx - $(M)gzip -nc9 $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) >$@ - -%.ptx.c: %.ptx.gz $(BIN2CEXE) - $(BIN2C) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@))) -else %.ptx.c: %.ptx $(BIN2CEXE) - $(BIN2C) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@))) -endif + $(BIN2C) $(if $(CONFIG_PTX_COMPRESSION),--compress) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@))) # 1) Preprocess CSS to a minified version %.css.min: TAG = SED @@ -148,36 +139,13 @@ endif | sed 's/^ //; s/ $$//' \ > $@ -ifdef CONFIG_RESOURCE_COMPRESSION - -# 2) Gzip the minified CSS -%.css.min.gz: TAG = GZIP -%.css.min.gz: %.css.min - $(M)gzip -nc9 $< > $@ - -# 3) Convert the gzipped CSS to a .c array -%.css.c: %.css.min.gz $(BIN2CEXE) - $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@))) - -# 4) Gzip the HTML file (no minification needed) -%.html.gz: TAG = GZIP -%.html.gz: %.html - $(M)gzip -nc9 $< > $@ - -# 5) Convert the gzipped HTML to a .c array -%.html.c: %.html.gz $(BIN2CEXE) - $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@))) - -else # NO COMPRESSION - # 2) Convert the minified CSS to a .c array %.css.c: %.css.min $(BIN2CEXE) - $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@))) + $(BIN2C) $(if $(CONFIG_RESOURCE_COMPRESSION),--compress) $< $@ $(subst .,_,$(basename $(notdir $@))) # 3) Convert the plain HTML to a .c array %.html.c: %.html $(BIN2CEXE) - $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@))) -endif + $(BIN2C) $(if $(CONFIG_RESOURCE_COMPRESSION),--compress) $< $@ $(subst .,_,$(basename $(notdir $@))) clean:: $(RM) $(BIN2CEXE) $(CLEANSUFFIXES:%=ffbuild/%) @@ -232,7 +200,7 @@ PTXOBJS = $(filter %.ptx.o,$(OBJS)) RESOURCEOBJS = $(filter %.css.o %.html.o,$(OBJS)) $(HOBJS): CCFLAGS += $(CFLAGS_HEADERS) checkheaders: $(HOBJS) -.SECONDARY: $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=.gz) $(PTXOBJS:.o=) $(RESOURCEOBJS:.o=.c) $(RESOURCEOBJS:%.css.o=%.css.min) $(RESOURCEOBJS:%.css.o=%.css.min.gz) $(RESOURCEOBJS:%.html.o=%.html.gz) $(RESOURCEOBJS:.o=) +.SECONDARY: $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=) $(RESOURCEOBJS:.o=.c) $(RESOURCEOBJS:%.css.o=%.css.min) $(RESOURCEOBJS:.o=) alltools: $(TOOLS) @@ -252,7 +220,7 @@ $(TOOLOBJS): | tools OUTDIRS := $(OUTDIRS) $(dir $(OBJS) $(HOBJS) $(HOSTOBJS) $(SHLIBOBJS) $(STLIBOBJS) $(TESTOBJS)) -CLEANSUFFIXES = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.gz *.ptx.c *.ver *.version *.html.gz *.html.c *.css.gz *.css.c *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb +CLEANSUFFIXES = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.c *.ver *.version *.html.c *.css.c *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb LIBSUFFIXES = *.a *.lib *.so *.so.* *.dylib *.dll *.def *.dll.a define RULES diff --git a/fftools/resources/.gitignore b/fftools/resources/.gitignore index bda2c59a1c..65febdf13b 100644 --- a/fftools/resources/.gitignore +++ b/fftools/resources/.gitignore @@ -1,6 +1,3 @@ *.html.c *.css.c -*.html.gz -*.css.gz *.min -*.min.gz diff --git a/fftools/resources/Makefile b/fftools/resources/Makefile index 8579a52678..aeb631f206 100644 --- a/fftools/resources/Makefile +++ b/fftools/resources/Makefile @@ -5,7 +5,7 @@ vpath %.html $(SRC_PATH) vpath %.css $(SRC_PATH) # Uncomment to prevent deletion during build -#.PRECIOUS: %.css.c %.css.min %.css.gz %.css.min.gz %.html.gz %.html.c +#.PRECIOUS: %.css.c %.css.min %.html.c OBJS-resman += \ fftools/resources/resman.o \ diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c index aa53e96bf4..fdf73c36d5 100644 --- a/fftools/resources/resman.c +++ b/fftools/resources/resman.c @@ -65,7 +65,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class }; #if CONFIG_RESOURCE_COMPRESSION -static int decompress_gzip(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len) +static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len) { z_stream strm; unsigned chunk = 65534; @@ -83,7 +83,7 @@ static int decompress_gzip(ResourceManagerContext *ctx, uint8_t *in, unsigned in } // 15 + 16 tells zlib to detect GZIP or zlib automatically - ret = inflateInit2(&strm, 15 + 16); + ret = inflateInit(&strm); if (ret != Z_OK) { av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg); av_free(buf); @@ -156,7 +156,7 @@ char *ff_resman_get_string(FFResourceId resource_id) char *out = NULL; size_t out_len; - int ret = decompress_gzip(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len); + int ret = decompress_zlib(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len); if (ret) { av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id); diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c index b049ec7130..7c62793077 100644 --- a/libavfilter/cuda/load_helper.c +++ b/libavfilter/cuda/load_helper.c @@ -43,7 +43,7 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo uint64_t buf_size; int ret; - if (inflateInit2(&stream, 32 + 15) != Z_OK) { + if (inflateInit(&stream) != Z_OK) { av_log(avctx, AV_LOG_ERROR, "Error during zlib initialisation: %s\n", stream.msg); return AVERROR(ENOSYS); } -- 2.45.2