From a24b6e897fc5049e38ca4082033984869625965d Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt Date: Sun, 1 Jun 2025 23:37:54 +0200 Subject: [PATCH WIP v2 05/10] ffbuild/bin2c: Store compressed size in-band There are three types of users of bin2c: a) Users that want to embed a C-string. The resources for the fftools as well as cuda modules can be of this type (if compression is disabled). b) If compression is not disabled, the same users want to store compressed data. c) Users that just want to store arbitrary binary data. This is currently only the videotoolbox yadif filter for its metal shaders. Up until now, bin2c unconditionally created a length field for the data. Users of type a) didn't need it (and in fact actually didn't use it); for users of type b), putting the size of the compressed data in-band (i.e. before the actual compressed data) is simpler. Users of type c) could currently use either an in-band size field or an external size field. This commit changes the cases a) and b) to no longer output an external length field; in case b) the size is stored in-band instead (and the data is no longer zero-terminated). Nothing changes for c), as there is no real benefit in doing so and I wanted to retain the ability of using external sizes just in case. Signed-off-by: Andreas Rheinhardt --- ffbuild/bin2c.c | 30 +++++++++++++++++++----------- ffbuild/common.mak | 2 +- fftools/resources/resman.c | 14 +++++--------- fftools/resources/resman.h | 2 -- libavfilter/cuda/load_helper.c | 4 ++-- libavfilter/cuda/load_helper.h | 2 +- libavfilter/vf_bilateral_cuda.c | 3 +-- libavfilter/vf_bwdif_cuda.c | 3 +-- libavfilter/vf_chromakey_cuda.c | 3 +-- libavfilter/vf_colorspace_cuda.c | 4 +--- libavfilter/vf_overlay_cuda.c | 3 +-- libavfilter/vf_scale_cuda.c | 3 +-- libavfilter/vf_thumbnail_cuda.c | 3 +-- libavfilter/vf_yadif_cuda.c | 3 +-- 14 files changed, 36 insertions(+), 43 deletions(-) diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c index b4581f9872..e8643ca792 100644 --- a/ffbuild/bin2c.c +++ b/ffbuild/bin2c.c @@ -29,7 +29,6 @@ #include #include -#include "libavutil/attributes.h" #include "libavutil/macros.h" #if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION @@ -116,7 +115,7 @@ static void write_u32(FILE *output, uint32_t num) (num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff); } -static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep) +static int handle_compressed_file(FILE *input, FILE *output) { unsigned char *compressed_data; uint32_t compressed_size, uncompressed_size; @@ -126,9 +125,8 @@ static int handle_compressed_file(FILE *input, FILE *output, unsigned *compresse if (err) return err; - *compressed_sizep = compressed_size; - write_u32(output, uncompressed_size); + write_u32(output, compressed_size); for (unsigned i = 0; i < compressed_size; ++i) fprintf(output, "0x%02x, ", compressed_data[i]); @@ -143,10 +141,9 @@ int main(int argc, char **argv) { const char *name; FILE *input, *output; - unsigned int length = 0; unsigned char data; - av_unused int compression = 0; - int arg_idx = 1; + int compression = 0; + int arg_idx = 1, external_size = 0; if (argc < 3) return 1; @@ -160,6 +157,14 @@ int main(int argc, char **argv) compression = 1; ++arg_idx; } + if (!strcmp(argv[arg_idx], "--external-size")) { + if (compression) { + fprintf(stderr, "Compression and external size make no sense.\n"); + return -1; + } + external_size = 1; + ++arg_idx; + } if (argc - arg_idx < 2 || argc - arg_idx > 3) return 1; @@ -193,24 +198,27 @@ int main(int argc, char **argv) #if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION if (compression) { - int err = handle_compressed_file(input, output, &length); + int err = handle_compressed_file(input, output); if (err) { fclose(input); fclose(output); return err; } + fprintf(output, "};\n"); } else #endif { + unsigned int length = 0; + while (fread(&data, 1, 1, input) > 0) { fprintf(output, "0x%02x, ", data); length++; } + fprintf(output, "0x00 };\n"); + if (external_size) + fprintf(output, "const unsigned int ff_%s_len = %u;\n", name, length); } - fprintf(output, "0x00 };\n"); - fprintf(output, "const unsigned int ff_%s_len = %u;\n", name, length); - fclose(output); if (ferror(input) || !feof(input)) { diff --git a/ffbuild/common.mak b/ffbuild/common.mak index b830f9862b..021bb521d8 100644 --- a/ffbuild/common.mak +++ b/ffbuild/common.mak @@ -122,7 +122,7 @@ $(BIN2CEXE): ffbuild/bin2c_host.o $(METALLIB) --split-module-without-linking $< -o $@ %.metallib.c: %.metallib $(BIN2CEXE) - $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@))) + $(BIN2C) --external-size $< $@ $(subst .,_,$(basename $(notdir $@))) %.ptx: %.cu $(SRC_PATH)/compat/cuda/cuda_runtime.h $(COMPILE_NVCC) diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c index 698529aca1..fc00465775 100644 --- a/fftools/resources/resman.c +++ b/fftools/resources/resman.c @@ -37,14 +37,11 @@ #include "libavutil/dict.h" extern const unsigned char ff_graph_html_data[]; -extern const unsigned int ff_graph_html_len; - extern const unsigned char ff_graph_css_data[]; -extern const unsigned ff_graph_css_len; static const FFResourceDefinition resource_definitions[] = { - [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", &ff_graph_css_data[0], &ff_graph_css_len }, - [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", &ff_graph_html_data[0], &ff_graph_html_len }, + [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", ff_graph_css_data }, + [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", ff_graph_html_data }, }; @@ -64,7 +61,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class }; #if CONFIG_RESOURCE_COMPRESSION -static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out) +static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out) { // Allocate output buffer with extra byte for null termination uint32_t uncompressed_size = AV_RN32(in); @@ -74,7 +71,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsig return AVERROR(ENOMEM); } uLongf buf_size = uncompressed_size; - int ret = uncompress(buf, &buf_size, in + 4, in_len); + int ret = uncompress(buf, &buf_size, in + 8, AV_RN32(in + 4)); if (ret != Z_OK || uncompressed_size != buf_size) { av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret); av_free(buf); @@ -126,8 +123,7 @@ char *ff_resman_get_string(FFResourceId resource_id) char *out = NULL; - int ret = decompress_zlib(ctx, resource_definition.data, *resource_definition.data_len, &out); - + int ret = decompress_zlib(ctx, resource_definition.data, &out); if (ret) { av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id); goto end; diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h index 6485db5091..ad57a10204 100644 --- a/fftools/resources/resman.h +++ b/fftools/resources/resman.h @@ -39,8 +39,6 @@ typedef struct FFResourceDefinition { const char *name; const unsigned char *data; - const unsigned *data_len; - } FFResourceDefinition; void ff_resman_uninit(void); diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c index 0288ea49bb..e47f0421ef 100644 --- a/libavfilter/cuda/load_helper.c +++ b/libavfilter/cuda/load_helper.c @@ -33,7 +33,7 @@ #define CHECK_CU(x) FF_CUDA_CHECK_DL(avctx, cu, x) int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_module, - const unsigned char *data, const unsigned int length) + const unsigned char *data) { CudaFunctions *cu = hwctx->internal->cuda_dl; @@ -44,7 +44,7 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo return AVERROR(ENOMEM); uLongf buf_size = uncompressed_size; - int ret = uncompress(buf, &buf_size, data + 4, length); + int ret = uncompress(buf, &buf_size, data + 8, AV_RN32(data + 4)); if (ret != Z_OK || uncompressed_size != buf_size) { av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret); ret = AVERROR_EXTERNAL; diff --git a/libavfilter/cuda/load_helper.h b/libavfilter/cuda/load_helper.h index 455bf36a23..d25aa0ae27 100644 --- a/libavfilter/cuda/load_helper.h +++ b/libavfilter/cuda/load_helper.h @@ -23,6 +23,6 @@ * Loads a CUDA module and applies any decompression, if necessary. */ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_module, - const unsigned char *data, const unsigned int length); + const unsigned char *data); #endif /* AVFILTER_CUDA_LOAD_HELPER_H */ diff --git a/libavfilter/vf_bilateral_cuda.c b/libavfilter/vf_bilateral_cuda.c index 7115fa9e05..50ddaad213 100644 --- a/libavfilter/vf_bilateral_cuda.c +++ b/libavfilter/vf_bilateral_cuda.c @@ -217,14 +217,13 @@ static av_cold int cuda_bilateral_load_functions(AVFilterContext *ctx) int ret; extern const unsigned char ff_vf_bilateral_cuda_ptx_data[]; - extern const unsigned int ff_vf_bilateral_cuda_ptx_len; ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx)); if (ret < 0) return ret; ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, - ff_vf_bilateral_cuda_ptx_data, ff_vf_bilateral_cuda_ptx_len); + ff_vf_bilateral_cuda_ptx_data); if (ret < 0) goto fail; diff --git a/libavfilter/vf_bwdif_cuda.c b/libavfilter/vf_bwdif_cuda.c index cc954ab1d3..82c46886f3 100644 --- a/libavfilter/vf_bwdif_cuda.c +++ b/libavfilter/vf_bwdif_cuda.c @@ -29,7 +29,6 @@ #include "cuda/load_helper.h" extern const unsigned char ff_vf_bwdif_cuda_ptx_data[]; -extern const unsigned int ff_vf_bwdif_cuda_ptx_len; typedef struct DeintCUDAContext { YADIFContext yadif; @@ -305,7 +304,7 @@ static int config_output(AVFilterLink *link) if (ret < 0) goto exit; - ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_bwdif_cuda_ptx_data, ff_vf_bwdif_cuda_ptx_len); + ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_bwdif_cuda_ptx_data); if (ret < 0) goto exit; diff --git a/libavfilter/vf_chromakey_cuda.c b/libavfilter/vf_chromakey_cuda.c index 43f50c5a9a..e606da01ac 100644 --- a/libavfilter/vf_chromakey_cuda.c +++ b/libavfilter/vf_chromakey_cuda.c @@ -220,14 +220,13 @@ static av_cold int cudachromakey_load_functions(AVFilterContext *ctx) int ret; extern const unsigned char ff_vf_chromakey_cuda_ptx_data[]; - extern const unsigned int ff_vf_chromakey_cuda_ptx_len; ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx)); if (ret < 0) return ret; ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, - ff_vf_chromakey_cuda_ptx_data, ff_vf_chromakey_cuda_ptx_len); + ff_vf_chromakey_cuda_ptx_data); if (ret < 0) goto fail; diff --git a/libavfilter/vf_colorspace_cuda.c b/libavfilter/vf_colorspace_cuda.c index 54d6228cd1..2d922430e4 100644 --- a/libavfilter/vf_colorspace_cuda.c +++ b/libavfilter/vf_colorspace_cuda.c @@ -198,15 +198,13 @@ static av_cold int cudacolorspace_load_functions(AVFilterContext* ctx) int ret; extern const unsigned char ff_vf_colorspace_cuda_ptx_data[]; - extern const unsigned int ff_vf_colorspace_cuda_ptx_len; ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx)); if (ret < 0) return ret; ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, - ff_vf_colorspace_cuda_ptx_data, - ff_vf_colorspace_cuda_ptx_len); + ff_vf_colorspace_cuda_ptx_data); if (ret < 0) goto fail; diff --git a/libavfilter/vf_overlay_cuda.c b/libavfilter/vf_overlay_cuda.c index 4708b34d30..77da5b5e01 100644 --- a/libavfilter/vf_overlay_cuda.c +++ b/libavfilter/vf_overlay_cuda.c @@ -415,7 +415,6 @@ static int overlay_cuda_activate(AVFilterContext *avctx) static int overlay_cuda_config_output(AVFilterLink *outlink) { extern const unsigned char ff_vf_overlay_cuda_ptx_data[]; - extern const unsigned int ff_vf_overlay_cuda_ptx_len; int err; FilterLink *outl = ff_filter_link(outlink); @@ -494,7 +493,7 @@ static int overlay_cuda_config_output(AVFilterLink *outlink) return err; } - err = ff_cuda_load_module(ctx, ctx->hwctx, &ctx->cu_module, ff_vf_overlay_cuda_ptx_data, ff_vf_overlay_cuda_ptx_len); + err = ff_cuda_load_module(ctx, ctx->hwctx, &ctx->cu_module, ff_vf_overlay_cuda_ptx_data); if (err < 0) { CHECK_CU(cu->cuCtxPopCurrent(&dummy)); return err; diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 44eef207ca..6e238c7c73 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -290,7 +290,6 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx) const char *function_infix = ""; extern const unsigned char ff_vf_scale_cuda_ptx_data[]; - extern const unsigned int ff_vf_scale_cuda_ptx_len; switch(s->interp_algo) { case INTERP_ALGO_NEAREST: @@ -324,7 +323,7 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx) return ret; ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, - ff_vf_scale_cuda_ptx_data, ff_vf_scale_cuda_ptx_len); + ff_vf_scale_cuda_ptx_data); if (ret < 0) goto fail; diff --git a/libavfilter/vf_thumbnail_cuda.c b/libavfilter/vf_thumbnail_cuda.c index 121274de11..bea07f7041 100644 --- a/libavfilter/vf_thumbnail_cuda.c +++ b/libavfilter/vf_thumbnail_cuda.c @@ -368,7 +368,6 @@ static int config_props(AVFilterLink *inlink) int ret; extern const unsigned char ff_vf_thumbnail_cuda_ptx_data[]; - extern const unsigned int ff_vf_thumbnail_cuda_ptx_len; s->hwctx = device_hwctx; s->cu_stream = s->hwctx->stream; @@ -377,7 +376,7 @@ static int config_props(AVFilterLink *inlink) if (ret < 0) return ret; - ret = ff_cuda_load_module(ctx, device_hwctx, &s->cu_module, ff_vf_thumbnail_cuda_ptx_data, ff_vf_thumbnail_cuda_ptx_len); + ret = ff_cuda_load_module(ctx, device_hwctx, &s->cu_module, ff_vf_thumbnail_cuda_ptx_data); if (ret < 0) return ret; diff --git a/libavfilter/vf_yadif_cuda.c b/libavfilter/vf_yadif_cuda.c index 50ac61ad8a..cd9a5ba2e7 100644 --- a/libavfilter/vf_yadif_cuda.c +++ b/libavfilter/vf_yadif_cuda.c @@ -29,7 +29,6 @@ #include "cuda/load_helper.h" extern const unsigned char ff_vf_yadif_cuda_ptx_data[]; -extern const unsigned int ff_vf_yadif_cuda_ptx_len; typedef struct DeintCUDAContext { YADIFContext yadif; @@ -291,7 +290,7 @@ static int config_output(AVFilterLink *link) if (ret < 0) goto exit; - ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_yadif_cuda_ptx_data, ff_vf_yadif_cuda_ptx_len); + ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_yadif_cuda_ptx_data); if (ret < 0) goto exit; -- 2.45.2