From 38436c69fc818b989594fb9912d4e99d8c4f3d1b Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt Date: Sun, 1 Jun 2025 23:19:29 +0200 Subject: [PATCH WIP 04/10] ffbuild/bin2c: Avoid reallocations when uncompressing code,resources This can be easily achieved by storing the uncompressed size together with the actual zlib data. It also allows to use zlib's uncompress() utility function. Signed-off-by: Andreas Rheinhardt --- ffbuild/bin2c.c | 26 ++++++++++++--- fftools/resources/resman.c | 44 +++++-------------------- libavfilter/cuda/load_helper.c | 60 ++++++++-------------------------- 3 files changed, 43 insertions(+), 87 deletions(-) diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c index a3e49f0a11..7bcb126c7e 100644 --- a/ffbuild/bin2c.c +++ b/ffbuild/bin2c.c @@ -38,7 +38,7 @@ #define MAX_BUF_SIZE UINT32_MAX static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *compressed_sizep, - FILE *input) + uint32_t *uncompressed_sizep, FILE *input) { z_stream zstream = { 0 }; int ret, zret = deflateInit(&zstream, 9); @@ -46,7 +46,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co if (zret != Z_OK) return -1; - uint32_t buffer_size = 0, compressed_size = 0; + uint32_t buffer_size = 0, compressed_size = 0, uncompressed_size = 0; unsigned char *compressed_data = NULL; int flush = Z_NO_FLUSH; @@ -56,6 +56,12 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co if (read < sizeof(tmp)) flush = Z_FINISH; + uncompressed_size += read; + if (uncompressed_size < read) { // overflow + ret = -1; + goto fail; + } + zstream.next_in = tmp; zstream.avail_in = read; @@ -92,6 +98,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co deflateEnd(&zstream); *compressed_datap = compressed_data; *compressed_sizep = compressed_size; + *uncompressed_sizep = uncompressed_size; return 0; fail: @@ -101,17 +108,28 @@ fail: return ret; } +/// Write a 32bit integer according to the target's endianness. +static void write_u32(FILE *output, uint32_t num) +{ + for (int i = 0; i < 4; ++i) + fprintf(output, "0x%02x, ", + (num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff); +} + static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep) { unsigned char *compressed_data; - uint32_t compressed_size; + uint32_t compressed_size, uncompressed_size; - int err = read_file_and_compress(&compressed_data, &compressed_size, input); + int err = read_file_and_compress(&compressed_data, &compressed_size, + &uncompressed_size, input); if (err) return err; *compressed_sizep = compressed_size; + write_u32(output, uncompressed_size); + for (unsigned i = 0; i < compressed_size; ++i) fprintf(output, "0x%02x, ", compressed_data[i]); diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c index 99d312bad8..698529aca1 100644 --- a/fftools/resources/resman.c +++ b/fftools/resources/resman.c @@ -33,9 +33,8 @@ #include "resman.h" #include "libavutil/avassert.h" -#include "libavutil/pixdesc.h" +#include "libavutil/intreadwrite.h" #include "libavutil/dict.h" -#include "libavutil/common.h" extern const unsigned char ff_graph_html_data[]; extern const unsigned int ff_graph_html_len; @@ -67,50 +66,23 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class }; static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out) { - z_stream strm; - unsigned chunk = 65534; - int ret; - uint8_t *buf; - - *out = NULL; - memset(&strm, 0, sizeof(strm)); - // Allocate output buffer with extra byte for null termination - buf = (uint8_t *)av_mallocz(chunk + 1); + uint32_t uncompressed_size = AV_RN32(in); + uint8_t *buf = av_malloc(uncompressed_size + 1); if (!buf) { av_log(ctx, AV_LOG_ERROR, "Failed to allocate decompression buffer\n"); return AVERROR(ENOMEM); } - - // 15 + 16 tells zlib to detect GZIP or zlib automatically - ret = inflateInit(&strm); - if (ret != Z_OK) { - av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg); - av_free(buf); - return AVERROR(ENOSYS); - } - - strm.avail_in = in_len; - strm.next_in = in; - strm.avail_out = chunk; - strm.next_out = buf; - - ret = inflate(&strm, Z_FINISH); - if (ret != Z_OK && ret != Z_STREAM_END) { - av_log(ctx, AV_LOG_ERROR, "Inflate failed: %d, %s\n", ret, strm.msg); - inflateEnd(&strm); + uLongf buf_size = uncompressed_size; + int ret = uncompress(buf, &buf_size, in + 4, in_len); + if (ret != Z_OK || uncompressed_size != buf_size) { + av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret); av_free(buf); return AVERROR_EXTERNAL; } - if (strm.avail_out == 0) { - // TODO: Error or loop decoding? - av_log(ctx, AV_LOG_WARNING, "Decompression buffer may be too small\n"); - } - - buf[chunk - strm.avail_out] = 0; // Ensure null termination + buf[uncompressed_size] = 0; // Ensure null termination - inflateEnd(&strm); *out = (char *)buf; return 0; } diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c index 7c62793077..0288ea49bb 100644 --- a/libavfilter/cuda/load_helper.c +++ b/libavfilter/cuda/load_helper.c @@ -21,11 +21,11 @@ #include "libavutil/hwcontext.h" #include "libavutil/hwcontext_cuda_internal.h" #include "libavutil/cuda_check.h" +#include "libavutil/intreadwrite.h" #include "libavutil/mem.h" #if CONFIG_PTX_COMPRESSION #include -#define CHUNK_SIZE 1024 * 64 #endif #include "load_helper.h" @@ -38,57 +38,23 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo CudaFunctions *cu = hwctx->internal->cuda_dl; #if CONFIG_PTX_COMPRESSION - z_stream stream = { 0 }; - uint8_t *buf, *tmp; - uint64_t buf_size; - int ret; - - if (inflateInit(&stream) != Z_OK) { - av_log(avctx, AV_LOG_ERROR, "Error during zlib initialisation: %s\n", stream.msg); - return AVERROR(ENOSYS); - } - - buf_size = CHUNK_SIZE * 4; - buf = av_realloc(NULL, buf_size); - if (!buf) { - inflateEnd(&stream); + uint32_t uncompressed_size = AV_RN32(data); + uint8_t *buf = av_realloc(NULL, uncompressed_size + 1); + if (!buf) return AVERROR(ENOMEM); - } - - stream.next_in = data; - stream.avail_in = length; - - do { - stream.avail_out = buf_size - stream.total_out; - stream.next_out = buf + stream.total_out; - - ret = inflate(&stream, Z_FINISH); - if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) { - av_log(avctx, AV_LOG_ERROR, "zlib inflate error(%d): %s\n", ret, stream.msg); - inflateEnd(&stream); - av_free(buf); - return AVERROR(EINVAL); - } - - if (stream.avail_out == 0) { - buf_size += CHUNK_SIZE; - tmp = av_realloc(buf, buf_size); - if (!tmp) { - inflateEnd(&stream); - av_free(buf); - return AVERROR(ENOMEM); - } - buf = tmp; - } - } while (ret != Z_STREAM_END); + uLongf buf_size = uncompressed_size; + int ret = uncompress(buf, &buf_size, data + 4, length); + if (ret != Z_OK || uncompressed_size != buf_size) { + av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret); + ret = AVERROR_EXTERNAL; + goto fail; + } // NULL-terminate string - // there is guaranteed to be space for this, due to condition in loop - buf[stream.total_out] = 0; - - inflateEnd(&stream); + buf[uncompressed_size] = 0; ret = CHECK_CU(cu->cuModuleLoadData(cu_module, buf)); +fail: av_free(buf); return ret; #else -- 2.45.2