From 1d49944d7e1d89d8e2d2f8472b08b69ffa2c1f26 Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt Date: Sun, 1 Jun 2025 23:19:29 +0200 Subject: [PATCH WIP v2 04/10] ffbuild/bin2c: Avoid reallocations when uncompressing code,resources This can be easily achieved by storing the uncompressed size together with the actual zlib data. It also allows to use zlib's uncompress() utility function. Signed-off-by: Andreas Rheinhardt --- ffbuild/bin2c.c | 26 ++++++++++++--- fftools/resources/resman.c | 44 +++++-------------------- libavfilter/cuda/load_helper.c | 60 ++++++++-------------------------- 3 files changed, 43 insertions(+), 87 deletions(-) diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c index 21bd5ecb75..b4581f9872 100644 --- a/ffbuild/bin2c.c +++ b/ffbuild/bin2c.c @@ -38,7 +38,7 @@ #define MAX_BUF_SIZE UINT32_MAX static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *compressed_sizep, - FILE *input) + uint32_t *uncompressed_sizep, FILE *input) { z_stream zstream = { 0 }; int ret, zret = deflateInit(&zstream, 9); @@ -46,7 +46,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co if (zret != Z_OK) return -1; - uint32_t buffer_size = 0, compressed_size = 0; + uint32_t buffer_size = 0, compressed_size = 0, uncompressed_size = 0; unsigned char *compressed_data = NULL; int flush = Z_NO_FLUSH; @@ -56,6 +56,12 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co if (read < sizeof(tmp)) flush = Z_FINISH; + uncompressed_size += read; + if (uncompressed_size < read) { // overflow + ret = -1; + goto fail; + } + zstream.next_in = tmp; zstream.avail_in = read; @@ -92,6 +98,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co deflateEnd(&zstream); *compressed_datap = compressed_data; *compressed_sizep = compressed_size; + *uncompressed_sizep = uncompressed_size; return 0; fail: @@ -101,17 +108,28 @@ fail: return ret; } +/// Write a 32bit integer according to the target's endianness. +static void write_u32(FILE *output, uint32_t num) +{ + for (int i = 0; i < 4; ++i) + fprintf(output, "0x%02x, ", + (num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff); +} + static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep) { unsigned char *compressed_data; - uint32_t compressed_size; + uint32_t compressed_size, uncompressed_size; - int err = read_file_and_compress(&compressed_data, &compressed_size, input); + int err = read_file_and_compress(&compressed_data, &compressed_size, + &uncompressed_size, input); if (err) return err; *compressed_sizep = compressed_size; + write_u32(output, uncompressed_size); + for (unsigned i = 0; i < compressed_size; ++i) fprintf(output, "0x%02x, ", compressed_data[i]); diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c index 99d312bad8..698529aca1 100644 --- a/fftools/resources/resman.c +++ b/fftools/resources/resman.c @@ -33,9 +33,8 @@ #include "resman.h" #include "libavutil/avassert.h" -#include "libavutil/pixdesc.h" +#include "libavutil/intreadwrite.h" #include "libavutil/dict.h" -#include "libavutil/common.h" extern const unsigned char ff_graph_html_data[]; extern const unsigned int ff_graph_html_len; @@ -67,50 +66,23 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class }; static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out) { - z_stream strm; - unsigned chunk = 65534; - int ret; - uint8_t *buf; - - *out = NULL; - memset(&strm, 0, sizeof(strm)); - // Allocate output buffer with extra byte for null termination - buf = (uint8_t *)av_mallocz(chunk + 1); + uint32_t uncompressed_size = AV_RN32(in); + uint8_t *buf = av_malloc(uncompressed_size + 1); if (!buf) { av_log(ctx, AV_LOG_ERROR, "Failed to allocate decompression buffer\n"); return AVERROR(ENOMEM); } - - // 15 + 16 tells zlib to detect GZIP or zlib automatically - ret = inflateInit(&strm); - if (ret != Z_OK) { - av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg); - av_free(buf); - return AVERROR(ENOSYS); - } - - strm.avail_in = in_len; - strm.next_in = in; - strm.avail_out = chunk; - strm.next_out = buf; - - ret = inflate(&strm, Z_FINISH); - if (ret != Z_OK && ret != Z_STREAM_END) { - av_log(ctx, AV_LOG_ERROR, "Inflate failed: %d, %s\n", ret, strm.msg); - inflateEnd(&strm); + uLongf buf_size = uncompressed_size; + int ret = uncompress(buf, &buf_size, in + 4, in_len); + if (ret != Z_OK || uncompressed_size != buf_size) { + av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret); av_free(buf); return AVERROR_EXTERNAL; } - if (strm.avail_out == 0) { - // TODO: Error or loop decoding? - av_log(ctx, AV_LOG_WARNING, "Decompression buffer may be too small\n"); - } - - buf[chunk - strm.avail_out] = 0; // Ensure null termination + buf[uncompressed_size] = 0; // Ensure null termination - inflateEnd(&strm); *out = (char *)buf; return 0; } diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c index 7c62793077..0288ea49bb 100644 --- a/libavfilter/cuda/load_helper.c +++ b/libavfilter/cuda/load_helper.c @@ -21,11 +21,11 @@ #include "libavutil/hwcontext.h" #include "libavutil/hwcontext_cuda_internal.h" #include "libavutil/cuda_check.h" +#include "libavutil/intreadwrite.h" #include "libavutil/mem.h" #if CONFIG_PTX_COMPRESSION #include -#define CHUNK_SIZE 1024 * 64 #endif #include "load_helper.h" @@ -38,57 +38,23 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo CudaFunctions *cu = hwctx->internal->cuda_dl; #if CONFIG_PTX_COMPRESSION - z_stream stream = { 0 }; - uint8_t *buf, *tmp; - uint64_t buf_size; - int ret; - - if (inflateInit(&stream) != Z_OK) { - av_log(avctx, AV_LOG_ERROR, "Error during zlib initialisation: %s\n", stream.msg); - return AVERROR(ENOSYS); - } - - buf_size = CHUNK_SIZE * 4; - buf = av_realloc(NULL, buf_size); - if (!buf) { - inflateEnd(&stream); + uint32_t uncompressed_size = AV_RN32(data); + uint8_t *buf = av_realloc(NULL, uncompressed_size + 1); + if (!buf) return AVERROR(ENOMEM); - } - - stream.next_in = data; - stream.avail_in = length; - - do { - stream.avail_out = buf_size - stream.total_out; - stream.next_out = buf + stream.total_out; - - ret = inflate(&stream, Z_FINISH); - if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) { - av_log(avctx, AV_LOG_ERROR, "zlib inflate error(%d): %s\n", ret, stream.msg); - inflateEnd(&stream); - av_free(buf); - return AVERROR(EINVAL); - } - - if (stream.avail_out == 0) { - buf_size += CHUNK_SIZE; - tmp = av_realloc(buf, buf_size); - if (!tmp) { - inflateEnd(&stream); - av_free(buf); - return AVERROR(ENOMEM); - } - buf = tmp; - } - } while (ret != Z_STREAM_END); + uLongf buf_size = uncompressed_size; + int ret = uncompress(buf, &buf_size, data + 4, length); + if (ret != Z_OK || uncompressed_size != buf_size) { + av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret); + ret = AVERROR_EXTERNAL; + goto fail; + } // NULL-terminate string - // there is guaranteed to be space for this, due to condition in loop - buf[stream.total_out] = 0; - - inflateEnd(&stream); + buf[uncompressed_size] = 0; ret = CHECK_CU(cu->cuModuleLoadData(cu_module, buf)); +fail: av_free(buf); return ret; #else -- 2.45.2