Re: [FFmpeg-devel] [PATCH 4/4] Revert "fftools/resources: Add resource manager files with build-time compression"

[Kacper Michajlow <kasper93-at-gmail.com@ffmpeg.org>]

On Sat, 17 May 2025 at 01:11, Marton Balint <cus@passwd.hu> wrote:
>
> This reverts commit 517a8055655798970d94a4c5ea912511362520ea.
>
> Building resources directly in the ffmpeg util needs quite a bit of code,
> increases binary size and makes it harder for the users to change those
> resources at will.
>
> The only user of this, the mermaid and mermaidhtml graph formats were converted
> in an erlier commit to use file loading from the FFMPEG data dirs similar to
> ffpreset files. Therefore the infrastucture to build resources directly into
> ffmpeg CLI tools can be removed now.
>
> Signed-off-by: Marton Balint <cus@passwd.hu>
> ---
>  configure                    |   5 -
>  ffbuild/common.mak           |  43 +------
>  fftools/Makefile             |   5 +-
>  fftools/resources/.gitignore |   4 -
>  fftools/resources/Makefile   |  13 --
>  fftools/resources/resman.c   | 231 -----------------------------------
>  fftools/resources/resman.h   |  50 --------
>  7 files changed, 3 insertions(+), 348 deletions(-)
>  delete mode 100644 fftools/resources/.gitignore
>  delete mode 100644 fftools/resources/Makefile
>  delete mode 100644 fftools/resources/resman.c
>  delete mode 100644 fftools/resources/resman.h
>
> diff --git a/configure b/configure
> index 0609dac4ab..2e69b3c56c 100755
> --- a/configure
> +++ b/configure
> @@ -523,7 +523,6 @@ Developer options (useful when working on FFmpeg itself):
>    --enable-macos-kperf     enable macOS kperf (private) API
>    --disable-large-tests    disable tests that use a large amount of memory
>    --disable-ptx-compression don't compress CUDA PTX code even when possible
> -  --disable-resource-compression don't compress resources even when possible
>    --disable-version-tracking don't include the git/release version in the build
>
>  NOTE: Object files are built at the place where configure is launched.
> @@ -2124,7 +2123,6 @@ CONFIG_LIST="
>      ossfuzz
>      pic
>      ptx_compression
> -    resource_compression
>      thumb
>      valgrind_backtrace
>      xmm_clobber_test
> @@ -4181,7 +4179,6 @@ enable iamf
>  enable large_tests
>  enable optimizations
>  enable ptx_compression
> -enable resource_compression
>  enable runtime_cpudetect
>  enable safe_bitstream_reader
>  enable static
> @@ -6907,8 +6904,6 @@ EOF
>
>  enabled zlib_gzip && enabled gzip || disable ptx_compression
>
> -enabled zlib_gzip && enabled gzip || disable resource_compression
> -
>  # On some systems dynamic loading requires no extra linker flags
>  check_lib libdl dlfcn.h "dlopen dlsym" || check_lib libdl dlfcn.h "dlopen dlsym" -ldl
>
> diff --git a/ffbuild/common.mak b/ffbuild/common.mak
> index 0e1eb1f62b..ca45a0f368 100644
> --- a/ffbuild/common.mak
> +++ b/ffbuild/common.mak
> @@ -139,44 +139,6 @@ else
>         $(BIN2C) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
>  endif
>
> -# 1) Preprocess CSS to a minified version
> -%.css.min: %.css
> -       # Must start with a tab in the real Makefile
> -       sed 's!/\\*.*\\*/!!g' $< \
> -       | tr '\n' ' ' \
> -       | tr -s ' ' \
> -       | sed 's/^ //; s/ $$//' \
> -       > $@
> -
> -ifdef CONFIG_RESOURCE_COMPRESSION
> -
> -# 2) Gzip the minified CSS
> -%.css.min.gz: %.css.min
> -       $(M)gzip -nc9 $< > $@
> -
> -# 3) Convert the gzipped CSS to a .c array
> -%.css.c: %.css.min.gz $(BIN2CEXE)
> -       $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
> -
> -# 4) Gzip the HTML file (no minification needed)
> -%.html.gz: %.html
> -       $(M)gzip -nc9 $< > $@
> -
> -# 5) Convert the gzipped HTML to a .c array
> -%.html.c: %.html.gz $(BIN2CEXE)
> -       $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
> -
> -else   # NO COMPRESSION
> -
> -# 2) Convert the minified CSS to a .c array
> -%.css.c: %.css.min $(BIN2CEXE)
> -       $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
> -
> -# 3) Convert the plain HTML to a .c array
> -%.html.c: %.html $(BIN2CEXE)
> -       $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
> -endif
> -
>  clean::
>         $(RM) $(BIN2CEXE) $(CLEANSUFFIXES:%=ffbuild/%)
>
> @@ -229,10 +191,9 @@ SKIPHEADERS += $(ARCH_HEADERS:%=$(ARCH)/%) $(SKIPHEADERS-)
>  SKIPHEADERS := $(SKIPHEADERS:%=$(SUBDIR)%)
>  HOBJS        = $(filter-out $(SKIPHEADERS:.h=.h.o),$(ALLHEADERS:.h=.h.o))
>  PTXOBJS      = $(filter %.ptx.o,$(OBJS))
> -RESOURCEOBJS = $(filter %.css.o %.html.o,$(OBJS))
>  $(HOBJS):     CCFLAGS += $(CFLAGS_HEADERS)
>  checkheaders: $(HOBJS)
> -.SECONDARY:   $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=.gz) $(PTXOBJS:.o=) $(RESOURCEOBJS:.o=.c) $(RESOURCEOBJS:%.css.o=%.css.min) $(RESOURCEOBJS:%.css.o=%.css.min.gz) $(RESOURCEOBJS:%.html.o=%.html.gz) $(RESOURCEOBJS:.o=)
> +.SECONDARY:   $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=.gz) $(PTXOBJS:.o=)
>
>  alltools: $(TOOLS)
>
> @@ -253,7 +214,7 @@ $(TOOLOBJS): | tools
>
>  OUTDIRS := $(OUTDIRS) $(dir $(OBJS) $(HOBJS) $(HOSTOBJS) $(SLIBOBJS) $(SHLIBOBJS) $(STLIBOBJS) $(TESTOBJS))
>
> -CLEANSUFFIXES     = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.gz *.ptx.c *.ver *.version *.html.gz *.html.c *.css.gz *.css.c  *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb
> +CLEANSUFFIXES     = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.gz *.ptx.c *.ver *.version *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb
>  LIBSUFFIXES       = *.a *.lib *.so *.so.* *.dylib *.dll *.def *.dll.a
>
>  define RULES
> diff --git a/fftools/Makefile b/fftools/Makefile
> index 361a4fd574..57e3c0e518 100644
> --- a/fftools/Makefile
> +++ b/fftools/Makefile
> @@ -9,8 +9,6 @@ AVBASENAMES  = ffmpeg ffplay ffprobe
>  ALLAVPROGS   = $(AVBASENAMES:%=%$(PROGSSUF)$(EXESUF))
>  ALLAVPROGS_G = $(AVBASENAMES:%=%$(PROGSSUF)_g$(EXESUF))
>
> -include $(SRC_PATH)/fftools/resources/Makefile
> -
>  OBJS-ffmpeg +=                  \
>      fftools/ffmpeg_dec.o        \
>      fftools/ffmpeg_demux.o      \
> @@ -59,7 +57,7 @@ ifdef HAVE_GNU_WINDRES
>  OBJS-$(1) += fftools/fftoolsres.o
>  endif
>  $(1)$(PROGSSUF)_g$(EXESUF): $$(OBJS-$(1))
> -$$(OBJS-$(1)): | fftools fftools/textformat fftools/resources fftools/graph
> +$$(OBJS-$(1)): | fftools fftools/textformat fftools/graph
>  $$(OBJS-$(1)): CFLAGS  += $(CFLAGS-$(1))
>  $(1)$(PROGSSUF)_g$(EXESUF): LDFLAGS += $(LDFLAGS-$(1))
>  $(1)$(PROGSSUF)_g$(EXESUF): FF_EXTRALIBS += $(EXTRALIBS-$(1))
> @@ -73,7 +71,6 @@ all: $(AVPROGS)
>  fftools/ffprobe.o fftools/cmdutils.o: libavutil/ffversion.h | fftools
>  OUTDIRS += fftools
>  OUTDIRS += fftools/textformat
> -OUTDIRS += fftools/resources
>  OUTDIRS += fftools/graph
>
>  ifdef AVPROGS
> diff --git a/fftools/resources/.gitignore b/fftools/resources/.gitignore
> deleted file mode 100644
> index 5f496535a6..0000000000
> --- a/fftools/resources/.gitignore
> +++ /dev/null
> @@ -1,4 +0,0 @@
> -*.html.c
> -*.css.c
> -*.html.gz
> -*.css.gz
> diff --git a/fftools/resources/Makefile b/fftools/resources/Makefile
> deleted file mode 100644
> index 8579a52678..0000000000
> --- a/fftools/resources/Makefile
> +++ /dev/null
> @@ -1,13 +0,0 @@
> -clean::
> -       $(RM) $(CLEANSUFFIXES:%=fftools/resources/%)
> -
> -vpath %.html $(SRC_PATH)
> -vpath %.css  $(SRC_PATH)
> -
> -# Uncomment to prevent deletion during build
> -#.PRECIOUS: %.css.c %.css.min %.css.gz %.css.min.gz %.html.gz %.html.c
> -
> -OBJS-resman +=                     \
> -    fftools/resources/resman.o     \
> -    fftools/resources/graph.html.o \
> -    fftools/resources/graph.css.o  \
> diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
> deleted file mode 100644
> index a9e21626fa..0000000000
> --- a/fftools/resources/resman.c
> +++ /dev/null
> @@ -1,231 +0,0 @@
> -/*
> - * Copyright (c) 2025 - softworkz
> - *
> - * This file is part of FFmpeg.
> - *
> - * FFmpeg is free software; you can redistribute it and/or
> - * modify it under the terms of the GNU Lesser General Public
> - * License as published by the Free Software Foundation; either
> - * version 2.1 of the License, or (at your option) any later version.
> - *
> - * FFmpeg is distributed in the hope that it will be useful,
> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> - * Lesser General Public License for more details.
> - *
> - * You should have received a copy of the GNU Lesser General Public
> - * License along with FFmpeg; if not, write to the Free Software
> - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> - */
> -
> -/**
> - * @file
> - * output writers for filtergraph details
> - */
> -
> -#include "config.h"
> -
> -#include <string.h>
> -
> -#if CONFIG_RESOURCE_COMPRESSION
> -#include <zlib.h>
> -#endif
> -
> -#include "resman.h"
> -#include "fftools/ffmpeg_filter.h"
> -#include "libavutil/avassert.h"
> -#include "libavutil/pixdesc.h"
> -#include "libavutil/dict.h"
> -#include "libavutil/common.h"
> -
> -extern const unsigned char ff_graph_html_data[];
> -extern const unsigned int ff_graph_html_len;
> -
> -extern const unsigned char ff_graph_css_data[];
> -extern const unsigned ff_graph_css_len;
> -
> -static const FFResourceDefinition resource_definitions[] = {
> -    [FF_RESOURCE_GRAPH_CSS]   = { FF_RESOURCE_GRAPH_CSS,   "graph.css",   &ff_graph_css_data[0],   &ff_graph_css_len   },
> -    [FF_RESOURCE_GRAPH_HTML]  = { FF_RESOURCE_GRAPH_HTML,  "graph.html",  &ff_graph_html_data[0],  &ff_graph_html_len  },
> -};
> -
> -
> -static const AVClass resman_class = {
> -    .class_name = "ResourceManager",
> -};
> -
> -typedef struct ResourceManagerContext {
> -    const AVClass *class;
> -    AVDictionary *resource_dic;
> -} ResourceManagerContext;
> -
> -static AVMutex mutex = AV_MUTEX_INITIALIZER;
> -
> -ResourceManagerContext *resman_ctx = NULL;
> -
> -
> -#if CONFIG_RESOURCE_COMPRESSION
> -
> -static int decompress_gzip(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
> -{
> -    z_stream strm;
> -    unsigned chunk = 65534;
> -    int ret;
> -    uint8_t *buf;
> -
> -    *out = NULL;
> -    memset(&strm, 0, sizeof(strm));
> -
> -    // Allocate output buffer with extra byte for null termination
> -    buf = (uint8_t *)av_mallocz(chunk + 1);
> -    if (!buf) {
> -        av_log(ctx, AV_LOG_ERROR, "Failed to allocate decompression buffer\n");
> -        return AVERROR(ENOMEM);
> -    }
> -
> -    // 15 + 16 tells zlib to detect GZIP or zlib automatically
> -    ret = inflateInit2(&strm, 15 + 16);
> -    if (ret != Z_OK) {
> -        av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg);
> -        av_free(buf);
> -        return AVERROR(ENOSYS);
> -    }
> -
> -    strm.avail_in  = in_len;
> -    strm.next_in   = in;
> -    strm.avail_out = chunk;
> -    strm.next_out  = buf;
> -
> -    ret = inflate(&strm, Z_FINISH);
> -    if (ret != Z_OK && ret != Z_STREAM_END) {
> -        av_log(ctx, AV_LOG_ERROR, "Inflate failed: %d, %s\n", ret, strm.msg);
> -        inflateEnd(&strm);
> -        av_free(buf);
> -        return (ret == Z_STREAM_END) ? Z_OK : ((ret == Z_OK) ? Z_BUF_ERROR : ret);
> -    }
> -
> -    if (strm.avail_out == 0) {
> -        // TODO: Error or loop decoding?
> -        av_log(ctx, AV_LOG_WARNING, "Decompression buffer may be too small\n");
> -    }
> -
> -    *out_len = chunk - strm.avail_out;
> -    buf[*out_len] = 0; // Ensure null termination
> -
> -    inflateEnd(&strm);
> -    *out = (char *)buf;
> -    return Z_OK;
> -}
> -#endif
> -
> -static ResourceManagerContext *get_resman_context(void)
> -{
> -    ResourceManagerContext *res = resman_ctx;
> -
> -    ff_mutex_lock(&mutex);
> -
> -    if (res)
> -        goto end;
> -
> -    res = av_mallocz(sizeof(ResourceManagerContext));
> -    if (!res) {
> -        av_log(NULL, AV_LOG_ERROR, "Failed to allocate resource manager context\n");
> -        goto end;
> -    }
> -
> -    res->class = &resman_class;
> -    resman_ctx = res;
> -
> -end:
> -    ff_mutex_unlock(&mutex);
> -    return res;
> -}
> -
> -
> -void ff_resman_uninit(void)
> -{
> -    ff_mutex_lock(&mutex);
> -
> -    if (resman_ctx) {
> -        if (resman_ctx->resource_dic)
> -            av_dict_free(&resman_ctx->resource_dic);
> -        av_freep(&resman_ctx);
> -    }
> -
> -    ff_mutex_unlock(&mutex);
> -}
> -
> -
> -char *ff_resman_get_string(FFResourceId resource_id)
> -{
> -    ResourceManagerContext *ctx               = get_resman_context();
> -    FFResourceDefinition resource_definition = { 0 };
> -    AVDictionaryEntry *dic_entry;
> -    char *res = NULL;
> -
> -    if (!ctx)
> -        return NULL;
> -
> -    for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
> -        FFResourceDefinition def = resource_definitions[i];
> -        if (def.resource_id == resource_id) {
> -            resource_definition = def;
> -            break;
> -        }
> -    }
> -
> -    if (!resource_definition.name) {
> -        av_log(ctx, AV_LOG_ERROR, "Unable to find resource with ID %d\n", resource_id);
> -        return NULL;
> -    }
> -
> -    ff_mutex_lock(&mutex);
> -
> -    dic_entry = av_dict_get(ctx->resource_dic, resource_definition.name, NULL, 0);
> -
> -    if (!dic_entry) {
> -        int dict_ret;
> -
> -#if CONFIG_RESOURCE_COMPRESSION
> -
> -        char *out = NULL;
> -        size_t out_len;
> -
> -        int ret = decompress_gzip(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
> -
> -        if (ret) {
> -            av_log(NULL, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
> -            goto end;
> -        }
> -
> -        dict_ret = av_dict_set(&ctx->resource_dic, resource_definition.name, out, 0);
> -        if (dict_ret < 0) {
> -            av_log(NULL, AV_LOG_ERROR, "Failed to store decompressed resource in dictionary: %d\n", dict_ret);
> -            av_freep(&out);
> -            goto end;
> -        }
> -
> -        av_freep(&out);
> -#else
> -
> -        dict_ret = av_dict_set(&ctx->resource_dic, resource_definition.name, (const char *)resource_definition.data, 0);
> -        if (dict_ret < 0) {
> -            av_log(NULL, AV_LOG_ERROR, "Failed to store resource in dictionary: %d\n", dict_ret);
> -            goto end;
> -        }
> -
> -#endif
> -        dic_entry = av_dict_get(ctx->resource_dic, resource_definition.name, NULL, 0);
> -
> -        if (!dic_entry) {
> -            av_log(NULL, AV_LOG_ERROR, "Failed to retrieve resource from dictionary after storing it\n");
> -            goto end;
> -        }
> -    }
> -
> -    res = dic_entry->value;
> -
> -end:
> -    ff_mutex_unlock(&mutex);
> -    return res;
> -}
> diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
> deleted file mode 100644
> index 6485db5091..0000000000
> --- a/fftools/resources/resman.h
> +++ /dev/null
> @@ -1,50 +0,0 @@
> -/*
> - * Copyright (c) 2025 - softworkz
> - *
> - * This file is part of FFmpeg.
> - *
> - * FFmpeg is free software; you can redistribute it and/or
> - * modify it under the terms of the GNU Lesser General Public
> - * License as published by the Free Software Foundation; either
> - * version 2.1 of the License, or (at your option) any later version.
> - *
> - * FFmpeg is distributed in the hope that it will be useful,
> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> - * Lesser General Public License for more details.
> - *
> - * You should have received a copy of the GNU Lesser General Public
> - * License along with FFmpeg; if not, write to the Free Software
> - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> - */
> -
> -#ifndef FFTOOLS_RESOURCES_RESMAN_H
> -#define FFTOOLS_RESOURCES_RESMAN_H
> -
> -#include <stdint.h>
> -
> -#include "config.h"
> -#include "fftools/ffmpeg.h"
> -#include "libavutil/avutil.h"
> -#include "libavutil/bprint.h"
> -#include "fftools/textformat/avtextformat.h"
> -
> -typedef enum {
> -    FF_RESOURCE_GRAPH_CSS,
> -    FF_RESOURCE_GRAPH_HTML,
> -} FFResourceId;
> -
> -typedef struct FFResourceDefinition {
> -    FFResourceId resource_id;
> -    const char *name;
> -
> -    const unsigned char *data;
> -    const unsigned *data_len;
> -
> -} FFResourceDefinition;
> -
> -void ff_resman_uninit(void);
> -
> -char *ff_resman_get_string(FFResourceId resource_id);
> -
> -#endif /* FFTOOLS_RESOURCES_RESMAN_H */
> --
> 2.43.0
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Hi Marton,

Is this still on track to be merged? Resource embedding is broken on
some Windows builds and not sure if I should care to debug/fix that or
this code will be nuked anyway?

Thanks,
Kacper
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".