[FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Gyan Doshi]

Avoids echo failing due to the same ARG_MAX limit that prompted
response files to be used with the linker.
---
 ffbuild/library.mak | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/ffbuild/library.mak b/ffbuild/library.mak
index 7e1871b74c..15302852ec 100644
--- a/ffbuild/library.mak
+++ b/ffbuild/library.mak
@@ -36,7 +36,8 @@ endif
 $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
 	$(RM) $@
 ifeq ($(AR_OBJS),true)
-	$(Q)echo $^ > $@.objs
+	-$(RM) $@.objs
+	$(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
 	$(AR) $(ARFLAGS) $(AR_O) @$@.objs
 else
 	$(AR) $(ARFLAGS) $(AR_O) $^
@@ -73,7 +74,9 @@ $(SUBDIR)$(SLIBNAME): $(SUBDIR)$(SLIBNAME_WITH_MAJOR)
 $(SUBDIR)$(SLIBNAME_WITH_MAJOR): $(OBJS) $(SHLIBOBJS) $(SLIBOBJS) $(SUBDIR)lib$(NAME).ver
 	$(SLIB_CREATE_DEF_CMD)
 ifeq ($(AR_OBJS),true)
-	$(Q)echo $$(filter %.o,$$^) > $$@.objs
+	-$(RM) $$@.objs
+	$(Q)$(eval LDARGS=$$(filter %.o,$$^))
+	$(Q)$(foreach ARG,$$(LDARGS),echo -n "$(ARG) " >> $$@.objs;)
 	$$(LD) $(SHFLAGS) $(LDFLAGS) $(LDSOFLAGS) $$(LD_O) @$$@.objs $(FFEXTRALIBS)
 else
 	$$(LD) $(SHFLAGS) $(LDFLAGS) $(LDSOFLAGS) $$(LD_O) $$(filter %.o,$$^) $(FFEXTRALIBS)
-- 
2.46.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 2/2] configure: add option to select use of response files

[Gyan Doshi]

---
 configure           | 27 ++++++++++++++++++++-------
 ffbuild/library.mak |  4 ++--
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/configure b/configure
index 14f7bcde0e..863b9adf22 100755
--- a/configure
+++ b/configure
@@ -427,6 +427,8 @@ Toolchain options:
   --enable-thumb           compile for Thumb instruction set
   --enable-lto[=arg]       use link-time optimization
   --env="ENV=override"     override the environment variables
+  --response-files=ARG     Pass list of objects to linker in a file to avoid command line length limits
+                           Valid values are: yes, no, auto(default)
 
 Advanced options (experts only):
   --malloc-prefix=PREFIX   prefix malloc and related names with PREFIX
@@ -4142,6 +4144,7 @@ objformat="elf32"
 x86asmexe_default="nasm"
 windres_default="windres"
 striptype="direct"
+ar_objs_default="auto"
 
 # OS
 target_os_default=$(tolower $(uname -s))
@@ -4438,6 +4441,10 @@ for opt do
         --enable-lto*)
             lto=-f${opt#--enable-}
         ;;
+        --response-files=*)
+            ar_objs=$optval
+            is_in $ar_objs "yes no auto" || die "Invalid value for response-files: $optval; Valid values are: yes, no, or auto"
+        ;;
         --enable-*=*|--disable-*=*)
             eval $(echo "${opt%%=*}" | sed 's/--/action=/;s/-/ thing=/')
             is_in "${thing}s" $COMPONENT_LIST || die_unknown "$opt"
@@ -5187,7 +5194,7 @@ test -n "$cc_type" && enable $cc_type ||
 : ${dep_cc_default:=$cc}
 : ${ld_default:=$cc}
 : ${host_ld_default:=$host_cc}
-set_default ar as objcc dep_cc ld ln_s host_ld windres
+set_default ar as objcc dep_cc ld ln_s host_ld windres ar_objs
 
 probe_cc as "$as"
 asflags_filter=$_flags_filter
@@ -7753,12 +7760,18 @@ case $ld_type in
     ;;
 esac
 
-{
-ar_out=${FFTMPDIR}/test$LIBSUF
-respfile="@/dev/null"
-out_arg="$(echo $ar_o | sed "s;\$@;$ar_out;g")"
-test_cmd $ar $arflags $out_arg $respfile && ar_objs="true" || ar_objs=""
-}
+if [ "$ar_objs" != "no" ]; then
+    ar_out=${FFTMPDIR}/test$LIBSUF
+    respfile="@/dev/null"
+    out_arg="$(echo $ar_o | sed "s;\$@;$ar_out;g")"
+    if test_cmd $ar $arflags $out_arg $respfile; then
+        ar_objs="yes"
+    elif [ "$ar_objs" = "auto" ]; then
+        ar_objs="no"
+    else
+        die "Response files are not available with this toolchain. Exiting"
+    fi
+fi
 
 enable frame_thread_encoder
 
diff --git a/ffbuild/library.mak b/ffbuild/library.mak
index 15302852ec..50a4cf74f2 100644
--- a/ffbuild/library.mak
+++ b/ffbuild/library.mak
@@ -35,7 +35,7 @@ OBJS += $(SHLIBOBJS)
 endif
 $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
 	$(RM) $@
-ifeq ($(AR_OBJS),true)
+ifeq ($(AR_OBJS),yes)
 	-$(RM) $@.objs
 	$(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
 	$(AR) $(ARFLAGS) $(AR_O) @$@.objs
@@ -73,7 +73,7 @@ $(SUBDIR)$(SLIBNAME): $(SUBDIR)$(SLIBNAME_WITH_MAJOR)
 
 $(SUBDIR)$(SLIBNAME_WITH_MAJOR): $(OBJS) $(SHLIBOBJS) $(SLIBOBJS) $(SUBDIR)lib$(NAME).ver
 	$(SLIB_CREATE_DEF_CMD)
-ifeq ($(AR_OBJS),true)
+ifeq ($(AR_OBJS),yes)
 	-$(RM) $$@.objs
 	$(Q)$(eval LDARGS=$$(filter %.o,$$^))
 	$(Q)$(foreach ARG,$$(LDARGS),echo -n "$(ARG) " >> $$@.objs;)
-- 
2.46.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Martin Storsjö]

On Fri, 21 Mar 2025, Gyan Doshi wrote:

> Avoids echo failing due to the same ARG_MAX limit that prompted
> response files to be used with the linker.

I presume this is only a fix for a hypothetical issue, _if_ echo would be 
a native windows executable and not the msys2/cygwin one which bypasses 
the limit?

> ---
> ffbuild/library.mak | 7 +++++--
> 1 file changed, 5 insertions(+), 2 deletions(-)
>
> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
> index 7e1871b74c..15302852ec 100644
> --- a/ffbuild/library.mak
> +++ b/ffbuild/library.mak
> @@ -36,7 +36,8 @@ endif
> $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
> 	$(RM) $@
> ifeq ($(AR_OBJS),true)
> -	$(Q)echo $^ > $@.objs
> +	-$(RM) $@.objs
> +	$(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)

Does this instance even work, it looks broken, like it is missing 
something?

> 	$(AR) $(ARFLAGS) $(AR_O) @$@.objs
> else
> 	$(AR) $(ARFLAGS) $(AR_O) $^
> @@ -73,7 +74,9 @@ $(SUBDIR)$(SLIBNAME): $(SUBDIR)$(SLIBNAME_WITH_MAJOR)
> $(SUBDIR)$(SLIBNAME_WITH_MAJOR): $(OBJS) $(SHLIBOBJS) $(SLIBOBJS) $(SUBDIR)lib$(NAME).ver
> 	$(SLIB_CREATE_DEF_CMD)
> ifeq ($(AR_OBJS),true)
> -	$(Q)echo $$(filter %.o,$$^) > $$@.objs
> +	-$(RM) $$@.objs
> +	$(Q)$(eval LDARGS=$$(filter %.o,$$^))
> +	$(Q)$(foreach ARG,$$(LDARGS),echo -n "$(ARG) " >> $$@.objs;)

Wouldn't this be quite significantly slow on msys2, where process creation 
is much slower than on unix? I think it's not worth to make things that 
much slower (which I only guess here) to fix a hypothetical issue.

// Martin

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Gyan Doshi]

On 2025-03-22 03:29 am, Martin Storsjö wrote:
> On Fri, 21 Mar 2025, Gyan Doshi wrote:
>
>> Avoids echo failing due to the same ARG_MAX limit that prompted
>> response files to be used with the linker.
>
> I presume this is only a fix for a hypothetical issue, _if_ echo would 
> be a native windows executable and not the msys2/cygwin one which 
> bypasses the limit?

Yes, trying to address edge cases.

>
>> ---
>> ffbuild/library.mak | 7 +++++--
>> 1 file changed, 5 insertions(+), 2 deletions(-)
>>
>> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
>> index 7e1871b74c..15302852ec 100644
>> --- a/ffbuild/library.mak
>> +++ b/ffbuild/library.mak
>> @@ -36,7 +36,8 @@ endif
>> $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
>>     $(RM) $@
>> ifeq ($(AR_OBJS),true)
>> -    $(Q)echo $^ > $@.objs
>> +    -$(RM) $@.objs
>> +    $(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
>
> Does this instance even work, it looks broken, like it is missing 
> something?

Tested both static and shared building. All working with no noticeable 
speed difference although I didn't formally bench.

Regards,
Gyan


>
>>     $(AR) $(ARFLAGS) $(AR_O) @$@.objs
>> else
>>     $(AR) $(ARFLAGS) $(AR_O) $^
>> @@ -73,7 +74,9 @@ $(SUBDIR)$(SLIBNAME): $(SUBDIR)$(SLIBNAME_WITH_MAJOR)
>> $(SUBDIR)$(SLIBNAME_WITH_MAJOR): $(OBJS) $(SHLIBOBJS) $(SLIBOBJS) 
>> $(SUBDIR)lib$(NAME).ver
>>     $(SLIB_CREATE_DEF_CMD)
>> ifeq ($(AR_OBJS),true)
>> -    $(Q)echo $$(filter %.o,$$^) > $$@.objs
>> +    -$(RM) $$@.objs
>> +    $(Q)$(eval LDARGS=$$(filter %.o,$$^))
>> +    $(Q)$(foreach ARG,$$(LDARGS),echo -n "$(ARG) " >> $$@.objs;)
>
> Wouldn't this be quite significantly slow on msys2, where process 
> creation is much slower than on unix? I think it's not worth to make 
> things that much slower (which I only guess here) to fix a 
> hypothetical issue.
>
> // Martin
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Ramiro Polla]

Hi,

On Sat, Mar 22, 2025 at 5:33 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
> On 2025-03-22 03:29 am, Martin Storsjö wrote:
> > On Fri, 21 Mar 2025, Gyan Doshi wrote:
> >> ffbuild/library.mak | 7 +++++--
> >> 1 file changed, 5 insertions(+), 2 deletions(-)
> >>
> >> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
> >> index 7e1871b74c..15302852ec 100644
> >> --- a/ffbuild/library.mak
> >> +++ b/ffbuild/library.mak
> >> @@ -36,7 +36,8 @@ endif
> >> $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
> >>     $(RM) $@
> >> ifeq ($(AR_OBJS),true)
> >> -    $(Q)echo $^ > $@.objs
> >> +    -$(RM) $@.objs
> >> +    $(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
> >
> > Does this instance even work, it looks broken, like it is missing
> > something?
>
> Tested both static and shared building. All working with no noticeable
> speed difference although I didn't formally bench.

It's generally a good idea to check for and report the speed
differences. Even if just for the sake of curiosity.

> >>     $(AR) $(ARFLAGS) $(AR_O) @$@.objs
> >> else
> >>     $(AR) $(ARFLAGS) $(AR_O) $^
> >> @@ -73,7 +74,9 @@ $(SUBDIR)$(SLIBNAME): $(SUBDIR)$(SLIBNAME_WITH_MAJOR)
> >> $(SUBDIR)$(SLIBNAME_WITH_MAJOR): $(OBJS) $(SHLIBOBJS) $(SLIBOBJS)
> >> $(SUBDIR)lib$(NAME).ver
> >>     $(SLIB_CREATE_DEF_CMD)
> >> ifeq ($(AR_OBJS),true)
> >> -    $(Q)echo $$(filter %.o,$$^) > $$@.objs
> >> +    -$(RM) $$@.objs
> >> +    $(Q)$(eval LDARGS=$$(filter %.o,$$^))
> >> +    $(Q)$(foreach ARG,$$(LDARGS),echo -n "$(ARG) " >> $$@.objs;)
> >
> > Wouldn't this be quite significantly slow on msys2, where process
> > creation is much slower than on unix? I think it's not worth to make
> > things that much slower (which I only guess here) to fix a
> > hypothetical issue.

GNU make suggests using its file function for that:
https://www.gnu.org/software/make/manual/html_node/File-Function.html

Ramiro
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 2/2] configure: add option to select use of response files

[Ramiro Polla]

Hi,

On Fri, Mar 21, 2025 at 11:42 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
>
> ---
>  configure           | 27 ++++++++++++++++++++-------
>  ffbuild/library.mak |  4 ++--
>  2 files changed, 22 insertions(+), 9 deletions(-)
>
> diff --git a/configure b/configure
> index 14f7bcde0e..863b9adf22 100755
> --- a/configure
> +++ b/configure
> @@ -427,6 +427,8 @@ Toolchain options:
>    --enable-thumb           compile for Thumb instruction set
>    --enable-lto[=arg]       use link-time optimization
>    --env="ENV=override"     override the environment variables
> +  --response-files=ARG     Pass list of objects to linker in a file to avoid command line length limits
> +                           Valid values are: yes, no, auto(default)

I think --disable-response-files with default [autodetect] would be
more in line with the other options in configure.

>  Advanced options (experts only):
>    --malloc-prefix=PREFIX   prefix malloc and related names with PREFIX
> @@ -4142,6 +4144,7 @@ objformat="elf32"
>  x86asmexe_default="nasm"
>  windres_default="windres"
>  striptype="direct"
> +ar_objs_default="auto"
>
>  # OS
>  target_os_default=$(tolower $(uname -s))
> @@ -4438,6 +4441,10 @@ for opt do
>          --enable-lto*)
>              lto=-f${opt#--enable-}
>          ;;
> +        --response-files=*)
> +            ar_objs=$optval
> +            is_in $ar_objs "yes no auto" || die "Invalid value for response-files: $optval; Valid values are: yes, no, or auto"
> +        ;;
>          --enable-*=*|--disable-*=*)
>              eval $(echo "${opt%%=*}" | sed 's/--/action=/;s/-/ thing=/')
>              is_in "${thing}s" $COMPONENT_LIST || die_unknown "$opt"
> @@ -5187,7 +5194,7 @@ test -n "$cc_type" && enable $cc_type ||
>  : ${dep_cc_default:=$cc}
>  : ${ld_default:=$cc}
>  : ${host_ld_default:=$host_cc}
> -set_default ar as objcc dep_cc ld ln_s host_ld windres
> +set_default ar as objcc dep_cc ld ln_s host_ld windres ar_objs
>
>  probe_cc as "$as"
>  asflags_filter=$_flags_filter
> @@ -7753,12 +7760,18 @@ case $ld_type in
>      ;;
>  esac
>
> -{
> -ar_out=${FFTMPDIR}/test$LIBSUF
> -respfile="@/dev/null"
> -out_arg="$(echo $ar_o | sed "s;\$@;$ar_out;g")"
> -test_cmd $ar $arflags $out_arg $respfile && ar_objs="true" || ar_objs=""
> -}
> +if [ "$ar_objs" != "no" ]; then
> +    ar_out=${FFTMPDIR}/test$LIBSUF
> +    respfile="@/dev/null"
> +    out_arg="$(echo $ar_o | sed "s;\$@;$ar_out;g")"
> +    if test_cmd $ar $arflags $out_arg $respfile; then
> +        ar_objs="yes"
> +    elif [ "$ar_objs" = "auto" ]; then
> +        ar_objs="no"
> +    else
> +        die "Response files are not available with this toolchain. Exiting"
> +    fi
> +fi
>
>  enable frame_thread_encoder
>
> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
> index 15302852ec..50a4cf74f2 100644
> --- a/ffbuild/library.mak
> +++ b/ffbuild/library.mak
> @@ -35,7 +35,7 @@ OBJS += $(SHLIBOBJS)
>  endif
>  $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
>         $(RM) $@
> -ifeq ($(AR_OBJS),true)
> +ifeq ($(AR_OBJS),yes)
>         -$(RM) $@.objs
>         $(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
>         $(AR) $(ARFLAGS) $(AR_O) @$@.objs
> @@ -73,7 +73,7 @@ $(SUBDIR)$(SLIBNAME): $(SUBDIR)$(SLIBNAME_WITH_MAJOR)
>
>  $(SUBDIR)$(SLIBNAME_WITH_MAJOR): $(OBJS) $(SHLIBOBJS) $(SLIBOBJS) $(SUBDIR)lib$(NAME).ver
>         $(SLIB_CREATE_DEF_CMD)
> -ifeq ($(AR_OBJS),true)
> +ifeq ($(AR_OBJS),yes)
>         -$(RM) $$@.objs
>         $(Q)$(eval LDARGS=$$(filter %.o,$$^))
>         $(Q)$(foreach ARG,$$(LDARGS),echo -n "$(ARG) " >> $$@.objs;)

Could you rename AR_OBJS to something like RESPONSE_FILES?

Ramiro
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH v2] configure: add option to select use of response files

[Gyan Doshi]

---
v2:
   *switched state and make var name to response_files
   *changed option to standard action-optname format

 configure           | 25 +++++++++++++++++--------
 ffbuild/library.mak |  4 ++--
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/configure b/configure
index 2fdbe8cbbe..eef7826788 100755
--- a/configure
+++ b/configure
@@ -427,6 +427,7 @@ Toolchain options:
   --enable-thumb           compile for Thumb instruction set
   --enable-lto[=arg]       use link-time optimization
   --env="ENV=override"     override the environment variables
+  --disable-response-files Don't pass the list of objects to linker in a file [autodetect]
 
 Advanced options (experts only):
   --malloc-prefix=PREFIX   prefix malloc and related names with PREFIX
@@ -2669,6 +2670,7 @@ CMDLINE_SELECT="
     extra_warnings
     logging
     optimizations
+    response_files
     rpath
     stripping
     version_tracking
@@ -4143,6 +4145,7 @@ objformat="elf32"
 x86asmexe_default="nasm"
 windres_default="windres"
 striptype="direct"
+response_files_default="auto"
 
 # OS
 target_os_default=$(tolower $(uname -s))
@@ -5188,7 +5191,7 @@ test -n "$cc_type" && enable $cc_type ||
 : ${dep_cc_default:=$cc}
 : ${ld_default:=$cc}
 : ${host_ld_default:=$host_cc}
-set_default ar as objcc dep_cc ld ln_s host_ld windres
+set_default ar as objcc dep_cc ld ln_s host_ld windres response_files
 
 probe_cc as "$as"
 asflags_filter=$_flags_filter
@@ -7754,12 +7757,18 @@ case $ld_type in
     ;;
 esac
 
-{
-ar_out=${FFTMPDIR}/test$LIBSUF
-respfile="@/dev/null"
-out_arg="$(echo $ar_o | sed "s;\$@;$ar_out;g")"
-test_cmd $ar $arflags $out_arg $respfile && ar_objs="true" || ar_objs=""
-}
+if [ "$response_files" != "no" ]; then
+    ar_out=${FFTMPDIR}/test$LIBSUF
+    respfile="@/dev/null"
+    out_arg="$(echo $ar_o | sed "s;\$@;$ar_out;g")"
+    if test_cmd $ar $arflags $out_arg $respfile; then
+        response_files="yes"
+    elif [ "$response_files" = "auto" ]; then
+        response_files="no"
+    else
+        die "Response files are not available with this toolchain. Exiting"
+    fi
+fi
 
 enable frame_thread_encoder
 
@@ -8141,7 +8150,7 @@ DEPX86ASM=$x86asmexe
 DEPX86ASMFLAGS=\$(X86ASMFLAGS)
 AR=$ar
 ARFLAGS=$arflags
-AR_OBJS=$ar_objs
+RESPONSE_FILES=$response_files
 AR_O=$ar_o
 AR_CMD=$ar
 NM_CMD=$nm
diff --git a/ffbuild/library.mak b/ffbuild/library.mak
index 15302852ec..3f7d604086 100644
--- a/ffbuild/library.mak
+++ b/ffbuild/library.mak
@@ -35,7 +35,7 @@ OBJS += $(SHLIBOBJS)
 endif
 $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
 	$(RM) $@
-ifeq ($(AR_OBJS),true)
+ifeq ($(RESPONSE_FILES),yes)
 	-$(RM) $@.objs
 	$(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
 	$(AR) $(ARFLAGS) $(AR_O) @$@.objs
@@ -73,7 +73,7 @@ $(SUBDIR)$(SLIBNAME): $(SUBDIR)$(SLIBNAME_WITH_MAJOR)
 
 $(SUBDIR)$(SLIBNAME_WITH_MAJOR): $(OBJS) $(SHLIBOBJS) $(SLIBOBJS) $(SUBDIR)lib$(NAME).ver
 	$(SLIB_CREATE_DEF_CMD)
-ifeq ($(AR_OBJS),true)
+ifeq ($(RESPONSE_FILES),yes)
 	-$(RM) $$@.objs
 	$(Q)$(eval LDARGS=$$(filter %.o,$$^))
 	$(Q)$(foreach ARG,$$(LDARGS),echo -n "$(ARG) " >> $$@.objs;)
-- 
2.46.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Gyan Doshi]

On 2025-03-25 05:11 pm, Ramiro Polla wrote:
> Hi,
>
> On Sat, Mar 22, 2025 at 5:33 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
>> On 2025-03-22 03:29 am, Martin Storsjö wrote:
>>> On Fri, 21 Mar 2025, Gyan Doshi wrote:
>>>> ffbuild/library.mak | 7 +++++--
>>>> 1 file changed, 5 insertions(+), 2 deletions(-)
>>>>
>>>> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
>>>> index 7e1871b74c..15302852ec 100644
>>>> --- a/ffbuild/library.mak
>>>> +++ b/ffbuild/library.mak
>>>> @@ -36,7 +36,8 @@ endif
>>>> $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
>>>>      $(RM) $@
>>>> ifeq ($(AR_OBJS),true)
>>>> -    $(Q)echo $^ > $@.objs
>>>> +    -$(RM) $@.objs
>>>> +    $(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
>>> Does this instance even work, it looks broken, like it is missing
>>> something?
>> Tested both static and shared building. All working with no noticeable
>> speed difference although I didn't formally bench.
> It's generally a good idea to check for and report the speed
> differences. Even if just for the sake of curiosity.

So, with this configure:

   --disable-doc --disable-w32threads --enable-pthreads 
--disable-autodetect --cc="ccache gcc" --enable-static --enable-shared 
--extra-ldflags="-static" --enable-gpl --enable-version3

I ran one priming run + 5 repeat runs of    `time make -j4`.

Before, real ranged from 2m10.188s to 2m12.154s
After, real ranged from 2m12.486s to 2m13.405s

I also ran one priming run + 5 repeat runs of    `time make -j4 
libavcodec/libavcodec.a libavcodec/avcodec.dll`.

Before, real ranged from 1m34.232s to 1m42.721s
After, real ranged from 1m32.360s to 1m39.484s

Regards,
Gyan

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Andreas Rheinhardt]

Gyan Doshi:
> 
> 
> On 2025-03-25 05:11 pm, Ramiro Polla wrote:
>> Hi,
>>
>> On Sat, Mar 22, 2025 at 5:33 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
>>> On 2025-03-22 03:29 am, Martin Storsjö wrote:
>>>> On Fri, 21 Mar 2025, Gyan Doshi wrote:
>>>>> ffbuild/library.mak | 7 +++++--
>>>>> 1 file changed, 5 insertions(+), 2 deletions(-)
>>>>>
>>>>> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
>>>>> index 7e1871b74c..15302852ec 100644
>>>>> --- a/ffbuild/library.mak
>>>>> +++ b/ffbuild/library.mak
>>>>> @@ -36,7 +36,8 @@ endif
>>>>> $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
>>>>>      $(RM) $@
>>>>> ifeq ($(AR_OBJS),true)
>>>>> -    $(Q)echo $^ > $@.objs
>>>>> +    -$(RM) $@.objs
>>>>> +    $(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
>>>> Does this instance even work, it looks broken, like it is missing
>>>> something?
>>> Tested both static and shared building. All working with no noticeable
>>> speed difference although I didn't formally bench.
>> It's generally a good idea to check for and report the speed
>> differences. Even if just for the sake of curiosity.
> 
> So, with this configure:
> 
>   --disable-doc --disable-w32threads --enable-pthreads --disable-
> autodetect --cc="ccache gcc" --enable-static --enable-shared --extra-
> ldflags="-static" --enable-gpl --enable-version3
> 
> I ran one priming run + 5 repeat runs of    `time make -j4`.
> 
> Before, real ranged from 2m10.188s to 2m12.154s
> After, real ranged from 2m12.486s to 2m13.405s
> 
> I also ran one priming run + 5 repeat runs of    `time make -j4
> libavcodec/libavcodec.a libavcodec/avcodec.dll`.
> 
> Before, real ranged from 1m34.232s to 1m42.721s
> After, real ranged from 1m32.360s to 1m39.484s
> 

I presume this command compiled the whole of FFmpeg/libavcodec, i.e. it
did not just link, didn't it?

- Andreas

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Gyan Doshi]

On 2025-03-26 11:37 pm, Andreas Rheinhardt wrote:
> Gyan Doshi:
>>
>> On 2025-03-25 05:11 pm, Ramiro Polla wrote:
>>> Hi,
>>>
>>> On Sat, Mar 22, 2025 at 5:33 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
>>>> On 2025-03-22 03:29 am, Martin Storsjö wrote:
>>>>> On Fri, 21 Mar 2025, Gyan Doshi wrote:
>>>>>> ffbuild/library.mak | 7 +++++--
>>>>>> 1 file changed, 5 insertions(+), 2 deletions(-)
>>>>>>
>>>>>> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
>>>>>> index 7e1871b74c..15302852ec 100644
>>>>>> --- a/ffbuild/library.mak
>>>>>> +++ b/ffbuild/library.mak
>>>>>> @@ -36,7 +36,8 @@ endif
>>>>>> $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
>>>>>>       $(RM) $@
>>>>>> ifeq ($(AR_OBJS),true)
>>>>>> -    $(Q)echo $^ > $@.objs
>>>>>> +    -$(RM) $@.objs
>>>>>> +    $(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
>>>>> Does this instance even work, it looks broken, like it is missing
>>>>> something?
>>>> Tested both static and shared building. All working with no noticeable
>>>> speed difference although I didn't formally bench.
>>> It's generally a good idea to check for and report the speed
>>> differences. Even if just for the sake of curiosity.
>> So, with this configure:
>>
>>    --disable-doc --disable-w32threads --enable-pthreads --disable-
>> autodetect --cc="ccache gcc" --enable-static --enable-shared --extra-
>> ldflags="-static" --enable-gpl --enable-version3
>>
>> I ran one priming run + 5 repeat runs of    `time make -j4`.
>>
>> Before, real ranged from 2m10.188s to 2m12.154s
>> After, real ranged from 2m12.486s to 2m13.405s
>>
>> I also ran one priming run + 5 repeat runs of    `time make -j4
>> libavcodec/libavcodec.a libavcodec/avcodec.dll`.
>>
>> Before, real ranged from 1m34.232s to 1m42.721s
>> After, real ranged from 1m32.360s to 1m39.484s
>>
> I presume this command compiled the whole of FFmpeg/libavcodec, i.e. it
> did not just link, didn't it?

Its deps, are *.o in lavu, lswr, lavc, but after the priming run, all 
the cc steps would be cache hits,
so the bulk of the fresh executions are the linking steps.

Regards,
Gyan

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Gyan Doshi]

On 2025-03-27 10:03 am, Gyan Doshi wrote:
>
>
> On 2025-03-26 11:37 pm, Andreas Rheinhardt wrote:
>> Gyan Doshi:
>>>
>>> On 2025-03-25 05:11 pm, Ramiro Polla wrote:
>>>> Hi,
>>>>
>>>> On Sat, Mar 22, 2025 at 5:33 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
>>>>> On 2025-03-22 03:29 am, Martin Storsjö wrote:
>>>>>> On Fri, 21 Mar 2025, Gyan Doshi wrote:
>>>>>>> ffbuild/library.mak | 7 +++++--
>>>>>>> 1 file changed, 5 insertions(+), 2 deletions(-)
>>>>>>>
>>>>>>> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
>>>>>>> index 7e1871b74c..15302852ec 100644
>>>>>>> --- a/ffbuild/library.mak
>>>>>>> +++ b/ffbuild/library.mak
>>>>>>> @@ -36,7 +36,8 @@ endif
>>>>>>> $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
>>>>>>>       $(RM) $@
>>>>>>> ifeq ($(AR_OBJS),true)
>>>>>>> -    $(Q)echo $^ > $@.objs
>>>>>>> +    -$(RM) $@.objs
>>>>>>> +    $(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
>>>>>> Does this instance even work, it looks broken, like it is missing
>>>>>> something?
>>>>> Tested both static and shared building. All working with no 
>>>>> noticeable
>>>>> speed difference although I didn't formally bench.
>>>> It's generally a good idea to check for and report the speed
>>>> differences. Even if just for the sake of curiosity.
>>> So, with this configure:
>>>
>>>    --disable-doc --disable-w32threads --enable-pthreads --disable-
>>> autodetect --cc="ccache gcc" --enable-static --enable-shared --extra-
>>> ldflags="-static" --enable-gpl --enable-version3
>>>
>>> I ran one priming run + 5 repeat runs of    `time make -j4`.
>>>
>>> Before, real ranged from 2m10.188s to 2m12.154s
>>> After, real ranged from 2m12.486s to 2m13.405s
>>>
>>> I also ran one priming run + 5 repeat runs of    `time make -j4
>>> libavcodec/libavcodec.a libavcodec/avcodec.dll`.
>>>
>>> Before, real ranged from 1m34.232s to 1m42.721s
>>> After, real ranged from 1m32.360s to 1m39.484s
>>>
>> I presume this command compiled the whole of FFmpeg/libavcodec, i.e. it
>> did not just link, didn't it?
>
> Its deps, are *.o in lavu, lswr, lavc, but after the priming run, all 
> the cc steps would be cache hits,
> so the bulk of the fresh executions are the linking steps.

Plan to push tomorrow.

Regards,
Gyan

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v2] configure: add option to select use of response files

[Gyan Doshi]

On 2025-03-26 07:38 pm, Gyan Doshi wrote:
> ---
> v2:
>     *switched state and make var name to response_files
>     *changed option to standard action-optname format

Plan to push tomorrow.

Regards,
Gyan


>
>   configure           | 25 +++++++++++++++++--------
>   ffbuild/library.mak |  4 ++--
>   2 files changed, 19 insertions(+), 10 deletions(-)
>
> diff --git a/configure b/configure
> index 2fdbe8cbbe..eef7826788 100755
> --- a/configure
> +++ b/configure
> @@ -427,6 +427,7 @@ Toolchain options:
>     --enable-thumb           compile for Thumb instruction set
>     --enable-lto[=arg]       use link-time optimization
>     --env="ENV=override"     override the environment variables
> +  --disable-response-files Don't pass the list of objects to linker in a file [autodetect]
>   
>   Advanced options (experts only):
>     --malloc-prefix=PREFIX   prefix malloc and related names with PREFIX
> @@ -2669,6 +2670,7 @@ CMDLINE_SELECT="
>       extra_warnings
>       logging
>       optimizations
> +    response_files
>       rpath
>       stripping
>       version_tracking
> @@ -4143,6 +4145,7 @@ objformat="elf32"
>   x86asmexe_default="nasm"
>   windres_default="windres"
>   striptype="direct"
> +response_files_default="auto"
>   
>   # OS
>   target_os_default=$(tolower $(uname -s))
> @@ -5188,7 +5191,7 @@ test -n "$cc_type" && enable $cc_type ||
>   : ${dep_cc_default:=$cc}
>   : ${ld_default:=$cc}
>   : ${host_ld_default:=$host_cc}
> -set_default ar as objcc dep_cc ld ln_s host_ld windres
> +set_default ar as objcc dep_cc ld ln_s host_ld windres response_files
>   
>   probe_cc as "$as"
>   asflags_filter=$_flags_filter
> @@ -7754,12 +7757,18 @@ case $ld_type in
>       ;;
>   esac
>   
> -{
> -ar_out=${FFTMPDIR}/test$LIBSUF
> -respfile="@/dev/null"
> -out_arg="$(echo $ar_o | sed "s;\$@;$ar_out;g")"
> -test_cmd $ar $arflags $out_arg $respfile && ar_objs="true" || ar_objs=""
> -}
> +if [ "$response_files" != "no" ]; then
> +    ar_out=${FFTMPDIR}/test$LIBSUF
> +    respfile="@/dev/null"
> +    out_arg="$(echo $ar_o | sed "s;\$@;$ar_out;g")"
> +    if test_cmd $ar $arflags $out_arg $respfile; then
> +        response_files="yes"
> +    elif [ "$response_files" = "auto" ]; then
> +        response_files="no"
> +    else
> +        die "Response files are not available with this toolchain. Exiting"
> +    fi
> +fi
>   
>   enable frame_thread_encoder
>   
> @@ -8141,7 +8150,7 @@ DEPX86ASM=$x86asmexe
>   DEPX86ASMFLAGS=\$(X86ASMFLAGS)
>   AR=$ar
>   ARFLAGS=$arflags
> -AR_OBJS=$ar_objs
> +RESPONSE_FILES=$response_files
>   AR_O=$ar_o
>   AR_CMD=$ar
>   NM_CMD=$nm
> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
> index 15302852ec..3f7d604086 100644
> --- a/ffbuild/library.mak
> +++ b/ffbuild/library.mak
> @@ -35,7 +35,7 @@ OBJS += $(SHLIBOBJS)
>   endif
>   $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
>   	$(RM) $@
> -ifeq ($(AR_OBJS),true)
> +ifeq ($(RESPONSE_FILES),yes)
>   	-$(RM) $@.objs
>   	$(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
>   	$(AR) $(ARFLAGS) $(AR_O) @$@.objs
> @@ -73,7 +73,7 @@ $(SUBDIR)$(SLIBNAME): $(SUBDIR)$(SLIBNAME_WITH_MAJOR)
>   
>   $(SUBDIR)$(SLIBNAME_WITH_MAJOR): $(OBJS) $(SHLIBOBJS) $(SLIBOBJS) $(SUBDIR)lib$(NAME).ver
>   	$(SLIB_CREATE_DEF_CMD)
> -ifeq ($(AR_OBJS),true)
> +ifeq ($(RESPONSE_FILES),yes)
>   	-$(RM) $$@.objs
>   	$(Q)$(eval LDARGS=$$(filter %.o,$$^))
>   	$(Q)$(foreach ARG,$$(LDARGS),echo -n "$(ARG) " >> $$@.objs;)

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Ramiro Polla]

Hi Gyan,

On Sat, Mar 29, 2025 at 6:11 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
> On 2025-03-27 10:03 am, Gyan Doshi wrote:
> > On 2025-03-26 11:37 pm, Andreas Rheinhardt wrote:
> >> Gyan Doshi:
> >>> On 2025-03-25 05:11 pm, Ramiro Polla wrote:
> >>>> On Sat, Mar 22, 2025 at 5:33 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
> >>>>> On 2025-03-22 03:29 am, Martin Storsjö wrote:
> >>>>>> On Fri, 21 Mar 2025, Gyan Doshi wrote:
> >>>>>>> ffbuild/library.mak | 7 +++++--
> >>>>>>> 1 file changed, 5 insertions(+), 2 deletions(-)
> >>>>>>>
> >>>>>>> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
> >>>>>>> index 7e1871b74c..15302852ec 100644
> >>>>>>> --- a/ffbuild/library.mak
> >>>>>>> +++ b/ffbuild/library.mak
> >>>>>>> @@ -36,7 +36,8 @@ endif
> >>>>>>> $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
> >>>>>>>       $(RM) $@
> >>>>>>> ifeq ($(AR_OBJS),true)
> >>>>>>> -    $(Q)echo $^ > $@.objs
> >>>>>>> +    -$(RM) $@.objs
> >>>>>>> +    $(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
> >>>>>> Does this instance even work, it looks broken, like it is missing
> >>>>>> something?
> >>>>> Tested both static and shared building. All working with no
> >>>>> noticeable
> >>>>> speed difference although I didn't formally bench.
> >>>> It's generally a good idea to check for and report the speed
> >>>> differences. Even if just for the sake of curiosity.
> >>> So, with this configure:
> >>>
> >>>    --disable-doc --disable-w32threads --enable-pthreads --disable-
> >>> autodetect --cc="ccache gcc" --enable-static --enable-shared --extra-
> >>> ldflags="-static" --enable-gpl --enable-version3
> >>>
> >>> I ran one priming run + 5 repeat runs of    `time make -j4`.
> >>>
> >>> Before, real ranged from 2m10.188s to 2m12.154s
> >>> After, real ranged from 2m12.486s to 2m13.405s
> >>>
> >>> I also ran one priming run + 5 repeat runs of    `time make -j4
> >>> libavcodec/libavcodec.a libavcodec/avcodec.dll`.
> >>>
> >>> Before, real ranged from 1m34.232s to 1m42.721s
> >>> After, real ranged from 1m32.360s to 1m39.484s
> >>>
> >> I presume this command compiled the whole of FFmpeg/libavcodec, i.e. it
> >> did not just link, didn't it?
> >
> > Its deps, are *.o in lavu, lswr, lavc, but after the priming run, all
> > the cc steps would be cache hits,
> > so the bulk of the fresh executions are the linking steps.
>
> Plan to push tomorrow.

Did you not try to use GNU make's flie function?

https://www.gnu.org/software/make/manual/html_node/File-Function.html

Ramiro
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Gyan Doshi]

On 2025-03-29 05:51 pm, Ramiro Polla wrote:
> Hi Gyan,
>
> On Sat, Mar 29, 2025 at 6:11 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
>> On 2025-03-27 10:03 am, Gyan Doshi wrote:
>>> On 2025-03-26 11:37 pm, Andreas Rheinhardt wrote:
>>>> Gyan Doshi:
>>>>> On 2025-03-25 05:11 pm, Ramiro Polla wrote:
>>>>>> On Sat, Mar 22, 2025 at 5:33 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
>>>>>>> On 2025-03-22 03:29 am, Martin Storsjö wrote:
>>>>>>>> On Fri, 21 Mar 2025, Gyan Doshi wrote:
>>>>>>>>> ffbuild/library.mak | 7 +++++--
>>>>>>>>> 1 file changed, 5 insertions(+), 2 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/ffbuild/library.mak b/ffbuild/library.mak
>>>>>>>>> index 7e1871b74c..15302852ec 100644
>>>>>>>>> --- a/ffbuild/library.mak
>>>>>>>>> +++ b/ffbuild/library.mak
>>>>>>>>> @@ -36,7 +36,8 @@ endif
>>>>>>>>> $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS)
>>>>>>>>>        $(RM) $@
>>>>>>>>> ifeq ($(AR_OBJS),true)
>>>>>>>>> -    $(Q)echo $^ > $@.objs
>>>>>>>>> +    -$(RM) $@.objs
>>>>>>>>> +    $(Q)$(foreach ARG,$^,echo -n "$(ARG) " >> $@.objs;)
>>>>>>>> Does this instance even work, it looks broken, like it is missing
>>>>>>>> something?
>>>>>>> Tested both static and shared building. All working with no
>>>>>>> noticeable
>>>>>>> speed difference although I didn't formally bench.
>>>>>> It's generally a good idea to check for and report the speed
>>>>>> differences. Even if just for the sake of curiosity.
>>>>> So, with this configure:
>>>>>
>>>>>     --disable-doc --disable-w32threads --enable-pthreads --disable-
>>>>> autodetect --cc="ccache gcc" --enable-static --enable-shared --extra-
>>>>> ldflags="-static" --enable-gpl --enable-version3
>>>>>
>>>>> I ran one priming run + 5 repeat runs of    `time make -j4`.
>>>>>
>>>>> Before, real ranged from 2m10.188s to 2m12.154s
>>>>> After, real ranged from 2m12.486s to 2m13.405s
>>>>>
>>>>> I also ran one priming run + 5 repeat runs of    `time make -j4
>>>>> libavcodec/libavcodec.a libavcodec/avcodec.dll`.
>>>>>
>>>>> Before, real ranged from 1m34.232s to 1m42.721s
>>>>> After, real ranged from 1m32.360s to 1m39.484s
>>>>>
>>>> I presume this command compiled the whole of FFmpeg/libavcodec, i.e. it
>>>> did not just link, didn't it?
>>> Its deps, are *.o in lavu, lswr, lavc, but after the priming run, all
>>> the cc steps would be cache hits,
>>> so the bulk of the fresh executions are the linking steps.
>> Plan to push tomorrow.
> Did you not try to use GNU make's flie function?

I just benched this and it ranges from 1m28.093s to 1m29.971s (5% 
faster) for the lavc targets.
However, this was added in make 4.0. Are we supporting older make?

Regards,
Gyan

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Martin Storsjö]

On Sat, 29 Mar 2025, Gyan Doshi wrote:

>> Did you not try to use GNU make's flie function?
>
> I just benched this and it ranges from 1m28.093s to 1m29.971s (5% 
> faster) for the lavc targets.
> However, this was added in make 4.0. Are we supporting older make?

Yes, we generally do support older GNU make; macOS (even the latest 
versions) only ships with GNU make 3.81.

Regarding measuring the runtime cost of this change; measuring the whole 
build time is quite uninteresting, the interesting bit is measuring the 
time to build e.g. an .a library on its own. So after a full build, I do 
"rm libavcodec/libavcodec.a; time make libavcodec/libavcodec.a". This 
change raises that time from ~3.5 seconds to ~3.8 seconds. However do note 
that this is on a quite slow system in itself; without the "rm", it takes 
make 2.3 seconds just to figure out that nothing needs to be done.

So on that level, the change indeed is mostly tolerable.

However - this is very quick as long as "echo" is a shell builtin. If 
"echo" turns out to be an external executable instead of the shell 
builtin (which we can simulate by calling "/usr/bin/echo" instead of 
"echo"), then this suddenly takes >16 seconds rather than the earlier <4 
seconds. And that's quite a steep price to pay.

As noted before, this is only a fix for a potential, hypothetical problem. 
The fix is inexpensive in the case of a builtin echo, where we don't need 
the fix anyway. For the case of an external echo, where we potentially 
could need the fix, the fix is quite expensive though.

But even with the external /usr/bin/echo (on msys2), I still can produce a 
very long (>32k) .objs file with only one single invocation of 
/usr/bin/echo. So we don't actually have this problem even in that case.

So given that there are multiple concerns about the performance about 
this, and the problem that it tries to fix is entirely hypothetical at the 
moment, I would suggest that we skip this fix for now.

If someone actually manages to hit the problem in some setup and can tell 
us about it, we could reconsider of course.

// Martin

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Gyan Doshi]

On 2025-03-29 11:52 pm, Martin Storsjö wrote:
> On Sat, 29 Mar 2025, Gyan Doshi wrote:
>
>>> Did you not try to use GNU make's flie function?
>>
>> I just benched this and it ranges from 1m28.093s to 1m29.971s (5% 
>> faster) for the lavc targets.
>> However, this was added in make 4.0. Are we supporting older make?
>
> Yes, we generally do support older GNU make; macOS (even the latest 
> versions) only ships with GNU make 3.81.
>
> Regarding measuring the runtime cost of this change; measuring the 
> whole build time is quite uninteresting, the interesting bit is 
> measuring the time to build e.g. an .a library on its own. So after a 
> full build, I do "rm libavcodec/libavcodec.a; time make 
> libavcodec/libavcodec.a". This change raises that time from ~3.5 
> seconds to ~3.8 seconds. However do note that this is on a quite slow 
> system in itself; without the "rm", it takes make 2.3 seconds just to 
> figure out that nothing needs to be done.
>
> So on that level, the change indeed is mostly tolerable.
>
> However - this is very quick as long as "echo" is a shell builtin. If 
> "echo" turns out to be an external executable instead of the shell 
> builtin (which we can simulate by calling "/usr/bin/echo" instead of 
> "echo"), then this suddenly takes >16 seconds rather than the earlier 
> <4 seconds. And that's quite a steep price to pay.
>
> As noted before, this is only a fix for a potential, hypothetical 
> problem. The fix is inexpensive in the case of a builtin echo, where 
> we don't need the fix anyway. For the case of an external echo, where 
> we potentially could need the fix, the fix is quite expensive though.
>
> But even with the external /usr/bin/echo (on msys2), I still can 
> produce a very long (>32k) .objs file with only one single invocation 
> of /usr/bin/echo. So we don't actually have this problem even in that 
> case.
>
> So given that there are multiple concerns about the performance about 
> this, and the problem that it tries to fix is entirely hypothetical at 
> the moment, I would suggest that we skip this fix for now.
>
> If someone actually manages to hit the problem in some setup and can 
> tell us about it, we could reconsider of course.

Ok, I'll skip the piecewise patch.

But I'll note that just the linking step in isolation is not the 
relevant benchmark here. Most users who are not doing active ffmpeg 
development are building the whole thing. That means thousands of .o 
files. followed by linking external and internal libs.
So what they will see with an echo utility is closer to 3m30s vs 3m42s 
than 4s vs 16s, which is a minimal change for someone not iterating app 
development.

Regards,
Gyan

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Christopher Snowhill]

On Sat Mar 29, 2025 at 10:02 PM PDT, Gyan Doshi wrote:
>
>
> On 2025-03-29 11:52 pm, Martin Storsjö wrote:
>> On Sat, 29 Mar 2025, Gyan Doshi wrote:
>>
>>>> Did you not try to use GNU make's flie function?
>>>
>>> I just benched this and it ranges from 1m28.093s to 1m29.971s (5% 
>>> faster) for the lavc targets.
>>> However, this was added in make 4.0. Are we supporting older make?
>>
>> Yes, we generally do support older GNU make; macOS (even the latest 
>> versions) only ships with GNU make 3.81.
>>
>> Regarding measuring the runtime cost of this change; measuring the 
>> whole build time is quite uninteresting, the interesting bit is 
>> measuring the time to build e.g. an .a library on its own. So after a 
>> full build, I do "rm libavcodec/libavcodec.a; time make 
>> libavcodec/libavcodec.a". This change raises that time from ~3.5 
>> seconds to ~3.8 seconds. However do note that this is on a quite slow 
>> system in itself; without the "rm", it takes make 2.3 seconds just to 
>> figure out that nothing needs to be done.
>>
>> So on that level, the change indeed is mostly tolerable.
>>
>> However - this is very quick as long as "echo" is a shell builtin. If 
>> "echo" turns out to be an external executable instead of the shell 
>> builtin (which we can simulate by calling "/usr/bin/echo" instead of 
>> "echo"), then this suddenly takes >16 seconds rather than the earlier 
>> <4 seconds. And that's quite a steep price to pay.
>>
>> As noted before, this is only a fix for a potential, hypothetical 
>> problem. The fix is inexpensive in the case of a builtin echo, where 
>> we don't need the fix anyway. For the case of an external echo, where 
>> we potentially could need the fix, the fix is quite expensive though.
>>
>> But even with the external /usr/bin/echo (on msys2), I still can 
>> produce a very long (>32k) .objs file with only one single invocation 
>> of /usr/bin/echo. So we don't actually have this problem even in that 
>> case.
>>
>> So given that there are multiple concerns about the performance about 
>> this, and the problem that it tries to fix is entirely hypothetical at 
>> the moment, I would suggest that we skip this fix for now.
>>
>> If someone actually manages to hit the problem in some setup and can 
>> tell us about it, we could reconsider of course.
>
> Ok, I'll skip the piecewise patch.
>
> But I'll note that just the linking step in isolation is not the 
> relevant benchmark here. Most users who are not doing active ffmpeg 
> development are building the whole thing. That means thousands of .o 
> files. followed by linking external and internal libs.
> So what they will see with an echo utility is closer to 3m30s vs 3m42s 
> than 4s vs 16s, which is a minimal change for someone not iterating app 
> development.

I'm building on a Mac, so I'll either be using the Make supplied with
Xcode, or I'll have to install a newer gmake with Homebrew. I'm also
using overrides so the binaries will work on anything as old as 10.13.
(or 11.0 for Apple Silicon)

At least 10-15 seconds of each architecture's build process is running
the configure script with no feedback whatsoever until it completes.

>
> Regards,
> Gyan
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Andreas Rheinhardt]

Gyan Doshi:
> 
> 
> On 2025-03-29 11:52 pm, Martin Storsjö wrote:
>> On Sat, 29 Mar 2025, Gyan Doshi wrote:
>>
>>>> Did you not try to use GNU make's flie function?
>>>
>>> I just benched this and it ranges from 1m28.093s to 1m29.971s (5%
>>> faster) for the lavc targets.
>>> However, this was added in make 4.0. Are we supporting older make?
>>
>> Yes, we generally do support older GNU make; macOS (even the latest
>> versions) only ships with GNU make 3.81.
>>
>> Regarding measuring the runtime cost of this change; measuring the
>> whole build time is quite uninteresting, the interesting bit is
>> measuring the time to build e.g. an .a library on its own. So after a
>> full build, I do "rm libavcodec/libavcodec.a; time make libavcodec/
>> libavcodec.a". This change raises that time from ~3.5 seconds to ~3.8
>> seconds. However do note that this is on a quite slow system in
>> itself; without the "rm", it takes make 2.3 seconds just to figure out
>> that nothing needs to be done.
>>
>> So on that level, the change indeed is mostly tolerable.
>>
>> However - this is very quick as long as "echo" is a shell builtin. If
>> "echo" turns out to be an external executable instead of the shell
>> builtin (which we can simulate by calling "/usr/bin/echo" instead of
>> "echo"), then this suddenly takes >16 seconds rather than the earlier
>> <4 seconds. And that's quite a steep price to pay.
>>
>> As noted before, this is only a fix for a potential, hypothetical
>> problem. The fix is inexpensive in the case of a builtin echo, where
>> we don't need the fix anyway. For the case of an external echo, where
>> we potentially could need the fix, the fix is quite expensive though.
>>
>> But even with the external /usr/bin/echo (on msys2), I still can
>> produce a very long (>32k) .objs file with only one single invocation
>> of /usr/bin/echo. So we don't actually have this problem even in that
>> case.
>>
>> So given that there are multiple concerns about the performance about
>> this, and the problem that it tries to fix is entirely hypothetical at
>> the moment, I would suggest that we skip this fix for now.
>>
>> If someone actually manages to hit the problem in some setup and can
>> tell us about it, we could reconsider of course.
> 
> Ok, I'll skip the piecewise patch.
> 
> But I'll note that just the linking step in isolation is not the
> relevant benchmark here. Most users who are not doing active ffmpeg
> development are building the whole thing. That means thousands of .o
> files. followed by linking external and internal libs.
> So what they will see with an echo utility is closer to 3m30s vs 3m42s
> than 4s vs 16s, which is a minimal change for someone not iterating app
> development.

Completely wrong: People doing active ffmpeg development matter a lot.
The incremental build is the relevant benchmark.

- Andreas

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Gyan Doshi]

On 2025-03-30 11:57 am, Andreas Rheinhardt wrote:
> Gyan Doshi:
>>
>> On 2025-03-29 11:52 pm, Martin Storsjö wrote:
>>> On Sat, 29 Mar 2025, Gyan Doshi wrote:
>>>
>>>>> Did you not try to use GNU make's flie function?
>>>> I just benched this and it ranges from 1m28.093s to 1m29.971s (5%
>>>> faster) for the lavc targets.
>>>> However, this was added in make 4.0. Are we supporting older make?
>>> Yes, we generally do support older GNU make; macOS (even the latest
>>> versions) only ships with GNU make 3.81.
>>>
>>> Regarding measuring the runtime cost of this change; measuring the
>>> whole build time is quite uninteresting, the interesting bit is
>>> measuring the time to build e.g. an .a library on its own. So after a
>>> full build, I do "rm libavcodec/libavcodec.a; time make libavcodec/
>>> libavcodec.a". This change raises that time from ~3.5 seconds to ~3.8
>>> seconds. However do note that this is on a quite slow system in
>>> itself; without the "rm", it takes make 2.3 seconds just to figure out
>>> that nothing needs to be done.
>>>
>>> So on that level, the change indeed is mostly tolerable.
>>>
>>> However - this is very quick as long as "echo" is a shell builtin. If
>>> "echo" turns out to be an external executable instead of the shell
>>> builtin (which we can simulate by calling "/usr/bin/echo" instead of
>>> "echo"), then this suddenly takes >16 seconds rather than the earlier
>>> <4 seconds. And that's quite a steep price to pay.
>>>
>>> As noted before, this is only a fix for a potential, hypothetical
>>> problem. The fix is inexpensive in the case of a builtin echo, where
>>> we don't need the fix anyway. For the case of an external echo, where
>>> we potentially could need the fix, the fix is quite expensive though.
>>>
>>> But even with the external /usr/bin/echo (on msys2), I still can
>>> produce a very long (>32k) .objs file with only one single invocation
>>> of /usr/bin/echo. So we don't actually have this problem even in that
>>> case.
>>>
>>> So given that there are multiple concerns about the performance about
>>> this, and the problem that it tries to fix is entirely hypothetical at
>>> the moment, I would suggest that we skip this fix for now.
>>>
>>> If someone actually manages to hit the problem in some setup and can
>>> tell us about it, we could reconsider of course.
>> Ok, I'll skip the piecewise patch.
>>
>> But I'll note that just the linking step in isolation is not the
>> relevant benchmark here. Most users who are not doing active ffmpeg
>> development are building the whole thing. That means thousands of .o
>> files. followed by linking external and internal libs.
>> So what they will see with an echo utility is closer to 3m30s vs 3m42s
>> than 4s vs 16s, which is a minimal change for someone not iterating app
>> development.
> Completely wrong: People doing active ffmpeg development matter a lot.
> The incremental build is the relevant benchmark.

People doing active ffmpeg development i.e. those working on git master, 
should have modern shells with builtin echo
or can opt to disable response files thus avoiding the issue altogether. 
The primary beneficiaries of response files are
users of build scripts or binary providers like myself adding dozens of 
libraries.

Regards,
Gyan

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Ramiro Polla]

On Sun, Mar 30, 2025 at 7:02 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
> On 2025-03-29 11:52 pm, Martin Storsjö wrote:
> > On Sat, 29 Mar 2025, Gyan Doshi wrote:
> >
> >>> Did you not try to use GNU make's flie function?
> >>
> >> I just benched this and it ranges from 1m28.093s to 1m29.971s (5%
> >> faster) for the lavc targets.
> >> However, this was added in make 4.0. Are we supporting older make?
> >
> > Yes, we generally do support older GNU make; macOS (even the latest
> > versions) only ships with GNU make 3.81.
> >
> > Regarding measuring the runtime cost of this change; measuring the
> > whole build time is quite uninteresting, the interesting bit is
> > measuring the time to build e.g. an .a library on its own. So after a
> > full build, I do "rm libavcodec/libavcodec.a; time make
> > libavcodec/libavcodec.a". This change raises that time from ~3.5
> > seconds to ~3.8 seconds. However do note that this is on a quite slow
> > system in itself; without the "rm", it takes make 2.3 seconds just to
> > figure out that nothing needs to be done.
> >
> > So on that level, the change indeed is mostly tolerable.
> >
> > However - this is very quick as long as "echo" is a shell builtin. If
> > "echo" turns out to be an external executable instead of the shell
> > builtin (which we can simulate by calling "/usr/bin/echo" instead of
> > "echo"), then this suddenly takes >16 seconds rather than the earlier
> > <4 seconds. And that's quite a steep price to pay.
> >
> > As noted before, this is only a fix for a potential, hypothetical
> > problem. The fix is inexpensive in the case of a builtin echo, where
> > we don't need the fix anyway. For the case of an external echo, where
> > we potentially could need the fix, the fix is quite expensive though.
> >
> > But even with the external /usr/bin/echo (on msys2), I still can
> > produce a very long (>32k) .objs file with only one single invocation
> > of /usr/bin/echo. So we don't actually have this problem even in that
> > case.
> >
> > So given that there are multiple concerns about the performance about
> > this, and the problem that it tries to fix is entirely hypothetical at
> > the moment, I would suggest that we skip this fix for now.
> >
> > If someone actually manages to hit the problem in some setup and can
> > tell us about it, we could reconsider of course.

This sounds reasonable.

> Ok, I'll skip the piecewise patch.

Ok, and thank you for digging deeper into the issue.

Ramiro
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] ffbuild: compose linker response files in a loop

[Ramiro Polla]

On Sun, Mar 30, 2025 at 8:53 AM Gyan Doshi <ffmpeg@gyani.pro> wrote:
> On 2025-03-30 11:57 am, Andreas Rheinhardt wrote:
> > Gyan Doshi:
> >> On 2025-03-29 11:52 pm, Martin Storsjö wrote:
> >>> On Sat, 29 Mar 2025, Gyan Doshi wrote:
> >>>
> >>>>> Did you not try to use GNU make's flie function?
> >>>> I just benched this and it ranges from 1m28.093s to 1m29.971s (5%
> >>>> faster) for the lavc targets.
> >>>> However, this was added in make 4.0. Are we supporting older make?
> >>> Yes, we generally do support older GNU make; macOS (even the latest
> >>> versions) only ships with GNU make 3.81.
> >>>
> >>> Regarding measuring the runtime cost of this change; measuring the
> >>> whole build time is quite uninteresting, the interesting bit is
> >>> measuring the time to build e.g. an .a library on its own. So after a
> >>> full build, I do "rm libavcodec/libavcodec.a; time make libavcodec/
> >>> libavcodec.a". This change raises that time from ~3.5 seconds to ~3.8
> >>> seconds. However do note that this is on a quite slow system in
> >>> itself; without the "rm", it takes make 2.3 seconds just to figure out
> >>> that nothing needs to be done.
> >>>
> >>> So on that level, the change indeed is mostly tolerable.
> >>>
> >>> However - this is very quick as long as "echo" is a shell builtin. If
> >>> "echo" turns out to be an external executable instead of the shell
> >>> builtin (which we can simulate by calling "/usr/bin/echo" instead of
> >>> "echo"), then this suddenly takes >16 seconds rather than the earlier
> >>> <4 seconds. And that's quite a steep price to pay.
> >>>
> >>> As noted before, this is only a fix for a potential, hypothetical
> >>> problem. The fix is inexpensive in the case of a builtin echo, where
> >>> we don't need the fix anyway. For the case of an external echo, where
> >>> we potentially could need the fix, the fix is quite expensive though.
> >>>
> >>> But even with the external /usr/bin/echo (on msys2), I still can
> >>> produce a very long (>32k) .objs file with only one single invocation
> >>> of /usr/bin/echo. So we don't actually have this problem even in that
> >>> case.
> >>>
> >>> So given that there are multiple concerns about the performance about
> >>> this, and the problem that it tries to fix is entirely hypothetical at
> >>> the moment, I would suggest that we skip this fix for now.
> >>>
> >>> If someone actually manages to hit the problem in some setup and can
> >>> tell us about it, we could reconsider of course.
> >> Ok, I'll skip the piecewise patch.
> >>
> >> But I'll note that just the linking step in isolation is not the
> >> relevant benchmark here. Most users who are not doing active ffmpeg
> >> development are building the whole thing. That means thousands of .o
> >> files. followed by linking external and internal libs.
> >> So what they will see with an echo utility is closer to 3m30s vs 3m42s
> >> than 4s vs 16s, which is a minimal change for someone not iterating app
> >> development.

> > Completely wrong: People doing active ffmpeg development matter a lot.
> > The incremental build is the relevant benchmark.

Agreed.

> People doing active ffmpeg development i.e. those working on git master,
> should have modern shells with builtin echo
> or can opt to disable response files thus avoiding the issue altogether.

> The primary beneficiaries of response files are
> users of build scripts or binary providers like myself adding dozens of
> libraries.

Back when I used to do this on a daily basis, I found that it was
easier and faster to install virtualbox, install a linux distro, and
cross-compile to windows than it was to build everything natively. I
haven't tried building with msvc but I would not be surprised if it
was still easier and faster to cross-compile using wine.

Ramiro
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v2] configure: add option to select use of response files

[Gyan Doshi]

On 2025-03-29 10:42 am, Gyan Doshi wrote:
>
>
> On 2025-03-26 07:38 pm, Gyan Doshi wrote:
>> ---
>> v2:
>>     *switched state and make var name to response_files
>>     *changed option to standard action-optname format
>
> Plan to push tomorrow.

Pushed as 64087171f67bab220c4d3001eb6b074cf488284c

Regards,
Gyan

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".