[FFmpeg-devel] [PATCH] avfilter/vf_libplacebo: fix a more edge cases when combining rotated video with custom fit modes (PR #20884)

[FFmpeg-devel] [PATCH] avfilter/vf_libplacebo: fix a more edge cases when combining rotated video with custom fit modes (PR #20884)

[Niklas Haas via ffmpeg-devel]

PR #20884 opened by Niklas Haas (haasn)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20884
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20884.patch

This whole logic is beginning to get rather annoying and I'm starting to feel like a whole redesign is needed.


>From 2cc48b34340738f30fc9daa2a61a60e9cf1eb19d Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Mon, 10 Nov 2025 12:32:32 +0100
Subject: [PATCH 1/3] avfilter/vf_libplacebo: un-rotate image crop after
 fitting

When combining rotation with a FIT_ mode other than FIT_FILL, the fitting
logic was operating on the un-rotated rects, when it should have been
operating on the rotated (output) rects.
---
 libavfilter/vf_libplacebo.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/libavfilter/vf_libplacebo.c b/libavfilter/vf_libplacebo.c
index 42501c51f2..5a615aceed 100644
--- a/libavfilter/vf_libplacebo.c
+++ b/libavfilter/vf_libplacebo.c
@@ -913,14 +913,6 @@ static void update_crops(AVFilterContext *ctx, LibplaceboInput *in,
         image->crop.x1 = image->crop.x0 + s->var_values[VAR_CROP_W];
         image->crop.y1 = image->crop.y0 + s->var_values[VAR_CROP_H];
 
-        const pl_rotation rot_total = image->rotation - target->rotation;
-        if ((rot_total + PL_ROTATION_360) % PL_ROTATION_180 == PL_ROTATION_90) {
-            /* Libplacebo expects the input crop relative to the actual frame
-             * dimensions, so un-transpose them here */
-            FFSWAP(float, image->crop.x0, image->crop.y0);
-            FFSWAP(float, image->crop.x1, image->crop.y1);
-        }
-
         if (src == ref) {
             /* Only update the target crop once, for the 'reference' frame */
             target->crop.x0 = av_expr_eval(s->pos_x_pexpr, s->var_values, NULL);
@@ -955,6 +947,14 @@ static void update_crops(AVFilterContext *ctx, LibplaceboInput *in,
             case FIT_SCALE_DOWN:
                 pl_rect2df_aspect_fit(&target->crop, &fixed, 0.0);
             }
+
+            const pl_rotation rot_total = image->rotation - target->rotation;
+            if ((rot_total + PL_ROTATION_360) % PL_ROTATION_180 == PL_ROTATION_90) {
+                /* Libplacebo expects the input crop relative to the actual frame
+                 * dimensions, so un-transpose them here */
+                FFSWAP(float, image->crop.x0, image->crop.y0);
+                FFSWAP(float, image->crop.x1, image->crop.y1);
+            }
         }
     }
 }
-- 
2.49.1


>From c56c5f80f5256be33453b2aaad06e42132aba09d Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Mon, 10 Nov 2025 12:36:33 +0100
Subject: [PATCH 2/3] avfilter/vf_libplacebo: fix fitting math when SAR is
 undefined

---
 libavfilter/vf_libplacebo.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libavfilter/vf_libplacebo.c b/libavfilter/vf_libplacebo.c
index 5a615aceed..6ae7289265 100644
--- a/libavfilter/vf_libplacebo.c
+++ b/libavfilter/vf_libplacebo.c
@@ -920,9 +920,13 @@ static void update_crops(AVFilterContext *ctx, LibplaceboInput *in,
             target->crop.x1 = target->crop.x0 + s->var_values[VAR_POS_W];
             target->crop.y1 = target->crop.y0 + s->var_values[VAR_POS_H];
 
+
             /* Effective visual crop */
-            const float w_adj = av_q2d(inlink->sample_aspect_ratio) /
-                                av_q2d(outlink->sample_aspect_ratio);
+            double sar_in = inlink->sample_aspect_ratio.num ?
+                            av_q2d(inlink->sample_aspect_ratio) : 1.0;
+            double sar_out = outlink->sample_aspect_ratio.num ?
+                             av_q2d(outlink->sample_aspect_ratio) : 1.0;
+            const float w_adj = sar_in / sar_out;
 
             pl_rect2df fixed = image->crop;
             pl_rect2df_stretch(&fixed, w_adj, 1.0);
-- 
2.49.1


>From 16391108ba87bd735c1233276afa5699b2f3b657 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Mon, 10 Nov 2025 12:38:11 +0100
Subject: [PATCH 3/3] avfilter/vf_libplacebo: also rotate SAR when fitting

---
 libavfilter/vf_libplacebo.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/libavfilter/vf_libplacebo.c b/libavfilter/vf_libplacebo.c
index 6ae7289265..1a9dc05620 100644
--- a/libavfilter/vf_libplacebo.c
+++ b/libavfilter/vf_libplacebo.c
@@ -926,10 +926,13 @@ static void update_crops(AVFilterContext *ctx, LibplaceboInput *in,
                             av_q2d(inlink->sample_aspect_ratio) : 1.0;
             double sar_out = outlink->sample_aspect_ratio.num ?
                              av_q2d(outlink->sample_aspect_ratio) : 1.0;
-            const float w_adj = sar_in / sar_out;
+
+            pl_rotation rot_total = PL_ROTATION_360 + image->rotation - target->rotation;
+            if (rot_total % PL_ROTATION_180 == PL_ROTATION_90)
+                sar_in = 1.0 / sar_in;
 
             pl_rect2df fixed = image->crop;
-            pl_rect2df_stretch(&fixed, w_adj, 1.0);
+            pl_rect2df_stretch(&fixed, sar_in / sar_out, 1.0);
 
             switch (s->fit_mode) {
             case FIT_FILL:
@@ -952,8 +955,7 @@ static void update_crops(AVFilterContext *ctx, LibplaceboInput *in,
                 pl_rect2df_aspect_fit(&target->crop, &fixed, 0.0);
             }
 
-            const pl_rotation rot_total = image->rotation - target->rotation;
-            if ((rot_total + PL_ROTATION_360) % PL_ROTATION_180 == PL_ROTATION_90) {
+            if (rot_total % PL_ROTATION_180 == PL_ROTATION_90) {
                 /* Libplacebo expects the input crop relative to the actual frame
                  * dimensions, so un-transpose them here */
                 FFSWAP(float, image->crop.x0, image->crop.y0);
-- 
2.49.1

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org