[FFmpeg-devel] [PATCH 01/11] tests/swscale: allow choosing specific flags and dither mode

[FFmpeg-devel] [PATCH 01/11] tests/swscale: allow choosing specific flags and dither mode

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

So I can quickly iterate on the new swscale code.
---
 libswscale/tests/swscale.c | 42 ++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index 2e83197694..0027139154 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -47,6 +47,8 @@ struct options {
     int threads;
     int iters;
     int bench;
+    int flags;
+    int dither;
 };
 
 struct mode {
@@ -54,15 +56,15 @@ struct mode {
     SwsDither dither;
 };
 
-const struct mode modes[] = {
-    { SWS_FAST_BILINEAR },
-    { SWS_BILINEAR },
-    { SWS_BICUBIC },
-    { SWS_X | SWS_BITEXACT },
-    { SWS_POINT },
-    { SWS_AREA | SWS_ACCURATE_RND },
-    { SWS_BICUBIC | SWS_FULL_CHR_H_INT | SWS_FULL_CHR_H_INP },
-    {0}, // test defaults
+const SwsFlags flags[] = {
+    SWS_FAST_BILINEAR,
+    SWS_BILINEAR,
+    SWS_BICUBIC,
+    SWS_X | SWS_BITEXACT,
+    SWS_POINT,
+    SWS_AREA | SWS_ACCURATE_RND,
+    SWS_BICUBIC | SWS_FULL_CHR_H_INT | SWS_FULL_CHR_H_INP,
+    0, // test defaults
 };
 
 static FFSFC64 prng_state;
@@ -277,13 +279,21 @@ static int run_self_tests(const AVFrame *ref, struct options opts)
                 continue;
             for (int h = 0; h < FF_ARRAY_ELEMS(dst_h); h++)
                 for (int w = 0; w < FF_ARRAY_ELEMS(dst_w); w++)
-                    for (int m = 0; m < FF_ARRAY_ELEMS(modes); m++) {
+                    for (int f = 0; f < FF_ARRAY_ELEMS(flags); f++) {
+                        struct mode mode = {
+                            .flags  = opts.flags  >= 0 ? opts.flags  : flags[f],
+                            .dither = opts.dither >= 0 ? opts.dither : SWS_DITHER_AUTO,
+                        };
+
                         if (ff_sfc64_get(&prng_state) > UINT64_MAX * opts.prob)
                             continue;
 
                         if (run_test(src_fmt, dst_fmt, dst_w[w], dst_h[h],
-                                     modes[m], opts, ref, NULL) < 0)
+                                     mode, opts, ref, NULL) < 0)
                             return -1;
+
+                        if (opts.flags >= 0)
+                            break;
                     }
         }
     }
@@ -344,6 +354,8 @@ int main(int argc, char **argv)
         .threads = 1,
         .iters   = 1,
         .prob    = 1.0,
+        .flags   = -1,
+        .dither  = -1,
     };
 
     AVFrame *rgb = NULL, *ref = NULL;
@@ -368,6 +380,10 @@ int main(int argc, char **argv)
                     "       Only test the specified source pixel format\n"
                     "   -bench <iters>\n"
                     "       Run benchmarks with the specified number of iterations. This mode also increases the size of the test images\n"
+                    "   -flags <flags>\n"
+                    "       Test with a specific combination of flags\n"
+                    "   -dither <mode>\n"
+                    "       Test with a specific dither mode\n"
                     "   -threads <threads>\n"
                     "       Use the specified number of threads\n"
                     "   -cpuflags <cpuflags>\n"
@@ -409,6 +425,10 @@ int main(int argc, char **argv)
             opts.iters = FFMAX(opts.iters, 1);
             opts.w = 1920;
             opts.h = 1080;
+        } else if (!strcmp(argv[i], "-flags")) {
+            opts.flags = atoi(argv[i + 1]);
+        } else if (!strcmp(argv[i], "-dither")) {
+            opts.dither = atoi(argv[i + 1]);
         } else if (!strcmp(argv[i], "-threads")) {
             opts.threads = atoi(argv[i + 1]);
         } else if (!strcmp(argv[i], "-p")) {
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 02/11] tests/swscale: allow testing only unscaled convertors

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

I need this to be able to test the new unscaled conversion code more quickly.
We re-order the flags order to make 0 the first entry, so we don't set any
flags when performing unscaled tests.
---
 libswscale/tests/swscale.c | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index 0027139154..bdfea4e761 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -49,6 +49,7 @@ struct options {
     int bench;
     int flags;
     int dither;
+    int unscaled;
 };
 
 struct mode {
@@ -57,6 +58,7 @@ struct mode {
 };
 
 const SwsFlags flags[] = {
+    0, // test defaults
     SWS_FAST_BILINEAR,
     SWS_BILINEAR,
     SWS_BICUBIC,
@@ -64,7 +66,6 @@ const SwsFlags flags[] = {
     SWS_POINT,
     SWS_AREA | SWS_ACCURATE_RND,
     SWS_BICUBIC | SWS_FULL_CHR_H_INT | SWS_FULL_CHR_H_INP,
-    0, // test defaults
 };
 
 static FFSFC64 prng_state;
@@ -255,6 +256,12 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
     return ret;
 }
 
+static inline int fmt_is_subsampled(enum AVPixelFormat fmt)
+{
+    return av_pix_fmt_desc_get(fmt)->log2_chroma_w != 0 ||
+           av_pix_fmt_desc_get(fmt)->log2_chroma_h != 0;
+}
+
 static int run_self_tests(const AVFrame *ref, struct options opts)
 {
     const int dst_w[] = { opts.w, opts.w - opts.w / 3, opts.w + opts.w / 3 };
@@ -272,13 +279,17 @@ static int run_self_tests(const AVFrame *ref, struct options opts)
         dst_fmt_min = dst_fmt_max = opts.dst_fmt;
 
     for (src_fmt = src_fmt_min; src_fmt <= src_fmt_max; src_fmt++) {
+        if (opts.unscaled && fmt_is_subsampled(src_fmt))
+            continue;
         if (!sws_test_format(src_fmt, 0) || !sws_test_format(src_fmt, 1))
             continue;
         for (dst_fmt = dst_fmt_min; dst_fmt <= dst_fmt_max; dst_fmt++) {
+            if (opts.unscaled && fmt_is_subsampled(dst_fmt))
+                continue;
             if (!sws_test_format(dst_fmt, 0) || !sws_test_format(dst_fmt, 1))
                 continue;
-            for (int h = 0; h < FF_ARRAY_ELEMS(dst_h); h++)
-                for (int w = 0; w < FF_ARRAY_ELEMS(dst_w); w++)
+            for (int h = 0; h < FF_ARRAY_ELEMS(dst_h); h++) {
+                for (int w = 0; w < FF_ARRAY_ELEMS(dst_w); w++) {
                     for (int f = 0; f < FF_ARRAY_ELEMS(flags); f++) {
                         struct mode mode = {
                             .flags  = opts.flags  >= 0 ? opts.flags  : flags[f],
@@ -292,9 +303,15 @@ static int run_self_tests(const AVFrame *ref, struct options opts)
                                      mode, opts, ref, NULL) < 0)
                             return -1;
 
-                        if (opts.flags >= 0)
+                        if (opts.flags >= 0 || opts.unscaled)
                             break;
                     }
+                    if (opts.unscaled)
+                        break;
+                }
+                if (opts.unscaled)
+                    break;
+            }
         }
     }
 
@@ -384,6 +401,8 @@ int main(int argc, char **argv)
                     "       Test with a specific combination of flags\n"
                     "   -dither <mode>\n"
                     "       Test with a specific dither mode\n"
+                    "   -unscaled <1 or 0>\n"
+                    "       If 1, test only conversions that do not involve scaling\n"
                     "   -threads <threads>\n"
                     "       Use the specified number of threads\n"
                     "   -cpuflags <cpuflags>\n"
@@ -429,6 +448,8 @@ int main(int argc, char **argv)
             opts.flags = atoi(argv[i + 1]);
         } else if (!strcmp(argv[i], "-dither")) {
             opts.dither = atoi(argv[i + 1]);
+        } else if (!strcmp(argv[i], "-unscaled")) {
+            opts.unscaled = atoi(argv[i + 1]);
         } else if (!strcmp(argv[i], "-threads")) {
             opts.threads = atoi(argv[i + 1]);
         } else if (!strcmp(argv[i], "-p")) {
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 03/11] tests/swscale: print speedup numbers in color

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

---
 libswscale/tests/swscale.c | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index bdfea4e761..3f140f51f7 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -239,10 +239,17 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
     }
 
     if (opts.bench && time_ref) {
-        printf("  time=%"PRId64" us, ref=%"PRId64" us, speedup=%.3fx %s\n",
-                time / opts.iters, time_ref / opts.iters,
-                (double) time_ref / time,
-                time <= time_ref ? "faster" : "\033[1;33mslower\033[0m");
+        double ratio = (double) time_ref / time;
+        const char *color = ratio > 1.10 ? "\033[1;32m" : /* bold green */
+                            ratio > 1.02 ? "\033[32m"   : /* green */
+                            ratio > 0.98 ? ""           : /* default */
+                            ratio > 0.95 ? "\033[33m"   : /* yellow */
+                            ratio > 0.90 ? "\033[31m"   : /* red */
+                                           "\033[1;31m";  /* bold red */
+
+        printf("  time=%"PRId64" us, ref=%"PRId64" us, speedup=%.3fx %s%s\033[0m\n",
+               time / opts.iters, time_ref / opts.iters, ratio, color,
+               ratio >= 1.0 ? "faster" : "slower");
     } else if (opts.bench) {
         printf("  time=%"PRId64" us\n", time / opts.iters);
     }
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 04/11] tests/swscale: use yuva444p as reference

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

Instead of the lossy yuva420p. This does change the results compared to the
status quo, but is more reflective of the actual strength of a conversion,
since it will faithfully measure the round-trip error from subsampling and
upsampling.
---
 libswscale/tests/swscale.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index 3f140f51f7..8db5ee0af5 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -82,7 +82,7 @@ static int fmt_comps(enum AVPixelFormat fmt)
 
 static void get_mse(int mse[4], const AVFrame *a, const AVFrame *b, int comps)
 {
-    av_assert1(a->format == AV_PIX_FMT_YUVA420P);
+    av_assert1(a->format == AV_PIX_FMT_YUVA444P);
     av_assert1(b->format == a->format);
     av_assert1(b->width == a->width && b->height == a->height);
 
@@ -90,8 +90,8 @@ static void get_mse(int mse[4], const AVFrame *a, const AVFrame *b, int comps)
         const int is_chroma = p == 1 || p == 2;
         const int stride_a = a->linesize[p];
         const int stride_b = b->linesize[p];
-        const int w = (a->width + is_chroma) >> is_chroma;
-        const int h = (a->height + is_chroma) >> is_chroma;
+        const int w = a->width;
+        const int h = a->height;
         uint64_t sum = 0;
 
         if (comps & (1 << p)) {
@@ -499,7 +499,7 @@ bad_option:
         goto error;
     ref->width  = opts.w;
     ref->height = opts.h;
-    ref->format = AV_PIX_FMT_YUVA420P;
+    ref->format = AV_PIX_FMT_YUVA444P;
 
     if (sws_scale_frame(sws[0], ref, rgb) < 0)
         goto error;
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 05/11] tests/swscale: switch from MSE to SSIM

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

And bias it towards Y. This is much better at ignoring errors due to differing
dither patterns, and rewards algorithms that lower luma noise at the cost of
higher chroma noise.

The (0.8, 0.1, 0.1) weights for YCbCr are taken from the paper:
  "Understanding SSIM" by Jim Nilsson and Tomas Akenine-Möller
  (https://arxiv.org/abs/2006.13846)
---
 libswscale/tests/swscale.c | 108 ++++++++++++++++++++++---------------
 1 file changed, 65 insertions(+), 43 deletions(-)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index 8db5ee0af5..ea6c57ff2a 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -80,38 +80,51 @@ static int fmt_comps(enum AVPixelFormat fmt)
     return comps;
 }
 
-static void get_mse(int mse[4], const AVFrame *a, const AVFrame *b, int comps)
+static void get_ssim(float ssim[4], const AVFrame *out, const AVFrame *ref, int comps)
 {
-    av_assert1(a->format == AV_PIX_FMT_YUVA444P);
-    av_assert1(b->format == a->format);
-    av_assert1(b->width == a->width && b->height == a->height);
+    av_assert1(out->format == AV_PIX_FMT_YUVA444P);
+    av_assert1(ref->format == out->format);
+    av_assert1(ref->width == out->width && ref->height == out->height);
 
     for (int p = 0; p < 4; p++) {
+        const int stride_a = out->linesize[p];
+        const int stride_b = ref->linesize[p];
+        const int w = out->width;
+        const int h = out->height;
+
         const int is_chroma = p == 1 || p == 2;
-        const int stride_a = a->linesize[p];
-        const int stride_b = b->linesize[p];
-        const int w = a->width;
-        const int h = a->height;
-        uint64_t sum = 0;
-
-        if (comps & (1 << p)) {
-            for (int y = 0; y < h; y++) {
-                for (int x = 0; x < w; x++) {
-                    int d = a->data[p][y * stride_a + x] - b->data[p][y * stride_b + x];
-                    sum += d * d;
-                }
-            }
-        } else {
-            const int ref = is_chroma ? 128 : 0xFF;
-            for (int y = 0; y < h; y++) {
-                for (int x = 0; x < w; x++) {
-                    int d = a->data[p][y * stride_a + x] - ref;
-                    sum += d * d;
+        const uint8_t def = is_chroma ? 128 : 0xFF;
+        const int has_ref = comps & (1 << p);
+        double sum = 0;
+        int count = 0;
+
+        /* 4x4 SSIM */
+        for (int y = 0; y < (h & ~3); y += 4) {
+            for (int x = 0; x < (w & ~3); x += 4) {
+                const float c1 = .01 * .01 * 255 * 255 * 64;
+                const float c2 = .03 * .03 * 255 * 255 * 64 * 63;
+                int s1 = 0, s2 = 0, ss = 0, s12 = 0, var, covar;
+
+                for (int yy = 0; yy < 4; yy++) {
+                    for (int xx = 0; xx < 4; xx++) {
+                        int a = out->data[p][(y + yy) * stride_a + x + xx];
+                        int b = has_ref ? ref->data[p][(y + yy) * stride_b + x + xx] : def;
+                        s1  += a;
+                        s2  += b;
+                        ss  += a * a + b * b;
+                        s12 += a * b;
+                    }
                 }
+
+                var = ss * 64 - s1 * s1 - s2 * s2;
+                covar = s12 * 64 - s1 * s2;
+                sum += (2 * s1 * s2 + c1) * (2 * covar + c2) /
+                       ((s1 * s1 + s2 * s2 + c1) * (var + c2));
+                count++;
             }
         }
 
-        mse[p] = sum / (w * h);
+        ssim[p] = count ? sum / count : 0.0;
     }
 }
 
@@ -149,12 +162,13 @@ error:
 /* Runs a series of ref -> src -> dst -> out, and compares out vs ref */
 static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
                     int dst_w, int dst_h, struct mode mode, struct options opts,
-                    const AVFrame *ref, const int mse_ref[4])
+                    const AVFrame *ref, const float ssim_ref[4])
 {
     AVFrame *src = NULL, *dst = NULL, *out = NULL;
-    int mse[4], mse_sws[4], ret = -1;
+    float ssim[4], ssim_sws[4];
     const int comps = fmt_comps(src_fmt) & fmt_comps(dst_fmt);
     int64_t time, time_ref = 0;
+    int ret = -1;
 
     src = av_frame_alloc();
     dst = av_frame_alloc();
@@ -201,15 +215,15 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
         goto error;
     }
 
-    get_mse(mse, out, ref, comps);
+    get_ssim(ssim, out, ref, comps);
     printf("%s %dx%d -> %s %3dx%3d, flags=%u dither=%u, "
-           "MSE={%5d %5d %5d %5d}\n",
+           "SSIM {Y=%f U=%f V=%f A=%f}\n",
            av_get_pix_fmt_name(src->format), src->width, src->height,
            av_get_pix_fmt_name(dst->format), dst->width, dst->height,
            mode.flags, mode.dither,
-           mse[0], mse[1], mse[2], mse[3]);
+           ssim[0], ssim[1], ssim[2], ssim[3]);
 
-    if (!mse_ref) {
+    if (!ssim_ref) {
         /* Compare against the legacy swscale API as a reference */
         time_ref = av_gettime_relative();
         if (scale_legacy(dst, src, mode, opts) < 0) {
@@ -222,19 +236,26 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
         if (sws_scale_frame(sws[2], out, dst) < 0)
             goto error;
 
-        get_mse(mse_sws, out, ref, comps);
-        mse_ref = mse_sws;
+        get_ssim(ssim_sws, out, ref, comps);
+        ssim_ref = ssim_sws;
     }
 
-    for (int i = 0; i < 4; i++) {
-        if (mse[i] > mse_ref[i]) {
-            int bad = mse[i] > mse_ref[i] * 1.02 + 1;
-            printf("\033[1;31m  %s, ref MSE={%5d %5d %5d %5d}\033[0m\n",
-                   bad ? "WORSE" : "worse",
-                   mse_ref[0], mse_ref[1], mse_ref[2], mse_ref[3]);
+    if (ssim_ref) {
+        const float weights[4] = { 0.8, 0.1, 0.1, 1.0 }; /* tuned for Y'CrCr */
+        float err, sum = 0, sum_ref = 0;
+        for (int i = 0; i < 4; i++) {
+            sum     += weights[i] * ssim[i];
+            sum_ref += weights[i] * ssim_ref[i];
+        }
+
+        err = sum_ref / sum - 1.0; /* relative error */
+        if (err > 1e-4 /* 0.01% headroom for dither noise etc */) {
+            int bad = err > 1e-2; /* 1% */
+            printf("\033[1;31m  %s by %f%%, ref SSIM {Y=%f U=%f V=%f A=%f}\033[0m\n",
+                   bad ? "WORSE" : "worse", 100.0 * err,
+                   ssim_ref[0], ssim_ref[1], ssim_ref[2], ssim_ref[3]);
             if (bad)
                 goto error;
-            break;
         }
     }
 
@@ -334,15 +355,16 @@ static int run_file_tests(const AVFrame *ref, FILE *fp, struct options opts)
         char src_fmt_str[20], dst_fmt_str[20];
         enum AVPixelFormat src_fmt;
         enum AVPixelFormat dst_fmt;
-        int sw, sh, dw, dh, mse[4];
+        int sw, sh, dw, dh;
+        float ssim[4];
         struct mode mode;
 
         ret = sscanf(buf,
                      " %20s %dx%d -> %20s %dx%d, flags=%u dither=%u, "
-                     "MSE={%d %d %d %d}\n",
+                     "SSIM {Y=%f U=%f V=%f A=%f}\n",
                      src_fmt_str, &sw, &sh, dst_fmt_str, &dw, &dh,
                      &mode.flags, &mode.dither,
-                     &mse[0], &mse[1], &mse[2], &mse[3]);
+                     &ssim[0], &ssim[1], &ssim[2], &ssim[3]);
         if (ret != 12) {
             printf("%s", buf);
             continue;
@@ -361,7 +383,7 @@ static int run_file_tests(const AVFrame *ref, FILE *fp, struct options opts)
             opts.dst_fmt != AV_PIX_FMT_NONE && dst_fmt != opts.dst_fmt)
             continue;
 
-        if (run_test(src_fmt, dst_fmt, dw, dh, mode, opts, ref, mse) < 0)
+        if (run_test(src_fmt, dst_fmt, dw, dh, mode, opts, ref, ssim) < 0)
             return -1;
     }
 
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 06/11] tests/swscale: print performance stats on exit

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

---
 libswscale/tests/swscale.c | 47 ++++++++++++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 9 deletions(-)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index ea6c57ff2a..a2b960faa3 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -24,6 +24,7 @@
 #include <string.h>
 #include <inttypes.h>
 #include <stdarg.h>
+#include <signal.h>
 
 #undef HAVE_AV_CONFIG_H
 #include "libavutil/cpu.h"
@@ -71,6 +72,33 @@ const SwsFlags flags[] = {
 static FFSFC64 prng_state;
 static SwsContext *sws[3]; /* reused between tests for efficiency */
 
+static double speedup_logavg;
+static double speedup_min = 1e10;
+static double speedup_max = 0;
+static int speedup_count;
+
+static const char *speedup_color(double ratio)
+{
+    return ratio > 1.10 ? "\033[1;32m" : /* bold green */
+           ratio > 1.02 ? "\033[32m"   : /* green */
+           ratio > 0.98 ? ""           : /* default */
+           ratio > 0.95 ? "\033[33m"   : /* yellow */
+           ratio > 0.90 ? "\033[31m"   : /* red */
+            "\033[1;31m";  /* bold red */
+}
+
+static void exit_handler(int sig)
+{
+    if (speedup_count) {
+        double ratio = exp(speedup_logavg / speedup_count);
+        printf("Overall speedup=%.3fx %s%s\033[0m, min=%.3fx max=%.3fx\n", ratio,
+               speedup_color(ratio), ratio >= 1.0 ? "faster" : "slower",
+               speedup_min, speedup_max);
+    }
+
+    exit(sig);
+}
+
 static int fmt_comps(enum AVPixelFormat fmt)
 {
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
@@ -261,16 +289,16 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
 
     if (opts.bench && time_ref) {
         double ratio = (double) time_ref / time;
-        const char *color = ratio > 1.10 ? "\033[1;32m" : /* bold green */
-                            ratio > 1.02 ? "\033[32m"   : /* green */
-                            ratio > 0.98 ? ""           : /* default */
-                            ratio > 0.95 ? "\033[33m"   : /* yellow */
-                            ratio > 0.90 ? "\033[31m"   : /* red */
-                                           "\033[1;31m";  /* bold red */
+        if (FFMIN(time, time_ref) > 100 /* don't pollute stats with low precision */) {
+            speedup_min = FFMIN(speedup_min, ratio);
+            speedup_max = FFMAX(speedup_max, ratio);
+            speedup_logavg += log(ratio);
+            speedup_count++;
+        }
 
         printf("  time=%"PRId64" us, ref=%"PRId64" us, speedup=%.3fx %s%s\033[0m\n",
-               time / opts.iters, time_ref / opts.iters, ratio, color,
-               ratio >= 1.0 ? "faster" : "slower");
+               time / opts.iters, time_ref / opts.iters, ratio,
+               speedup_color(ratio), ratio >= 1.0 ? "faster" : "slower");
     } else if (opts.bench) {
         printf("  time=%"PRId64" us\n", time / opts.iters);
     }
@@ -492,6 +520,7 @@ bad_option:
 
     ff_sfc64_init(&prng_state, 0, 0, 0, 12);
     av_lfg_init(&rand, 1);
+    signal(SIGINT, exit_handler);
 
     for (int i = 0; i < 3; i++) {
         sws[i] = sws_alloc_context();
@@ -537,5 +566,5 @@ error:
     av_frame_free(&ref);
     if (fp)
         fclose(fp);
-    return ret;
+    exit_handler(ret);
 }
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 07/11] tests/swscale: check supported inputs for legacy swscale separately

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

The new code path supports more formats, so we can't test them all
against the legacy implementation.
---
 libswscale/tests/swscale.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index a2b960faa3..daf6cf769c 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -251,7 +251,7 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
            mode.flags, mode.dither,
            ssim[0], ssim[1], ssim[2], ssim[3]);
 
-    if (!ssim_ref) {
+    if (!ssim_ref && sws_isSupportedInput(src->format) && sws_isSupportedOutput(dst->format)) {
         /* Compare against the legacy swscale API as a reference */
         time_ref = av_gettime_relative();
         if (scale_legacy(dst, src, mode, opts) < 0) {
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 08/11] tests/swscale: remove stray whitespace in scanf format

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

---
 libswscale/tests/swscale.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index daf6cf769c..47c58524f6 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -388,7 +388,7 @@ static int run_file_tests(const AVFrame *ref, FILE *fp, struct options opts)
         struct mode mode;
 
         ret = sscanf(buf,
-                     " %20s %dx%d -> %20s %dx%d, flags=%u dither=%u, "
+                     "%20s %dx%d -> %20s %dx%d, flags=%u dither=%u, "
                      "SSIM {Y=%f U=%f V=%f A=%f}\n",
                      src_fmt_str, &sw, &sh, dst_fmt_str, &dw, &dh,
                      &mode.flags, &mode.dither,
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 09/11] tests/swscale: calculate theoretical expected SSIM

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

We can calculate with some confidence the theoretical expected SSIM
from an "ideal" conversion, by computing the reference SSIM level
for an image dithered with uniformly distributed quatization noise.

This gives us an additional safety net to check for regressions even in
the absence of a reference to compare against.
---
 libswscale/tests/swscale.c | 74 +++++++++++++++++++++++++++++++-------
 1 file changed, 62 insertions(+), 12 deletions(-)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index 47c58524f6..bce495db90 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -99,6 +99,29 @@ static void exit_handler(int sig)
     exit(sig);
 }
 
+/* Estimate luma variance assuming uniform dither noise distribution */
+static float estimate_quantization_noise(enum AVPixelFormat fmt)
+{
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
+    float variance = 1.0 / 12;
+    if (desc->comp[0].depth < 8) {
+        /* Extra headroom for very low bit depth output */
+        variance *= (8 - desc->comp[0].depth);
+    }
+
+    if (desc->flags & AV_PIX_FMT_FLAG_FLOAT) {
+        return 0.0;
+    } else if (desc->flags & AV_PIX_FMT_FLAG_RGB) {
+        const float r = 0.299 / (1 << desc->comp[0].depth);
+        const float g = 0.587 / (1 << desc->comp[1].depth);
+        const float b = 0.114 / (1 << desc->comp[2].depth);
+        return (r * r + g * g + b * b) * variance;
+    } else {
+        const float y = 1.0 / (1 << desc->comp[0].depth);
+        return y * y * variance;
+    }
+}
+
 static int fmt_comps(enum AVPixelFormat fmt)
 {
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
@@ -156,6 +179,18 @@ static void get_ssim(float ssim[4], const AVFrame *out, const AVFrame *ref, int
     }
 }
 
+static float get_loss(const float ssim[4])
+{
+    const float weights[3] = { 0.8, 0.1, 0.1 }; /* tuned for Y'CrCr */
+
+    float sum = 0;
+    for (int i = 0; i < 3; i++)
+        sum += weights[i] * ssim[i];
+    sum *= ssim[3]; /* ensure alpha errors get caught */
+
+    return 1.0 - sum;
+}
+
 static int scale_legacy(AVFrame *dst, const AVFrame *src, struct mode mode,
                         struct options opts)
 {
@@ -198,6 +233,18 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
     int64_t time, time_ref = 0;
     int ret = -1;
 
+    /* Estimate the expected amount of loss from bit depth reduction */
+    const float c1 = 0.01 * 0.01; /* stabilization constant */
+    const float ref_var = 1.0 / 12.0; /* uniformly distributed signal */
+    const float src_var = estimate_quantization_noise(src_fmt);
+    const float dst_var = estimate_quantization_noise(dst_fmt);
+    const float out_var = estimate_quantization_noise(ref->format);
+    const float total_var = src_var + dst_var + out_var;
+    const float ssim_luma = (2 * ref_var + c1) / (2 * ref_var + total_var + c1);
+    const float ssim_expected[4] = { ssim_luma, 1, 1, 1 }; /* for simplicity */
+    const float expected_loss = get_loss(ssim_expected);
+    float loss;
+
     src = av_frame_alloc();
     dst = av_frame_alloc();
     out = av_frame_alloc();
@@ -251,6 +298,15 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
            mode.flags, mode.dither,
            ssim[0], ssim[1], ssim[2], ssim[3]);
 
+    loss = get_loss(ssim);
+    if (loss - expected_loss > 1e-4 && dst_w >= ref->width && dst_h >= ref->height) {
+        int bad = loss - expected_loss > 1e-2;
+        printf("\033[1;31m  loss %g is %s by %g, expected loss %g\033[0m\n",
+               loss, bad ? "WORSE" : "worse", loss - expected_loss, expected_loss);
+        if (bad)
+            goto error;
+    }
+
     if (!ssim_ref && sws_isSupportedInput(src->format) && sws_isSupportedOutput(dst->format)) {
         /* Compare against the legacy swscale API as a reference */
         time_ref = av_gettime_relative();
@@ -269,18 +325,12 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
     }
 
     if (ssim_ref) {
-        const float weights[4] = { 0.8, 0.1, 0.1, 1.0 }; /* tuned for Y'CrCr */
-        float err, sum = 0, sum_ref = 0;
-        for (int i = 0; i < 4; i++) {
-            sum     += weights[i] * ssim[i];
-            sum_ref += weights[i] * ssim_ref[i];
-        }
-
-        err = sum_ref / sum - 1.0; /* relative error */
-        if (err > 1e-4 /* 0.01% headroom for dither noise etc */) {
-            int bad = err > 1e-2; /* 1% */
-            printf("\033[1;31m  %s by %f%%, ref SSIM {Y=%f U=%f V=%f A=%f}\033[0m\n",
-                   bad ? "WORSE" : "worse", 100.0 * err,
+        const float loss_ref = get_loss(ssim_ref);
+        if (loss - loss_ref > 1e-4) {
+            int bad = loss - loss_ref > 1e-2;
+            printf("\033[1;31m  loss %g is %s by %g, ref loss %g, "
+                   "SSIM {Y=%f U=%f V=%f A=%f}\033[0m\n",
+                   loss, bad ? "WORSE" : "worse", loss - loss_ref, loss_ref,
                    ssim_ref[0], ssim_ref[1], ssim_ref[2], ssim_ref[3]);
             if (bad)
                 goto error;
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 09/11] tests/swscale: calculate theoretical expected SSIM

[Niklas Haas]

On Mon, 17 Mar 2025 11:43:55 +0100 Niklas Haas <ffmpeg@haasn.xyz> wrote:
> From: Niklas Haas <git@haasn.dev>
>
> We can calculate with some confidence the theoretical expected SSIM
> from an "ideal" conversion, by computing the reference SSIM level
> for an image dithered with uniformly distributed quatization noise.
>
> This gives us an additional safety net to check for regressions even in
> the absence of a reference to compare against.

It's worth pointing out that this does reveal some bugs in the current
implementation that were not covered by any pre existing tests.

> ---
>  libswscale/tests/swscale.c | 74 +++++++++++++++++++++++++++++++-------
>  1 file changed, 62 insertions(+), 12 deletions(-)
>
> diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
> index 47c58524f6..bce495db90 100644
> --- a/libswscale/tests/swscale.c
> +++ b/libswscale/tests/swscale.c
> @@ -99,6 +99,29 @@ static void exit_handler(int sig)
>      exit(sig);
>  }
>
> +/* Estimate luma variance assuming uniform dither noise distribution */
> +static float estimate_quantization_noise(enum AVPixelFormat fmt)
> +{
> +    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
> +    float variance = 1.0 / 12;
> +    if (desc->comp[0].depth < 8) {
> +        /* Extra headroom for very low bit depth output */
> +        variance *= (8 - desc->comp[0].depth);
> +    }
> +
> +    if (desc->flags & AV_PIX_FMT_FLAG_FLOAT) {
> +        return 0.0;
> +    } else if (desc->flags & AV_PIX_FMT_FLAG_RGB) {
> +        const float r = 0.299 / (1 << desc->comp[0].depth);
> +        const float g = 0.587 / (1 << desc->comp[1].depth);
> +        const float b = 0.114 / (1 << desc->comp[2].depth);
> +        return (r * r + g * g + b * b) * variance;
> +    } else {
> +        const float y = 1.0 / (1 << desc->comp[0].depth);
> +        return y * y * variance;
> +    }
> +}
> +
>  static int fmt_comps(enum AVPixelFormat fmt)
>  {
>      const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
> @@ -156,6 +179,18 @@ static void get_ssim(float ssim[4], const AVFrame *out, const AVFrame *ref, int
>      }
>  }
>
> +static float get_loss(const float ssim[4])
> +{
> +    const float weights[3] = { 0.8, 0.1, 0.1 }; /* tuned for Y'CrCr */
> +
> +    float sum = 0;
> +    for (int i = 0; i < 3; i++)
> +        sum += weights[i] * ssim[i];
> +    sum *= ssim[3]; /* ensure alpha errors get caught */
> +
> +    return 1.0 - sum;
> +}
> +
>  static int scale_legacy(AVFrame *dst, const AVFrame *src, struct mode mode,
>                          struct options opts)
>  {
> @@ -198,6 +233,18 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
>      int64_t time, time_ref = 0;
>      int ret = -1;
>
> +    /* Estimate the expected amount of loss from bit depth reduction */
> +    const float c1 = 0.01 * 0.01; /* stabilization constant */
> +    const float ref_var = 1.0 / 12.0; /* uniformly distributed signal */
> +    const float src_var = estimate_quantization_noise(src_fmt);
> +    const float dst_var = estimate_quantization_noise(dst_fmt);
> +    const float out_var = estimate_quantization_noise(ref->format);
> +    const float total_var = src_var + dst_var + out_var;
> +    const float ssim_luma = (2 * ref_var + c1) / (2 * ref_var + total_var + c1);
> +    const float ssim_expected[4] = { ssim_luma, 1, 1, 1 }; /* for simplicity */
> +    const float expected_loss = get_loss(ssim_expected);
> +    float loss;
> +
>      src = av_frame_alloc();
>      dst = av_frame_alloc();
>      out = av_frame_alloc();
> @@ -251,6 +298,15 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
>             mode.flags, mode.dither,
>             ssim[0], ssim[1], ssim[2], ssim[3]);
>
> +    loss = get_loss(ssim);
> +    if (loss - expected_loss > 1e-4 && dst_w >= ref->width && dst_h >= ref->height) {
> +        int bad = loss - expected_loss > 1e-2;
> +        printf("\033[1;31m  loss %g is %s by %g, expected loss %g\033[0m\n",
> +               loss, bad ? "WORSE" : "worse", loss - expected_loss, expected_loss);
> +        if (bad)
> +            goto error;
> +    }
> +
>      if (!ssim_ref && sws_isSupportedInput(src->format) && sws_isSupportedOutput(dst->format)) {
>          /* Compare against the legacy swscale API as a reference */
>          time_ref = av_gettime_relative();
> @@ -269,18 +325,12 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
>      }
>
>      if (ssim_ref) {
> -        const float weights[4] = { 0.8, 0.1, 0.1, 1.0 }; /* tuned for Y'CrCr */
> -        float err, sum = 0, sum_ref = 0;
> -        for (int i = 0; i < 4; i++) {
> -            sum     += weights[i] * ssim[i];
> -            sum_ref += weights[i] * ssim_ref[i];
> -        }
> -
> -        err = sum_ref / sum - 1.0; /* relative error */
> -        if (err > 1e-4 /* 0.01% headroom for dither noise etc */) {
> -            int bad = err > 1e-2; /* 1% */
> -            printf("\033[1;31m  %s by %f%%, ref SSIM {Y=%f U=%f V=%f A=%f}\033[0m\n",
> -                   bad ? "WORSE" : "worse", 100.0 * err,
> +        const float loss_ref = get_loss(ssim_ref);
> +        if (loss - loss_ref > 1e-4) {
> +            int bad = loss - loss_ref > 1e-2;
> +            printf("\033[1;31m  loss %g is %s by %g, ref loss %g, "
> +                   "SSIM {Y=%f U=%f V=%f A=%f}\033[0m\n",
> +                   loss, bad ? "WORSE" : "worse", loss - loss_ref, loss_ref,
>                     ssim_ref[0], ssim_ref[1], ssim_ref[2], ssim_ref[3]);
>              if (bad)
>                  goto error;
> --
> 2.48.1
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 10/11] tests/swscale: constrain reference SSIM for low bit depth formats

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

Sometimes, the reference SSIM is significantly higher than the
SSIM level expected for the test. This is the case when the source format
has a much lower bit depth than the destination format. In this case, the fact
that legacy swscale does not accurately preserve the source dither pattern
gives it an unfair advantage in a direct comparison, leading to false
positives.

For example, conversion like rgb4 -> rgb565 should be lossless, but swscale
low passes / downscales the input chroma, throwing away massive amounts of
detail. This gives it a higher SSIM score since the lowpassed result removes
some of the dither noise that was present in the source.
---
 libswscale/tests/swscale.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index bce495db90..117ed2144e 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -321,6 +321,18 @@ static int run_test(enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt,
             goto error;
 
         get_ssim(ssim_sws, out, ref, comps);
+
+        /* Legacy swscale does not perform bit accurate upconversions of low
+         * bit depth RGB. This artificially improves the SSIM score because the
+         * resulting error deletes some of the input dither noise. This gives
+         * it an unfair advantage when compared against a bit exact reference.
+         * Work around this by ensuring that the reference SSIM score is not
+         * higher than it theoretically "should" be. */
+        if (src_var > dst_var) {
+            const float src_loss = (2 * ref_var + c1) / (2 * ref_var + src_var + c1);
+            ssim_sws[0] = FFMIN(ssim_sws[0], src_loss);
+        }
+
         ssim_ref = ssim_sws;
     }
 
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 11/11] tests/swscale: allow setting log verbosity

[Niklas Haas]

From: Niklas Haas <git@haasn.dev>

Helpful for debugging the new swscale code, since it dumps the
operations list in verbose logging mode.
---
 libswscale/tests/swscale.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libswscale/tests/swscale.c b/libswscale/tests/swscale.c
index 117ed2144e..0984536555 100644
--- a/libswscale/tests/swscale.c
+++ b/libswscale/tests/swscale.c
@@ -526,6 +526,8 @@ int main(int argc, char **argv)
                     "       Use the specified number of threads\n"
                     "   -cpuflags <cpuflags>\n"
                     "       Uses the specified cpuflags in the tests\n"
+                    "   -v <level>\n"
+                    "       Enable log verbosity at given level\n"
             );
             return 0;
         }
@@ -573,6 +575,8 @@ int main(int argc, char **argv)
             opts.threads = atoi(argv[i + 1]);
         } else if (!strcmp(argv[i], "-p")) {
             opts.prob = atof(argv[i + 1]);
+        } else if (!strcmp(argv[i], "-v")) {
+            av_log_set_level(atoi(argv[i + 1]));
         } else {
 bad_option:
             fprintf(stderr, "bad option or argument missing (%s) see -help\n", argv[i]);
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".