[FFmpeg-devel] [PATCH 1/2] fate: add mse and peak error comparison with reference image

[pal@sandflow.com]

From: Pierre-Anthony Lemieux <pal@palemieux.com>

As discussed at [1], this draft patch adds the ability to create FATE tests that
compare FFMPEG processing results with a reference image, using peak error and mse
error metrics.

[1] https://ffmpeg.org/pipermail/ffmpeg-devel/2024-November/335778.html

This is useful for testing codecs that are not necessarily bit accurate across platforms.

The patch applies this new test metric to a single JPEG 2000 conformance file.

Looking forward to your feedback on the overall approach before I extend it to all lossly
JPEG 2000 conformance files.

---
 libavfilter/vf_psnr.c | 70 +++++++++++++++++++++++++++++++++++--------
 tests/fate-run.sh     | 32 ++++++++++++++++++++
 2 files changed, 90 insertions(+), 12 deletions(-)

diff --git a/libavfilter/vf_psnr.c b/libavfilter/vf_psnr.c
index 4a173c73d9..3f55562cec 100644
--- a/libavfilter/vf_psnr.c
+++ b/libavfilter/vf_psnr.c
@@ -36,6 +36,11 @@
 #include "framesync.h"
 #include "psnr.h"
 
+typedef struct Score {
+    uint64_t mse;
+    uint64_t peak;
+} Score;
+
 typedef struct PSNRContext {
     const AVClass *class;
     FFFrameSync fs;
@@ -47,7 +52,9 @@ typedef struct PSNRContext {
     int stats_header_written;
     int stats_add_max;
     int max[4], average_max;
+    int peak[4];
     int is_rgb;
+    int bpp;
     uint8_t rgba_map[4];
     char comps[4];
     int nb_components;
@@ -55,7 +62,7 @@ typedef struct PSNRContext {
     int planewidth[4];
     int planeheight[4];
     double planeweight[4];
-    uint64_t **score;
+    Score **score;
     PSNRDSPContext dsp;
 } PSNRContext;
 
@@ -65,7 +72,7 @@ typedef struct PSNRContext {
 static const AVOption psnr_options[] = {
     {"stats_file", "Set file where to store per-frame difference information", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS },
     {"f",          "Set file where to store per-frame difference information", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS },
-    {"stats_version", "Set the format version for the stats file.",               OFFSET(stats_version),  AV_OPT_TYPE_INT,    {.i64=1},    1, 2, FLAGS },
+    {"stats_version", "Set the format version for the stats file.",               OFFSET(stats_version),  AV_OPT_TYPE_INT,    {.i64=1},    1, 3, FLAGS },
     {"output_max",  "Add raw stats (max values) to the output log.",            OFFSET(stats_add_max), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS},
     { NULL }
 };
@@ -89,17 +96,18 @@ typedef struct ThreadData {
     int ref_linesize[4];
     int planewidth[4];
     int planeheight[4];
-    uint64_t **score;
+    int bpp;
+    Score **score;
     int nb_components;
     PSNRDSPContext *dsp;
 } ThreadData;
 
 static
-int compute_images_mse(AVFilterContext *ctx, void *arg,
+int compute_images_stats(AVFilterContext *ctx, void *arg,
                        int jobnr, int nb_jobs)
 {
     ThreadData *td = arg;
-    uint64_t *score = td->score[jobnr];
+    Score *score = td->score[jobnr];
 
     for (int c = 0; c < td->nb_components; c++) {
         const int outw = td->planewidth[c];
@@ -111,12 +119,31 @@ int compute_images_mse(AVFilterContext *ctx, void *arg,
         const uint8_t *main_line = td->main_data[c] + main_linesize * slice_start;
         const uint8_t *ref_line = td->ref_data[c] + ref_linesize * slice_start;
         uint64_t m = 0;
+        uint64_t p = 0;
         for (int i = slice_start; i < slice_end; i++) {
             m += td->dsp->sse_line(main_line, ref_line, outw);
+            if (td->bpp > 8) {
+                uint16_t *mp = (uint16_t*) main_line;
+                uint16_t *rp = (uint16_t*) ref_line;
+                for (int j = 0; j < outw; j++) {
+                    int diff = abs(*mp++ - *rp++);
+                    if(diff > p)
+                        p = diff;
+                }
+            } else {
+                uint8_t *mp = main_line;
+                uint8_t *rp = ref_line;
+                for (int j = 0; j < outw; j++) {
+                    int diff = abs(*mp++ - *rp++);
+                    if(diff > p)
+                        p = diff;
+                }
+            }
             ref_line += ref_linesize;
             main_line += main_linesize;
         }
-        score[c] = m;
+        score[c].mse = m;
+        score[c].peak = p;
     }
 
     return 0;
@@ -163,6 +190,7 @@ static int do_psnr(FFFrameSync *fs)
         td.ref_linesize[c] = ref->linesize[c];
         td.planewidth[c] = s->planewidth[c];
         td.planeheight[c] = s->planeheight[c];
+        td.bpp = s->bpp;
     }
 
     if (master->color_range != ref->color_range) {
@@ -172,12 +200,15 @@ static int do_psnr(FFFrameSync *fs)
                av_color_range_name(ref->color_range));
     }
 
-    ff_filter_execute(ctx, compute_images_mse, &td, NULL,
+    ff_filter_execute(ctx, compute_images_stats, &td, NULL,
                       FFMIN(s->planeheight[1], s->nb_threads));
 
     for (int j = 0; j < s->nb_threads; j++) {
-        for (int c = 0; c < s->nb_components; c++)
-            comp_sum[c] += s->score[j][c];
+        for (int c = 0; c < s->nb_components; c++) {
+            comp_sum[c] += s->score[j][c].mse;
+            if (s->score[j][c].peak > s->peak[c])
+                s->peak[c] = s->score[j][c].peak;
+        }
     }
 
     for (int c = 0; c < s->nb_components; c++)
@@ -204,8 +235,8 @@ static int do_psnr(FFFrameSync *fs)
     set_meta(metadata, "lavfi.psnr.psnr_avg", 0, get_psnr(mse, 1, s->average_max));
 
     if (s->stats_file) {
-        if (s->stats_version == 2 && !s->stats_header_written) {
-            fprintf(s->stats_file, "psnr_log_version:2 fields:n");
+        if (s->stats_version >= 2 && !s->stats_header_written) {
+            fprintf(s->stats_file, "psnr_log_version:3 fields:n");
             fprintf(s->stats_file, ",mse_avg");
             for (int j = 0; j < s->nb_components; j++) {
                 fprintf(s->stats_file, ",mse_%c", s->comps[j]);
@@ -219,6 +250,11 @@ static int do_psnr(FFFrameSync *fs)
                 for (int j = 0; j < s->nb_components; j++) {
                     fprintf(s->stats_file, ",max_%c", s->comps[j]);
                 }
+                if (s->stats_version == 3) {
+                    for (int j = 0; j < s->nb_components; j++) {
+                        fprintf(s->stats_file, ",peak_%c", s->comps[j]);
+                    }
+                }
             }
             fprintf(s->stats_file, "\n");
             s->stats_header_written = 1;
@@ -234,12 +270,18 @@ static int do_psnr(FFFrameSync *fs)
             fprintf(s->stats_file, "psnr_%c:%0.2f ", s->comps[j],
                     get_psnr(comp_mse[c], 1, s->max[c]));
         }
-        if (s->stats_version == 2 && s->stats_add_max) {
+        if (s->stats_version >= 2 && s->stats_add_max) {
             fprintf(s->stats_file, "max_avg:%d ", s->average_max);
             for (int j = 0; j < s->nb_components; j++) {
                 int c = s->is_rgb ? s->rgba_map[j] : j;
                 fprintf(s->stats_file, "max_%c:%d ", s->comps[j], s->max[c]);
             }
+            if (s->stats_version == 3) {
+                for (int j = 0; j < s->nb_components; j++) {
+                    int c = s->is_rgb ? s->rgba_map[j] : j;
+                    fprintf(s->stats_file, "peak_%c:%d ", s->comps[j], s->peak[c]);
+                }
+            }
         }
         fprintf(s->stats_file, "\n");
     }
@@ -314,6 +356,9 @@ static int config_input_ref(AVFilterLink *inlink)
     s->max[2] = (1 << desc->comp[2].depth) - 1;
     s->max[3] = (1 << desc->comp[3].depth) - 1;
 
+    for (j = 0; j < s->nb_components; j++)
+        s->peak[j] = 0;
+
     s->is_rgb = ff_fill_rgba_map(s->rgba_map, inlink->format) >= 0;
     s->comps[0] = s->is_rgb ? 'r' : 'y' ;
     s->comps[1] = s->is_rgb ? 'g' : 'u' ;
@@ -334,6 +379,7 @@ static int config_input_ref(AVFilterLink *inlink)
     }
     s->average_max = lrint(average_max);
 
+    s->bpp = desc->comp[0].depth;
     ff_psnr_init(&s->dsp, desc->comp[0].depth);
 
     s->score = av_calloc(s->nb_threads, sizeof(*s->score));
diff --git a/tests/fate-run.sh b/tests/fate-run.sh
index 45dcb6e8dc..27c2e68ee0 100755
--- a/tests/fate-run.sh
+++ b/tests/fate-run.sh
@@ -72,6 +72,38 @@ do_tiny_psnr(){
     fi
 }
 
+# $1 is the reference image
+# $2 is the test image
+# $3 is the maximum MSE allowed across components
+# $4 is the maximum peak error allowed across components
+mse_peak_error(){
+    stats=$(run ffmpeg${PROGSUF}${EXECSUF} -i "$1" -i "$2" -filter_complex "psnr=stats_version=3:output_max=1:f=-" -f null - -hide_banner -loglevel error)
+    for item in y u v r g b a; do
+        mse=$(echo $stats | sed -nE "s/.*mse_$item:([0-9.]+).*/\1/p")
+        if [ -z "$mse" ]; then
+            continue
+        fi
+        mse_exceeded=$(echo "$mse > $3" | bc -l)
+        if [ "$mse_exceeded" != 0 ]; then
+            echo "MSE value $mse exceeded the specified maximum $3"
+            echo $stats
+            return 1
+        fi
+        peak=$(echo $stats | sed -nE "s/.*peak_$item:([0-9.]+).*/\1/p")
+        if [ -z "$peak" ]; then
+            echo "peak_$item value missing when mse_$item was present"
+        fi
+        peak_exceeded=$(echo "$peak > $4" | bc -l)
+        if [ "$peak_exceeded" != 0 ]; then
+            echo "Peak value $peak exceeded the specified maximum $4"
+            echo $stats
+            return 1
+        fi
+    done
+    echo "Passed"
+    return 0
+}
+
 oneoff(){
     do_tiny_psnr "$1" "$2" MAXDIFF
 }
-- 
2.34.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".