[FFmpeg-devel] [PATCH 1/2] postproc/postprocess_template: Fix reading uninitialized pixels in dering_C()

[FFmpeg-devel] [PATCH 1/2] postproc/postprocess_template: Fix reading uninitialized pixels in dering_C()

[Michael Niedermayer]

This issue was found through the new blocktest

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libpostproc/postprocess_altivec_template.c |  2 +-
 libpostproc/postprocess_template.c         | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/libpostproc/postprocess_altivec_template.c b/libpostproc/postprocess_altivec_template.c
index 827d6300e5f..1fc7c65ee47 100644
--- a/libpostproc/postprocess_altivec_template.c
+++ b/libpostproc/postprocess_altivec_template.c
@@ -530,7 +530,7 @@ static inline void doVertDefFilter_altivec(uint8_t src[], int stride, PPContext
      STORE(5)
 }
 
-static inline void dering_altivec(uint8_t src[], int stride, PPContext *c) {
+static inline void dering_altivec(uint8_t src[], int stride, PPContext *c, int leftborder, int rightborder) {
     const vector signed int vsint32_8 = vec_splat_s32(8);
     const vector unsigned int vuint32_4 = vec_splat_u32(4);
     const vector signed char neg1 = vec_splat_s8(-1);
diff --git a/libpostproc/postprocess_template.c b/libpostproc/postprocess_template.c
index f0e3c50d88c..d2e7a642ffb 100644
--- a/libpostproc/postprocess_template.c
+++ b/libpostproc/postprocess_template.c
@@ -831,7 +831,7 @@ static inline void RENAME(doVertDefFilter)(uint8_t src[], int stride, PPContext
 #endif //TEMPLATE_PP_ALTIVEC
 
 #if !TEMPLATE_PP_ALTIVEC
-static inline void RENAME(dering)(uint8_t src[], int stride, PPContext *c)
+static inline void RENAME(dering)(uint8_t src[], int stride, PPContext *c, int leftborder, int rightborder)
 {
 #if TEMPLATE_PP_MMXEXT && HAVE_7REGS
     DECLARE_ALIGNED(8, uint64_t, tmp)[3];
@@ -1046,7 +1046,7 @@ DERING_CORE((%0, %1, 8)       ,(%%FF_REGd, %1, 4),%%mm2,%%mm4,%%mm0,%%mm3,%%mm5,
     for(y=0; y<10; y++){
         int t = 0;
 
-        if(src[stride*y + 0] > avg) t+= 1;
+        if(!leftborder && src[stride*y + 0] > avg) t+= 1;
         if(src[stride*y + 1] > avg) t+= 2;
         if(src[stride*y + 2] > avg) t+= 4;
         if(src[stride*y + 3] > avg) t+= 8;
@@ -1055,7 +1055,7 @@ DERING_CORE((%0, %1, 8)       ,(%%FF_REGd, %1, 4),%%mm2,%%mm4,%%mm0,%%mm3,%%mm5,
         if(src[stride*y + 6] > avg) t+= 64;
         if(src[stride*y + 7] > avg) t+= 128;
         if(src[stride*y + 8] > avg) t+= 256;
-        if(src[stride*y + 9] > avg) t+= 512;
+        if(!rightborder && src[stride*y + 9] > avg) t+= 512;
 
         t |= (~t)<<16;
         t &= (t<<1) & (t>>1);
@@ -1072,8 +1072,8 @@ DERING_CORE((%0, %1, 8)       ,(%%FF_REGd, %1, 4),%%mm2,%%mm4,%%mm0,%%mm3,%%mm5,
         int x;
         int t = s[y-1];
 
-        p= src + stride*y;
-        for(x=1; x<9; x++){
+        p= src + stride*y + leftborder;
+        for(x=1+leftborder; x<9-rightborder; x++){
             p++;
             if(t & (1<<x)){
                 int f= (*(p-stride-1)) + 2*(*(p-stride)) + (*(p-stride+1))
@@ -3210,7 +3210,7 @@ static void RENAME(postProcess)(const uint8_t src[], int srcStride, uint8_t dst[
 #endif //TEMPLATE_PP_MMX
                 if(mode & DERING){
                 //FIXME filter first line
-                    if(y>0) RENAME(dering)(dstBlock - stride - 8, stride, c);
+                    if(y>0) RENAME(dering)(dstBlock - stride - 8, stride, c, x<=8, 0);
                 }
 
                 if(mode & TEMP_NOISE_FILTER)
@@ -3232,7 +3232,7 @@ static void RENAME(postProcess)(const uint8_t src[], int srcStride, uint8_t dst[
         }
 
         if(mode & DERING){
-            if(y > 0) RENAME(dering)(dstBlock - dstStride - 8, dstStride, c);
+            if(y > 0) RENAME(dering)(dstBlock - dstStride - 8, dstStride, c, 0, 1);
         }
 
         if((mode & TEMP_NOISE_FILTER)){
-- 
2.49.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 2/2] postproc/postprocess_template: fix handling of first row of dering_C

[Michael Niedermayer]

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libpostproc/postprocess_altivec_template.c |  5 ++++-
 libpostproc/postprocess_template.c         | 12 +++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/libpostproc/postprocess_altivec_template.c b/libpostproc/postprocess_altivec_template.c
index 1fc7c65ee47..feddab50356 100644
--- a/libpostproc/postprocess_altivec_template.c
+++ b/libpostproc/postprocess_altivec_template.c
@@ -530,7 +530,7 @@ static inline void doVertDefFilter_altivec(uint8_t src[], int stride, PPContext
      STORE(5)
 }
 
-static inline void dering_altivec(uint8_t src[], int stride, PPContext *c, int leftborder, int rightborder) {
+static inline void dering_altivec(uint8_t src[], int stride, PPContext *c, int leftborder, int rightborder, int topborder) {
     const vector signed int vsint32_8 = vec_splat_s32(8);
     const vector unsigned int vuint32_4 = vec_splat_u32(4);
     const vector signed char neg1 = vec_splat_s8(-1);
@@ -577,6 +577,9 @@ static inline void dering_altivec(uint8_t src[], int stride, PPContext *c, int l
     const vector signed int zero = vec_splat_s32(0);
     vector unsigned char v_dt = vec_splat(vec_ld(0, dt), 0);
 
+    if (topborder)
+        return;
+
 #define LOAD_LINE(i)                                                  \
     const vector unsigned char perm##i =                              \
         vec_lvsl(i * stride, srcCopy);                                \
diff --git a/libpostproc/postprocess_template.c b/libpostproc/postprocess_template.c
index d2e7a642ffb..0531e39477a 100644
--- a/libpostproc/postprocess_template.c
+++ b/libpostproc/postprocess_template.c
@@ -831,9 +831,11 @@ static inline void RENAME(doVertDefFilter)(uint8_t src[], int stride, PPContext
 #endif //TEMPLATE_PP_ALTIVEC
 
 #if !TEMPLATE_PP_ALTIVEC
-static inline void RENAME(dering)(uint8_t src[], int stride, PPContext *c, int leftborder, int rightborder)
+static inline void RENAME(dering)(uint8_t src[], int stride, PPContext *c, int leftborder, int rightborder, int topborder)
 {
 #if TEMPLATE_PP_MMXEXT && HAVE_7REGS
+    if (topborder)
+        return;
     DECLARE_ALIGNED(8, uint64_t, tmp)[3];
     __asm__ volatile(
         "pxor %%mm6, %%mm6                      \n\t"
@@ -1043,7 +1045,8 @@ DERING_CORE((%0, %1, 8)       ,(%%FF_REGd, %1, 4),%%mm2,%%mm4,%%mm0,%%mm3,%%mm5,
 
     if (max - min < DERING_THRESHOLD) return;
 
-    for(y=0; y<10; y++){
+    s[0] = 0;
+    for(y=topborder; y<10; y++){
         int t = 0;
 
         if(!leftborder && src[stride*y + 0] > avg) t+= 1;
@@ -3209,8 +3212,7 @@ static void RENAME(postProcess)(const uint8_t src[], int srcStride, uint8_t dst[
                 }
 #endif //TEMPLATE_PP_MMX
                 if(mode & DERING){
-                //FIXME filter first line
-                    if(y>0) RENAME(dering)(dstBlock - stride - 8, stride, c, x<=8, 0);
+                    RENAME(dering)(dstBlock - stride - 8, stride, c, x<=8, 0, y<=0);
                 }
 
                 if(mode & TEMP_NOISE_FILTER)
@@ -3232,7 +3234,7 @@ static void RENAME(postProcess)(const uint8_t src[], int srcStride, uint8_t dst[
         }
 
         if(mode & DERING){
-            if(y > 0) RENAME(dering)(dstBlock - dstStride - 8, dstStride, c, 0, 1);
+            RENAME(dering)(dstBlock - dstStride - 8, dstStride, c, 0, 1, y<=0);
         }
 
         if((mode & TEMP_NOISE_FILTER)){
-- 
2.49.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] postproc/postprocess_template: Fix reading uninitialized pixels in dering_C()

[Michael Niedermayer]

[-- Attachment #1.1: Type: text/plain, Size: 556 bytes --]

On Tue, Apr 22, 2025 at 09:28:50PM +0200, Michael Niedermayer wrote:
> This issue was found through the new blocktest
> 
> Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> ---
>  libpostproc/postprocess_altivec_template.c |  2 +-
>  libpostproc/postprocess_template.c         | 14 +++++++-------
>  2 files changed, 8 insertions(+), 8 deletions(-)

will apply patchset

[...]
-- 
Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB

Observe your enemies, for they first find out your faults. -- Antisthenes

[-- Attachment #1.2: signature.asc --]
[-- Type: application/pgp-signature, Size: 195 bytes --]

[-- Attachment #2: Type: text/plain, Size: 251 bytes --]

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".