[FFmpeg-devel] [PATCH 1/2] avcodec/vc2enc_dwt: Avoid NULL - 0

[FFmpeg-devel] [PATCH 1/2] avcodec/vc2enc_dwt: Avoid NULL - 0

[Andreas Rheinhardt]

It is sane, but UB. It could happen in case of allocation errors
in vc2_encode_init().

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
 libavcodec/vc2enc_dwt.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libavcodec/vc2enc_dwt.c b/libavcodec/vc2enc_dwt.c
index a8d3f1c669..441af040ec 100644
--- a/libavcodec/vc2enc_dwt.c
+++ b/libavcodec/vc2enc_dwt.c
@@ -276,6 +276,8 @@ av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_stride,
 
 av_cold void ff_vc2enc_free_transforms(VC2TransformContext *s)
 {
-    av_free(s->buffer - s->padding);
-    s->buffer = NULL;
+    if (s->buffer) {
+        av_free(s->buffer - s->padding);
+        s->buffer = NULL;
+    }
 }
-- 
2.32.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 2/2] avcodec/vc2enc: Cleanup generically on allocation failure

[Andreas Rheinhardt]

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
 libavcodec/vc2enc.c | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/libavcodec/vc2enc.c b/libavcodec/vc2enc.c
index f0d2cdf62d..ccca78d281 100644
--- a/libavcodec/vc2enc.c
+++ b/libavcodec/vc2enc.c
@@ -1135,7 +1135,7 @@ static av_cold int vc2_encode_init(AVCodecContext *avctx)
         p->coef_stride = FFALIGN(p->dwt_width, 32);
         p->coef_buf = av_mallocz(p->coef_stride*p->dwt_height*sizeof(dwtcoef));
         if (!p->coef_buf)
-            goto alloc_fail;
+            return AVERROR(ENOMEM);
         for (level = s->wavelet_depth-1; level >= 0; level--) {
             w = w >> 1;
             h = h >> 1;
@@ -1154,7 +1154,7 @@ static av_cold int vc2_encode_init(AVCodecContext *avctx)
                                       s->plane[i].coef_stride,
                                       s->plane[i].dwt_height,
                                       s->slice_width, s->slice_height))
-            goto alloc_fail;
+            return AVERROR(ENOMEM);
     }
 
     /* Slices */
@@ -1163,7 +1163,7 @@ static av_cold int vc2_encode_init(AVCodecContext *avctx)
 
     s->slice_args = av_calloc(s->num_x*s->num_y, sizeof(SliceArgs));
     if (!s->slice_args)
-        goto alloc_fail;
+        return AVERROR(ENOMEM);
 
     for (i = 0; i < 116; i++) {
         const uint64_t qf = ff_dirac_qscale_tab[i];
@@ -1183,11 +1183,6 @@ static av_cold int vc2_encode_init(AVCodecContext *avctx)
     }
 
     return 0;
-
-alloc_fail:
-    vc2_encode_end(avctx);
-    av_log(avctx, AV_LOG_ERROR, "Unable to allocate memory!\n");
-    return AVERROR(ENOMEM);
 }
 
 #define VC2ENC_FLAGS (AV_OPT_FLAG_ENCODING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
@@ -1234,10 +1229,10 @@ const AVCodec ff_vc2_encoder = {
     .type           = AVMEDIA_TYPE_VIDEO,
     .id             = AV_CODEC_ID_DIRAC,
     .capabilities   = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_SLICE_THREADS,
+    .caps_internal  = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
     .priv_data_size = sizeof(VC2EncContext),
     .init           = vc2_encode_init,
     .close          = vc2_encode_end,
-    .caps_internal  = FF_CODEC_CAP_INIT_THREADSAFE,
     .encode2        = vc2_encode_frame,
     .priv_class     = &vc2enc_class,
     .defaults       = vc2enc_defaults,
-- 
2.32.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] avcodec/vc2enc_dwt: Avoid NULL - 0

[Andreas Rheinhardt]

Andreas Rheinhardt:
> It is sane, but UB. It could happen in case of allocation errors
> in vc2_encode_init().
> 
> Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
> ---
>  libavcodec/vc2enc_dwt.c | 6 ++++--
>  1 file changed, 4 insertions(+), 2 deletions(-)
> 
> diff --git a/libavcodec/vc2enc_dwt.c b/libavcodec/vc2enc_dwt.c
> index a8d3f1c669..441af040ec 100644
> --- a/libavcodec/vc2enc_dwt.c
> +++ b/libavcodec/vc2enc_dwt.c
> @@ -276,6 +276,8 @@ av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_stride,
>  
>  av_cold void ff_vc2enc_free_transforms(VC2TransformContext *s)
>  {
> -    av_free(s->buffer - s->padding);
> -    s->buffer = NULL;
> +    if (s->buffer) {
> +        av_free(s->buffer - s->padding);
> +        s->buffer = NULL;
> +    }
>  }

Will apply these patches later tonight unless there are objections.

- Andreas
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".