[FFmpeg-devel] [PATCH v9 02/25] avutil/frame: Prepare AVFrame for subtitle handling

[softworkz <ffmpegagent@gmail.com>]

From: softworkz <softworkz@hotmail.com>

Root commit for adding subtitle filtering capabilities.
In detail:

- Add type (AVMediaType) field to AVFrame
  Replaces previous way of distinction which was based on checking
  width and height to determine whether a frame is audio or video
- Add subtitle fields to AVFrame
- Add new struct AVSubtitleArea, similar to AVSubtitleRect, but
  different allocation logic. Cannot and must not be used
  interchangeably, hence the new struct

Signed-off-by: softworkz <softworkz@hotmail.com>
---
 libavutil/Makefile |   1 +
 libavutil/frame.c  | 206 +++++++++++++++++++++++++++++++++++++++++----
 libavutil/frame.h  |  85 ++++++++++++++++++-
 libavutil/subfmt.c |  45 ++++++++++
 libavutil/subfmt.h |  47 +++++++++++
 5 files changed, 363 insertions(+), 21 deletions(-)
 create mode 100644 libavutil/subfmt.c

diff --git a/libavutil/Makefile b/libavutil/Makefile
index 74380cf917..4ab4fcb01b 100644
--- a/libavutil/Makefile
+++ b/libavutil/Makefile
@@ -165,6 +165,7 @@ OBJS = adler32.o                                                        \
        slicethread.o                                                    \
        spherical.o                                                      \
        stereo3d.o                                                       \
+       subfmt.o                                                         \
        threadmessage.o                                                  \
        time.o                                                           \
        timecode.o                                                       \
diff --git a/libavutil/frame.c b/libavutil/frame.c
index de4ad1f94d..230673e4e1 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -26,6 +26,7 @@
 #include "imgutils.h"
 #include "mem.h"
 #include "samplefmt.h"
+#include "subfmt.h"
 #include "hwcontext.h"
 
 #if FF_API_OLD_CHANNEL_LAYOUT
@@ -52,6 +53,9 @@ const char *av_get_colorspace_name(enum AVColorSpace val)
     return name[val];
 }
 #endif
+
+static int frame_copy_subtitles(AVFrame *dst, const AVFrame *src, int copy_data);
+
 static void get_frame_defaults(AVFrame *frame)
 {
     memset(frame, 0, sizeof(*frame));
@@ -77,7 +81,12 @@ FF_ENABLE_DEPRECATION_WARNINGS
     frame->colorspace          = AVCOL_SPC_UNSPECIFIED;
     frame->color_range         = AVCOL_RANGE_UNSPECIFIED;
     frame->chroma_location     = AVCHROMA_LOC_UNSPECIFIED;
-    frame->flags               = 0;
+    frame->num_subtitle_areas  = 0;
+    frame->subtitle_areas      = NULL;
+    frame->subtitle_header     = NULL;
+    frame->repeat_sub          = 0;
+    frame->subtitle_timing.start_pts = 0;
+    frame->subtitle_timing.duration  = 0;
 }
 
 static void free_side_data(AVFrameSideData **ptr_sd)
@@ -256,6 +265,23 @@ FF_ENABLE_DEPRECATION_WARNINGS
 
 }
 
+static int get_subtitle_buffer(AVFrame *frame)
+{
+    // Buffers in AVFrame->buf[] are not used in case of subtitle frames.
+    // To accomodate with existing code, checking ->buf[0] to determine
+    // whether a frame is ref-counted or has data, we're adding a 1-byte
+    // buffer here, which marks the subtitle frame to contain data.
+    frame->buf[0] = av_buffer_alloc(1);
+    if (!frame->buf[0]) {
+        av_frame_unref(frame);
+        return AVERROR(ENOMEM);
+    }
+
+    frame->extended_data = frame->data;
+
+    return 0;
+}
+
 int av_frame_get_buffer(AVFrame *frame, int align)
 {
     if (frame->format < 0)
@@ -263,23 +289,41 @@ int av_frame_get_buffer(AVFrame *frame, int align)
 
 FF_DISABLE_DEPRECATION_WARNINGS
     if (frame->width > 0 && frame->height > 0)
-        return get_video_buffer(frame, align);
+        frame->type = AVMEDIA_TYPE_VIDEO;
     else if (frame->nb_samples > 0 &&
              (av_channel_layout_check(&frame->ch_layout)
 #if FF_API_OLD_CHANNEL_LAYOUT
               || frame->channel_layout || frame->channels > 0
 #endif
              ))
-        return get_audio_buffer(frame, align);
+        frame->type = AVMEDIA_TYPE_AUDIO;
 FF_ENABLE_DEPRECATION_WARNINGS
 
-    return AVERROR(EINVAL);
+    return av_frame_get_buffer2(frame, align);
+}
+
+int av_frame_get_buffer2(AVFrame *frame, int align)
+{
+    if (frame->format < 0)
+        return AVERROR(EINVAL);
+
+    switch (frame->type) {
+    case AVMEDIA_TYPE_VIDEO:
+        return get_video_buffer(frame, align);
+    case AVMEDIA_TYPE_AUDIO:
+        return get_audio_buffer(frame, align);
+    case AVMEDIA_TYPE_SUBTITLE:
+        return get_subtitle_buffer(frame);
+    default:
+        return AVERROR(EINVAL);
+    }
 }
 
 static int frame_copy_props(AVFrame *dst, const AVFrame *src, int force_copy)
 {
     int ret, i;
 
+    dst->type                   = src->type;
     dst->key_frame              = src->key_frame;
     dst->pict_type              = src->pict_type;
     dst->sample_aspect_ratio    = src->sample_aspect_ratio;
@@ -316,6 +360,12 @@ FF_ENABLE_DEPRECATION_WARNINGS
     dst->colorspace             = src->colorspace;
     dst->color_range            = src->color_range;
     dst->chroma_location        = src->chroma_location;
+    dst->repeat_sub             = src->repeat_sub;
+    dst->subtitle_timing.start_pts = src->subtitle_timing.start_pts;
+    dst->subtitle_timing.duration  = src->subtitle_timing.duration;
+
+    if ((ret = av_buffer_replace(&dst->subtitle_header, src->subtitle_header)) < 0)
+        return ret;
 
     av_dict_copy(&dst->metadata, src->metadata, 0);
 
@@ -363,6 +413,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
     av_assert1(dst->ch_layout.nb_channels == 0 &&
                dst->ch_layout.order == AV_CHANNEL_ORDER_UNSPEC);
 
+    dst->type           = src->type;
     dst->format         = src->format;
     dst->width          = src->width;
     dst->height         = src->height;
@@ -395,7 +446,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
 
     /* duplicate the frame data if it's not refcounted */
     if (!src->buf[0]) {
-        ret = av_frame_get_buffer(dst, 0);
+        ret = av_frame_get_buffer2(dst, 0);
         if (ret < 0)
             goto fail;
 
@@ -417,6 +468,10 @@ FF_ENABLE_DEPRECATION_WARNINGS
         }
     }
 
+    /* copy subtitle areas */
+    if (src->type == AVMEDIA_TYPE_SUBTITLE)
+        frame_copy_subtitles(dst, src, 0);
+
     if (src->extended_buf) {
         dst->extended_buf = av_calloc(src->nb_extended_buf,
                                       sizeof(*dst->extended_buf));
@@ -486,7 +541,7 @@ AVFrame *av_frame_clone(const AVFrame *src)
 
 void av_frame_unref(AVFrame *frame)
 {
-    int i;
+    unsigned i, n;
 
     if (!frame)
         return;
@@ -505,6 +560,21 @@ void av_frame_unref(AVFrame *frame)
     av_buffer_unref(&frame->opaque_ref);
     av_buffer_unref(&frame->private_ref);
 
+    av_buffer_unref(&frame->subtitle_header);
+
+    for (i = 0; i < frame->num_subtitle_areas; i++) {
+        AVSubtitleArea *area = frame->subtitle_areas[i];
+
+        for (n = 0; n < FF_ARRAY_ELEMS(area->buf); n++)
+            av_buffer_unref(&area->buf[n]);
+
+        av_freep(&area->text);
+        av_freep(&area->ass);
+        av_free(area);
+    }
+
+    av_freep(&frame->subtitle_areas);
+
     if (frame->extended_data != frame->data)
         av_freep(&frame->extended_data);
 
@@ -532,18 +602,28 @@ FF_ENABLE_DEPRECATION_WARNINGS
 
 int av_frame_is_writable(AVFrame *frame)
 {
-    int i, ret = 1;
+    AVSubtitleArea *area;
+    int ret = 1;
 
     /* assume non-refcounted frames are not writable */
     if (!frame->buf[0])
         return 0;
 
-    for (i = 0; i < FF_ARRAY_ELEMS(frame->buf); i++)
+    for (unsigned i = 0; i < FF_ARRAY_ELEMS(frame->buf); i++)
         if (frame->buf[i])
             ret &= !!av_buffer_is_writable(frame->buf[i]);
-    for (i = 0; i < frame->nb_extended_buf; i++)
+    for (unsigned i = 0; i < frame->nb_extended_buf; i++)
         ret &= !!av_buffer_is_writable(frame->extended_buf[i]);
 
+    for (unsigned i = 0; i < frame->num_subtitle_areas; i++) {
+        area = frame->subtitle_areas[i];
+        if (area) {
+            for (unsigned n = 0; n < FF_ARRAY_ELEMS(area->buf); n++)
+                if (area->buf[n])
+                    ret &= !!av_buffer_is_writable(area->buf[n]);
+            }
+    }
+
     return ret;
 }
 
@@ -556,6 +636,7 @@ int av_frame_make_writable(AVFrame *frame)
         return 0;
 
     memset(&tmp, 0, sizeof(tmp));
+    tmp.type           = frame->type;
     tmp.format         = frame->format;
     tmp.width          = frame->width;
     tmp.height         = frame->height;
@@ -575,7 +656,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
     if (frame->hw_frames_ctx)
         ret = av_hwframe_get_buffer(frame->hw_frames_ctx, &tmp, 0);
     else
-        ret = av_frame_get_buffer(&tmp, 0);
+        ret = av_frame_get_buffer2(&tmp, 0);
     if (ret < 0)
         return ret;
 
@@ -610,7 +691,12 @@ AVBufferRef *av_frame_get_plane_buffer(AVFrame *frame, int plane)
     uint8_t *data;
     int planes, i;
 
-    if (frame->nb_samples) {
+    switch(frame->type) {
+    case AVMEDIA_TYPE_VIDEO:
+        planes = 4;
+        break;
+    case AVMEDIA_TYPE_AUDIO:
+        {
         int channels = frame->ch_layout.nb_channels;
 
 #if FF_API_OLD_CHANNEL_LAYOUT
@@ -624,8 +710,11 @@ FF_ENABLE_DEPRECATION_WARNINGS
         if (!channels)
             return NULL;
         planes = av_sample_fmt_is_planar(frame->format) ? channels : 1;
-    } else
-        planes = 4;
+        break;
+        }
+    default:
+        return NULL;
+    }
 
     if (plane < 0 || plane >= planes || !frame->extended_data[plane])
         return NULL;
@@ -768,22 +857,103 @@ FF_ENABLE_DEPRECATION_WARNINGS
     return 0;
 }
 
+static int av_subtitle_area2area(AVSubtitleArea *dst, const AVSubtitleArea *src, int copy_data)
+{
+    dst->x         =  src->x;
+    dst->y         =  src->y;
+    dst->w         =  src->w;
+    dst->h         =  src->h;
+    dst->nb_colors =  src->nb_colors;
+    dst->type      =  src->type;
+    dst->flags     =  src->flags;
+
+    switch (dst->type) {
+    case AV_SUBTITLE_FMT_BITMAP:
+
+        for (unsigned i = 0; i < AV_NUM_BUFFER_POINTERS; i++) {
+            if (src->h > 0 && src->w > 0 && src->buf[i]) {
+                dst->buf[0] = av_buffer_ref(src->buf[i]);
+                if (!dst->buf[i])
+                    return AVERROR(ENOMEM);
+
+                if (copy_data) {
+                    const int ret = av_buffer_make_writable(&dst->buf[i]);
+                    if (ret < 0)
+                        return ret;
+                }
+
+                dst->linesize[i] = src->linesize[i];
+            }
+        }
+
+        memcpy(&dst->pal[0], &src->pal[0], sizeof(src->pal[0]) * 256);
+
+        break;
+    case AV_SUBTITLE_FMT_TEXT:
+
+        if (src->text) {
+            dst->text = av_strdup(src->text);
+            if (!dst->text)
+                return AVERROR(ENOMEM);
+        }
+
+        break;
+    case AV_SUBTITLE_FMT_ASS:
+
+        if (src->ass) {
+            dst->ass = av_strdup(src->ass);
+            if (!dst->ass)
+                return AVERROR(ENOMEM);
+        }
+
+        break;
+    default:
+
+        av_log(NULL, AV_LOG_ERROR, "Subtitle area has invalid format: %d", dst->type);
+        return AVERROR(ENOMEM);
+    }
+
+    return 0;
+}
+
+static int frame_copy_subtitles(AVFrame *dst, const AVFrame *src, int copy_data)
+{
+    dst->format = src->format;
+
+    dst->num_subtitle_areas = src->num_subtitle_areas;
+
+    if (src->num_subtitle_areas) {
+        dst->subtitle_areas = av_malloc_array(src->num_subtitle_areas, sizeof(AVSubtitleArea *));
+
+        for (unsigned i = 0; i < src->num_subtitle_areas; i++) {
+            dst->subtitle_areas[i] = av_mallocz(sizeof(AVSubtitleArea));
+            av_subtitle_area2area(dst->subtitle_areas[i], src->subtitle_areas[i], copy_data);
+        }
+    }
+
+    return 0;
+}
+
 int av_frame_copy(AVFrame *dst, const AVFrame *src)
 {
     if (dst->format != src->format || dst->format < 0)
         return AVERROR(EINVAL);
 
-FF_DISABLE_DEPRECATION_WARNINGS
-    if (dst->width > 0 && dst->height > 0)
+    switch(dst->type) {
+    case AVMEDIA_TYPE_VIDEO:
         return frame_copy_video(dst, src);
-    else if (dst->nb_samples > 0 &&
+    case AVMEDIA_TYPE_AUDIO:
+        if (dst->nb_samples > 0 &&
              (av_channel_layout_check(&dst->ch_layout)
 #if FF_API_OLD_CHANNEL_LAYOUT
               || dst->channels > 0
 #endif
             ))
-        return frame_copy_audio(dst, src);
-FF_ENABLE_DEPRECATION_WARNINGS
+            return frame_copy_audio(dst, src);
+        break;
+    case AVMEDIA_TYPE_SUBTITLE:
+        return frame_copy_subtitles(dst, src, 1);
+    }
 
     return AVERROR(EINVAL);
 }
diff --git a/libavutil/frame.h b/libavutil/frame.h
index e60a82f6c0..9ce267bbc5 100644
--- a/libavutil/frame.h
+++ b/libavutil/frame.h
@@ -25,7 +25,6 @@
 #ifndef AVUTIL_FRAME_H
 #define AVUTIL_FRAME_H
 
-#include <stddef.h>
 #include <stdint.h>
 
 #include "avutil.h"
@@ -35,6 +34,7 @@
 #include "rational.h"
 #include "samplefmt.h"
 #include "pixfmt.h"
+#include "subfmt.h"
 #include "version.h"
 
 
@@ -293,7 +293,7 @@ typedef struct AVRegionOfInterest {
 } AVRegionOfInterest;
 
 /**
- * This structure describes decoded (raw) audio or video data.
+ * This structure describes decoded (raw) audio, video or subtitle data.
  *
  * AVFrame must be allocated using av_frame_alloc(). Note that this only
  * allocates the AVFrame itself, the buffers for the data must be managed
@@ -407,7 +407,7 @@ typedef struct AVFrame {
     /**
      * format of the frame, -1 if unknown or unset
      * Values correspond to enum AVPixelFormat for video frames,
-     * enum AVSampleFormat for audio)
+     * enum AVSampleFormat for audio, AVSubtitleType for subtitles)
      */
     int format;
 
@@ -712,6 +712,53 @@ typedef struct AVFrame {
      * Duration of the frame, in the same units as pts. 0 if unknown.
      */
     int64_t duration;
+
+    /**
+     * Media type of the frame (audio, video, subtitles..)
+     *
+     * See AVMEDIA_TYPE_xxx
+     */
+    enum AVMediaType type;
+
+    /**
+     * Number of items in the @ref subtitle_areas array.
+     */
+    unsigned num_subtitle_areas;
+
+    /**
+     * Array of subtitle areas, may be empty.
+     */
+    AVSubtitleArea **subtitle_areas;
+
+    /**
+     * Header containing style information for text subtitles.
+     */
+    AVBufferRef *subtitle_header;
+
+    /**
+     * Indicates that a subtitle frame is a repeated frame for arbitrating flow
+     * in a filter graph.
+     * The field subtitle_timing.start_pts always indicates the original presentation
+     * time, while the frame's pts field may be different.
+     */
+    int repeat_sub;
+
+    struct SubtitleTiming
+    {
+        /**
+         * The display start time, in AV_TIME_BASE.
+         *
+         * For subtitle frames, AVFrame.pts is populated from the packet pts value,
+         * which is not always the same as this value.
+         */
+        int64_t start_pts;
+
+        /**
+         * Display duration, in AV_TIME_BASE.
+         */
+        int64_t duration;
+
+    } subtitle_timing;
 } AVFrame;
 
 
@@ -788,6 +835,8 @@ void av_frame_move_ref(AVFrame *dst, AVFrame *src);
 /**
  * Allocate new buffer(s) for audio or video data.
  *
+ * Note: For subtitle data, use av_frame_get_buffer2
+ *
  * The following fields must be set on frame before calling this function:
  * - format (pixel format for video, sample format for audio)
  * - width and height for video
@@ -807,9 +856,39 @@ void av_frame_move_ref(AVFrame *dst, AVFrame *src);
  *              recommended to pass 0 here unless you know what you are doing.
  *
  * @return 0 on success, a negative AVERROR on error.
+ *
+ * @deprecated Use @ref av_frame_get_buffer2 instead and set @ref AVFrame.type
+ * before calling.
  */
+attribute_deprecated
 int av_frame_get_buffer(AVFrame *frame, int align);
 
+/**
+ * Allocate new buffer(s) for audio, video or subtitle data.
+ *
+ * The following fields must be set on frame before calling this function:
+ * - format (pixel format for video, sample format for audio)
+ * - width and height for video
+ * - nb_samples and channel_layout for audio
+ * - type (AVMediaType)
+ *
+ * This function will fill AVFrame.data and AVFrame.buf arrays and, if
+ * necessary, allocate and fill AVFrame.extended_data and AVFrame.extended_buf.
+ * For planar formats, one buffer will be allocated for each plane.
+ *
+ * @warning: if frame already has been allocated, calling this function will
+ *           leak memory. In addition, undefined behavior can occur in certain
+ *           cases.
+ *
+ * @param frame frame in which to store the new buffers.
+ * @param align Required buffer size alignment. If equal to 0, alignment will be
+ *              chosen automatically for the current CPU. It is highly
+ *              recommended to pass 0 here unless you know what you are doing.
+ *
+ * @return 0 on success, a negative AVERROR on error.
+ */
+int av_frame_get_buffer2(AVFrame *frame, int align);
+
 /**
  * Check if the frame data is writable.
  *
diff --git a/libavutil/subfmt.c b/libavutil/subfmt.c
new file mode 100644
index 0000000000..c72ebe2a43
--- /dev/null
+++ b/libavutil/subfmt.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2021 softworkz
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <string.h>
+#include "common.h"
+#include "subfmt.h"
+
+static const char sub_fmt_info[AV_SUBTITLE_FMT_NB][24] = {
+    [AV_SUBTITLE_FMT_UNKNOWN] = "Unknown subtitle format",
+    [AV_SUBTITLE_FMT_BITMAP]  = "Graphical subtitles",
+    [AV_SUBTITLE_FMT_TEXT]    = "Text subtitles (plain)",
+    [AV_SUBTITLE_FMT_ASS]     = "Text subtitles (ass)",
+};
+
+const char *av_get_subtitle_fmt_name(enum AVSubtitleType sub_fmt)
+{
+    if (sub_fmt < 0 || sub_fmt >= AV_SUBTITLE_FMT_NB)
+        return NULL;
+    return sub_fmt_info[sub_fmt];
+}
+
+enum AVSubtitleType av_get_subtitle_fmt(const char *name)
+{
+    for (int i = 0; i < AV_SUBTITLE_FMT_NB; i++)
+        if (!strcmp(sub_fmt_info[i], name))
+            return i;
+    return AV_SUBTITLE_FMT_NONE;
+}
diff --git a/libavutil/subfmt.h b/libavutil/subfmt.h
index 791b45519f..3b8a0613b2 100644
--- a/libavutil/subfmt.h
+++ b/libavutil/subfmt.h
@@ -21,6 +21,9 @@
 #ifndef AVUTIL_SUBFMT_H
 #define AVUTIL_SUBFMT_H
 
+#include <stdint.h>
+
+#include "buffer.h"
 #include "version.h"
 
 enum AVSubtitleType {
@@ -65,4 +68,48 @@ enum AVSubtitleType {
     AV_SUBTITLE_FMT_NB,         ///< number of subtitle formats, DO NOT USE THIS if you want to link with shared libav* because the number of formats might differ between versions.
 };
 
+typedef struct AVSubtitleArea {
+#define AV_NUM_BUFFER_POINTERS 1
+
+    enum AVSubtitleType type;
+    int flags;
+
+    int x;         ///< top left corner  of area.
+    int y;         ///< top left corner  of area.
+    int w;         ///< width            of area.
+    int h;         ///< height           of area.
+    int nb_colors; ///< number of colors in bitmap palette (@ref pal).
+
+    /**
+     * Buffers and line sizes for the bitmap of this subtitle.
+     *
+     * @{
+     */
+    AVBufferRef *buf[AV_NUM_BUFFER_POINTERS];
+    int linesize[AV_NUM_BUFFER_POINTERS];
+    /**
+     * @}
+     */
+
+    uint32_t pal[256]; ///< RGBA palette for the bitmap.
+
+    char *text;        ///< 0-terminated plain UTF-8 text
+    char *ass;         ///< 0-terminated ASS/SSA compatible event line.
+
+} AVSubtitleArea;
+
+/**
+ * Return the name of sub_fmt, or NULL if sub_fmt is not
+ * recognized.
+ */
+const char *av_get_subtitle_fmt_name(enum AVSubtitleType sub_fmt);
+
+/**
+ * Return a subtitle format corresponding to name, or AV_SUBTITLE_FMT_NONE
+ * on error.
+ *
+ * @param name Subtitle format name.
+ */
+enum AVSubtitleType av_get_subtitle_fmt(const char *name);
+
 #endif /* AVUTIL_SUBFMT_H */
-- 
ffmpeg-codebot

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".