Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement SEI parsing for QSV decoders

[Soft Works <softworkz@hotmail.com>]

> -----Original Message-----
> From: Xiang, Haihao <haihao.xiang@intel.com>
> Sent: Monday, November 21, 2022 3:45 AM
> To: ffmpeg-devel@ffmpeg.org
> Cc: softworkz@hotmail.com; kierank@obe.tv; haihao.xiang-at-
> intel.com@ffmpeg.org; andreas.rheinhardt@outlook.com
> Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement
> SEI parsing for QSV decoders
> 
> On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> > From: softworkz <softworkz@hotmail.com>
> >
> > Signed-off-by: softworkz <softworkz@hotmail.com>
> > ---
> >  libavcodec/Makefile |   2 +-
> >  libavcodec/qsvdec.c | 321
> ++++++++++++++++++++++++++++++++++++++++++++
> >  2 files changed, 322 insertions(+), 1 deletion(-)
> >
> > diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> > index 90c7f113a3..cbddbb0ace 100644
> > --- a/libavcodec/Makefile
> > +++ b/libavcodec/Makefile
> > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP)                +=
> mss34dsp.o
> >  OBJS-$(CONFIG_PIXBLOCKDSP)             += pixblockdsp.o
> >  OBJS-$(CONFIG_QPELDSP)                 += qpeldsp.o
> >  OBJS-$(CONFIG_QSV)                     += qsv.o
> > -OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o
> > +OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o h264_sei.o
> hevc_sei.o
> >  OBJS-$(CONFIG_QSVENC)                  += qsvenc.o
> >  OBJS-$(CONFIG_RANGECODER)              += rangecoder.o
> >  OBJS-$(CONFIG_RDFT)                    += rdft.o
> > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > index 73405b5747..467a248224 100644
> > --- a/libavcodec/qsvdec.c
> > +++ b/libavcodec/qsvdec.c
> > @@ -41,6 +41,7 @@
> >  #include "libavutil/time.h"
> >  #include "libavutil/imgutils.h"
> >  #include "libavutil/film_grain_params.h"
> > +#include <libavutil/reverse.h>
> >
> >  #include "avcodec.h"
> >  #include "codec_internal.h"
> > @@ -49,6 +50,9 @@
> >  #include "hwconfig.h"
> >  #include "qsv.h"
> >  #include "qsv_internal.h"
> > +#include "h264_sei.h"
> > +#include "hevc_ps.h"
> > +#include "hevc_sei.h"
> >
> >  #if QSV_ONEVPL
> >  #include <mfxdispatcher.h>
> > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> >      AV_NOPTS_VALUE : pts_tb.num ? \
> >      av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
> >
> > +#define PAYLOAD_BUFFER_SIZE 65535
> > +
> >  typedef struct QSVAsyncFrame {
> >      mfxSyncPoint *sync;
> >      QSVFrame     *frame;
> > @@ -107,6 +113,9 @@ typedef struct QSVContext {
> >
> >      mfxExtBuffer **ext_buffers;
> >      int         nb_ext_buffers;
> > +
> > +    mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> > +    AVBufferRef *a53_buf_ref;
> >  } QSVContext;
> >
> >  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > @@ -628,6 +637,299 @@ static int
> qsv_export_film_grain(AVCodecContext *avctx,
> > mfxExtAV1FilmGrainParam
> >  }
> >  #endif
> >
> > +static int find_start_offset(mfxU8 data[4])
> > +{
> > +    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> > +        return 3;
> > +
> > +    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
> 1)
> > +        return 4;
> > +
> > +    return 0;
> > +}
> > +
> > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q,
> AVFrame* out)
> > +{
> > +    H264SEIContext sei = { 0 };
> > +    GetBitContext gb = { 0 };
> > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize =
> > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > +    mfxU64 ts;
> > +    int ret;
> > +
> > +    while (1) {
> > +        int start;
> > +        memset(payload.Data, 0, payload.BufSize);
> > +
> > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on
> > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer),
> > payload.BufSize);
> > +            return 0;
> > +        }
> > +        if (ret != MFX_ERR_NONE)
> > +            return ret;
> > +
> > +        if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > +            break;
> > +
> > +        start = find_start_offset(payload.Data);
> > +
> > +        switch (payload.Type) {
> > +            case SEI_TYPE_BUFFERING_PERIOD:
> > +            case SEI_TYPE_PIC_TIMING:
> > +                continue;
> > +        }
> > +
> > +        if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start *
> > 8) < 0)
> > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader
> > SEI type: %d  Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > +        else {
> > +            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> > +
> > +            if (ret < 0)
> > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type:
> > %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > +            else
> > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits
> > %d\n", payload.Type, payload.NumBit);
> > +        }
> > +    }
> > +
> > +    if (out)
> > +        return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL,
> 0);
> > +
> > +    return 0;
> > +}
> > +
> > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q,
> QSVFrame*
> > out)
> > +{
> > +    HEVCSEI sei = { 0 };
> > +    HEVCParamSets ps = { 0 };
> > +    GetBitContext gb = { 0 };
> > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize =
> > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > +    mfxFrameSurface1 *surface = &out->surface;
> > +    mfxU64 ts;
> > +    int ret, has_logged = 0;
> > +
> > +    while (1) {
> > +        int start;
> > +        memset(payload.Data, 0, payload.BufSize);
> > +
> > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on
> > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer),
> > payload.BufSize);
> > +            return 0;
> > +        }
> > +        if (ret != MFX_ERR_NONE)
> > +            return ret;
> > +
> > +        if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > +            break;
> > +
> > +        if (!has_logged) {
> > +            has_logged = 1;
> > +            av_log(avctx, AV_LOG_VERBOSE, "-----------------------
> ---------
> > ---------\n");
> > +            av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI -
> payload
> > timestamp: %llu - surface timestamp: %llu\n", ts, surface-
> >Data.TimeStamp);
> > +        }
> > +
> > +        if (ts != surface->Data.TimeStamp) {
> > +            av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp
> (%llu) does
> > not match surface timestamp: (%llu)\n", ts, surface-
> >Data.TimeStamp);
> > +        }
> > +
> > +        start = find_start_offset(payload.Data);
> > +
> > +        av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d
> Numbits
> > %3d  Start: %d\n", payload.Type, payload.NumBit, start);
> > +
> > +        switch (payload.Type) {
> > +            case SEI_TYPE_BUFFERING_PERIOD:
> > +            case SEI_TYPE_PIC_TIMING:
> > +                continue;
> > +            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> > +                // There seems to be a bug in MSDK
> > +                payload.NumBit -= 8;
> > +
> > +                break;
> > +            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> > +                // There seems to be a bug in MSDK
> > +                payload.NumBit = 48;
> > +
> > +                break;
> > +            case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> > +                // There seems to be a bug in MSDK
> > +                if (payload.NumBit == 552)
> > +                    payload.NumBit = 528;
> > +                break;
> > +        }
> > +
> > +        if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start *
> > 8) < 0)
> > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader
> > SEI type: %d  Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > +        else {
> > +            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> > HEVC_NAL_SEI_PREFIX);
> > +
> > +            if (ret < 0)
> > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type:
> > %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > +            else
> > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits
> > %d\n", payload.Type, payload.NumBit);
> > +        }
> > +    }
> > +
> > +    if (has_logged) {
> > +        av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> > +    }
> > +
> > +    if (out && out->frame)
> > +        return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame,
> avctx-
> > >framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> >bit_depth_chroma);
> 
> I got segfault when trying your patchset,
> 
> Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
> 0x00007ffff67c0497 in parse_sei_hevc
> (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> 777             return ff_hevc_set_sei_to_frame(avctx, &sei, out-
> >frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> >bit_depth_chroma);
> (gdb) bt
> #0  0x00007ffff67c0497 in parse_sei_hevc
> (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> #1  0x00007ffff67c1afe in qsv_decode
> (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> frame=frame@entry=0x5555556df740,
> got_frame=got_frame@entry=0x7fffffffd6bc,
>     avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020


> BTW the SDK provides support for hevc HDR metadata, we needn't parse
> SEI payload
> in qsvdec and may get the corresponding info from the SDK, see
> https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html

I know. I was the one who had requested this to be added to MSDK :-)

But it's just one small part of SEI information, it's limited to
the latest MSDK versions and I'm not sure whether it's working
as reliably as this implementation. This would need to be tested.

You should still have access to the repo with the test files for 
demoing the offset problems (which my patchset is working around).

But it's also about dynamic HDR data, dovi data, etc. - which 
MSDK doesn't provide, so this single bit of SEI data doesn't 
help much.

Best regards,
softworkz



_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".