Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement SEI parsing for QSV decoders

["Xiang, Haihao" <haihao.xiang-at-intel.com@ffmpeg.org>]

On Mon, 2022-11-21 at 15:58 +0000, Soft Works wrote:
> > -----Original Message-----
> > From: Xiang, Haihao <haihao.xiang@intel.com>
> > Sent: Monday, November 21, 2022 3:45 AM
> > To: ffmpeg-devel@ffmpeg.org
> > Cc: softworkz@hotmail.com; kierank@obe.tv; haihao.xiang-at-
> > intel.com@ffmpeg.org; andreas.rheinhardt@outlook.com
> > Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement
> > SEI parsing for QSV decoders
> > 
> > On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> > > From: softworkz <softworkz@hotmail.com>
> > > 
> > > Signed-off-by: softworkz <softworkz@hotmail.com>
> > > ---
> > >  libavcodec/Makefile |   2 +-
> > >  libavcodec/qsvdec.c | 321
> > ++++++++++++++++++++++++++++++++++++++++++++
> > >  2 files changed, 322 insertions(+), 1 deletion(-)
> > > 
> > > diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> > > index 90c7f113a3..cbddbb0ace 100644
> > > --- a/libavcodec/Makefile
> > > +++ b/libavcodec/Makefile
> > > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP)                +=
> > mss34dsp.o
> > >  OBJS-$(CONFIG_PIXBLOCKDSP)             += pixblockdsp.o
> > >  OBJS-$(CONFIG_QPELDSP)                 += qpeldsp.o
> > >  OBJS-$(CONFIG_QSV)                     += qsv.o
> > > -OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o
> > > +OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o h264_sei.o
> > hevc_sei.o
> > >  OBJS-$(CONFIG_QSVENC)                  += qsvenc.o
> > >  OBJS-$(CONFIG_RANGECODER)              += rangecoder.o
> > >  OBJS-$(CONFIG_RDFT)                    += rdft.o
> > > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > > index 73405b5747..467a248224 100644
> > > --- a/libavcodec/qsvdec.c
> > > +++ b/libavcodec/qsvdec.c
> > > @@ -41,6 +41,7 @@
> > >  #include "libavutil/time.h"
> > >  #include "libavutil/imgutils.h"
> > >  #include "libavutil/film_grain_params.h"
> > > +#include <libavutil/reverse.h>
> > > 
> > >  #include "avcodec.h"
> > >  #include "codec_internal.h"
> > > @@ -49,6 +50,9 @@
> > >  #include "hwconfig.h"
> > >  #include "qsv.h"
> > >  #include "qsv_internal.h"
> > > +#include "h264_sei.h"
> > > +#include "hevc_ps.h"
> > > +#include "hevc_sei.h"
> > > 
> > >  #if QSV_ONEVPL
> > >  #include <mfxdispatcher.h>
> > > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> > >      AV_NOPTS_VALUE : pts_tb.num ? \
> > >      av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
> > > 
> > > +#define PAYLOAD_BUFFER_SIZE 65535
> > > +
> > >  typedef struct QSVAsyncFrame {
> > >      mfxSyncPoint *sync;
> > >      QSVFrame     *frame;
> > > @@ -107,6 +113,9 @@ typedef struct QSVContext {
> > > 
> > >      mfxExtBuffer **ext_buffers;
> > >      int         nb_ext_buffers;
> > > +
> > > +    mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> > > +    AVBufferRef *a53_buf_ref;
> > >  } QSVContext;
> > > 
> > >  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > > @@ -628,6 +637,299 @@ static int
> > qsv_export_film_grain(AVCodecContext *avctx,
> > > mfxExtAV1FilmGrainParam
> > >  }
> > >  #endif
> > > 
> > > +static int find_start_offset(mfxU8 data[4])
> > > +{
> > > +    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> > > +        return 3;
> > > +
> > > +    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
> > 1)
> > > +        return 4;
> > > +
> > > +    return 0;
> > > +}
> > > +
> > > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q,
> > AVFrame* out)
> > > +{
> > > +    H264SEIContext sei = { 0 };
> > > +    GetBitContext gb = { 0 };
> > > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> > .BufSize =
> > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > > +    mfxU64 ts;
> > > +    int ret;
> > > +
> > > +    while (1) {
> > > +        int start;
> > > +        memset(payload.Data, 0, payload.BufSize);
> > > +
> > > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> > &payload);
> > > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> > buffer on
> > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> > > payload_buffer),
> > > payload.BufSize);
> > > +            return 0;
> > > +        }
> > > +        if (ret != MFX_ERR_NONE)
> > > +            return ret;
> > > +
> > > +        if (payload.NumBit == 0 || payload.NumBit >=
> > payload.BufSize * 8)
> > > +            break;
> > > +
> > > +        start = find_start_offset(payload.Data);
> > > +
> > > +        switch (payload.Type) {
> > > +            case SEI_TYPE_BUFFERING_PERIOD:
> > > +            case SEI_TYPE_PIC_TIMING:
> > > +                continue;
> > > +        }
> > > +
> > > +        if (init_get_bits(&gb, &payload.Data[start],
> > payload.NumBit - start *
> > > 8) < 0)
> > > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> > bitstream reader
> > > SEI type: %d  Numbits %d error: %d\n", payload.Type,
> > payload.NumBit, ret);
> > > +        else {
> > > +            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> > > +
> > > +            if (ret < 0)
> > > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> > type:
> > > %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > > +            else
> > > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> > Numbits
> > > %d\n", payload.Type, payload.NumBit);
> > > +        }
> > > +    }
> > > +
> > > +    if (out)
> > > +        return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL,
> > 0);
> > > +
> > > +    return 0;
> > > +}
> > > +
> > > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q,
> > QSVFrame*
> > > out)
> > > +{
> > > +    HEVCSEI sei = { 0 };
> > > +    HEVCParamSets ps = { 0 };
> > > +    GetBitContext gb = { 0 };
> > > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> > .BufSize =
> > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > > +    mfxFrameSurface1 *surface = &out->surface;
> > > +    mfxU64 ts;
> > > +    int ret, has_logged = 0;
> > > +
> > > +    while (1) {
> > > +        int start;
> > > +        memset(payload.Data, 0, payload.BufSize);
> > > +
> > > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> > &payload);
> > > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> > buffer on
> > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> > > payload_buffer),
> > > payload.BufSize);
> > > +            return 0;
> > > +        }
> > > +        if (ret != MFX_ERR_NONE)
> > > +            return ret;
> > > +
> > > +        if (payload.NumBit == 0 || payload.NumBit >=
> > payload.BufSize * 8)
> > > +            break;
> > > +
> > > +        if (!has_logged) {
> > > +            has_logged = 1;
> > > +            av_log(avctx, AV_LOG_VERBOSE, "-----------------------
> > ---------
> > > ---------\n");
> > > +            av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI -
> > payload
> > > timestamp: %llu - surface timestamp: %llu\n", ts, surface-
> > > Data.TimeStamp);
> > > +        }
> > > +
> > > +        if (ts != surface->Data.TimeStamp) {
> > > +            av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp
> > (%llu) does
> > > not match surface timestamp: (%llu)\n", ts, surface-
> > > Data.TimeStamp);
> > > +        }
> > > +
> > > +        start = find_start_offset(payload.Data);
> > > +
> > > +        av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d
> > Numbits
> > > %3d  Start: %d\n", payload.Type, payload.NumBit, start);
> > > +
> > > +        switch (payload.Type) {
> > > +            case SEI_TYPE_BUFFERING_PERIOD:
> > > +            case SEI_TYPE_PIC_TIMING:
> > > +                continue;
> > > +            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> > > +                // There seems to be a bug in MSDK
> > > +                payload.NumBit -= 8;
> > > +
> > > +                break;
> > > +            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> > > +                // There seems to be a bug in MSDK
> > > +                payload.NumBit = 48;
> > > +
> > > +                break;
> > > +            case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> > > +                // There seems to be a bug in MSDK
> > > +                if (payload.NumBit == 552)
> > > +                    payload.NumBit = 528;
> > > +                break;
> > > +        }
> > > +
> > > +        if (init_get_bits(&gb, &payload.Data[start],
> > payload.NumBit - start *
> > > 8) < 0)
> > > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> > bitstream reader
> > > SEI type: %d  Numbits %d error: %d\n", payload.Type,
> > payload.NumBit, ret);
> > > +        else {
> > > +            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> > > HEVC_NAL_SEI_PREFIX);
> > > +
> > > +            if (ret < 0)
> > > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> > type:
> > > %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > > +            else
> > > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> > Numbits
> > > %d\n", payload.Type, payload.NumBit);
> > > +        }
> > > +    }
> > > +
> > > +    if (has_logged) {
> > > +        av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> > > +    }
> > > +
> > > +    if (out && out->frame)
> > > +        return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame,
> > avctx-
> > > > framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> > > bit_depth_chroma);
> > 
> > I got segfault when trying your patchset,
> > 
> > Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
> > 0x00007ffff67c0497 in parse_sei_hevc
> > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> > 777             return ff_hevc_set_sei_to_frame(avctx, &sei, out-
> > > frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> > > bit_depth_chroma);
> > (gdb) bt
> > #0  0x00007ffff67c0497 in parse_sei_hevc
> > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> > #1  0x00007ffff67c1afe in qsv_decode
> > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> > frame=frame@entry=0x5555556df740,
> > got_frame=got_frame@entry=0x7fffffffd6bc,
> >     avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020
> > BTW the SDK provides support for hevc HDR metadata, we needn't parse
> > SEI payload
> > in qsvdec and may get the corresponding info from the SDK, see
> > https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html
> 
> I know. I was the one who had requested this to be added to MSDK :-)
> 
> But it's just one small part of SEI information, it's limited to
> the latest MSDK versions and I'm not sure whether it's working
> as reliably as this implementation. This would need to be tested.
> 
> You should still have access to the repo with the test files for 
> demoing the offset problems (which my patchset is working around).


I worked out a patchset (https://github.com/intel-media-ci/ffmpeg/pull/518),
including https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html an
d others. I may use your command (with some changes) to convert all HDR clips in
your repo to SDR clips except one clip (w/wo my changes, there is a GPU hang
issue with this clip). 

We may only do a small update to support AV1 HDR in the future if applying 
https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html.

Note the command below doesn't work with all test files in your repo with your
patchset v6.

$ ffmpeg -hwaccel qsv -c:v hevc_qsv -i input.mp4 -f null -

Thanks
Haihao

> 
> But it's also about dynamic HDR data, dovi data, etc. - which 
> MSDK doesn't provide, so this single bit of SEI data doesn't 
> help much.
> 
> Best regards,
> softworkz
> 
> 
> 
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".