Re: [FFmpeg-devel] [PATCH v4 6/6] avcodec/qsvdec: Implement SEI parsing for QSV decoders

[Soft Works <softworkz@hotmail.com>]

> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Andreas Rheinhardt
> Sent: Tuesday, June 28, 2022 6:17 AM
> To: ffmpeg-devel@ffmpeg.org
> Subject: Re: [FFmpeg-devel] [PATCH v4 6/6] avcodec/qsvdec: Implement
> SEI parsing for QSV decoders
> 
> softworkz:
> > From: softworkz <softworkz@hotmail.com>
> >
> > Signed-off-by: softworkz <softworkz@hotmail.com>
> > ---
> >  libavcodec/qsvdec.c | 234
> ++++++++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 234 insertions(+)
> >
> > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > index 5fc5bed4c8..e854f363ec 100644
> > --- a/libavcodec/qsvdec.c
> > +++ b/libavcodec/qsvdec.c
> > @@ -49,6 +49,12 @@
> >  #include "hwconfig.h"
> >  #include "qsv.h"
> >  #include "qsv_internal.h"
> > +#include "h264dec.h"
> > +#include "h264_sei.h"
> > +#include "hevcdec.h"
> > +#include "hevc_ps.h"
> > +#include "hevc_sei.h"
> > +#include "mpeg12.h"
> >
> >  static const AVRational mfx_tb = { 1, 90000 };
> >
> > @@ -60,6 +66,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> >      AV_NOPTS_VALUE : pts_tb.num ? \
> >      av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
> >
> > +#define PAYLOAD_BUFFER_SIZE 65535
> > +
> >  typedef struct QSVAsyncFrame {
> >      mfxSyncPoint *sync;
> >      QSVFrame     *frame;
> > @@ -101,6 +109,9 @@ typedef struct QSVContext {
> >
> >      mfxExtBuffer **ext_buffers;
> >      int         nb_ext_buffers;
> > +
> > +    mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> > +    Mpeg1Context mpeg_ctx;
> >  } QSVContext;
> >
> >  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > @@ -599,6 +610,210 @@ static int
> qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
> >      return 0;
> >  }
> >  #endif
> > +static int find_start_offset(mfxU8 data[4])
> > +{
> > +    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> > +        return 3;
> > +
> > +    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
> 1)
> > +        return 4;
> > +
> > +    return 0;
> > +}
> > +
> > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q,
> AVFrame* out)
> > +{
> > +    H264SEIContext sei = { 0 };
> > +    GetBitContext gb = { 0 };
> > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize = sizeof(q->payload_buffer) };
> > +    mfxU64 ts;
> > +    int ret;
> > +
> > +    while (1) {
> > +        int start;
> > +        memset(payload.Data, 0, payload.BufSize);
> > +
> > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer), payload.BufSize);
> > +            return 0;
> > +        }
> > +        if (ret != MFX_ERR_NONE)
> > +            return ret;
> > +
> > +        if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > +            break;
> > +
> > +        start = find_start_offset(payload.Data);
> > +
> > +        switch (payload.Type) {
> > +            case SEI_TYPE_BUFFERING_PERIOD:
> > +            case SEI_TYPE_PIC_TIMING:
> > +                continue;
> > +        }
> > +
> > +        if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start * 8) < 0)
> > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader SEI type: %d  Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > +        else {
> > +            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> > +
> > +            if (ret < 0)
> > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit,
> ret);
> > +            else
> > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits %d\n", payload.Type, payload.NumBit);
> > +        }
> > +    }
> > +
> > +    if (out)
> > +        return ff_h264_export_frame_props(avctx, &sei, NULL, out);
> > +
> > +    return 0;
> > +}
> > +
> > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q,
> QSVFrame* out)
> > +{
> > +    HEVCSEI sei = { 0 };
> > +    HEVCParamSets ps = { 0 };
> > +    GetBitContext gb = { 0 };
> > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize = sizeof(q->payload_buffer) };
> > +    mfxFrameSurface1 *surface = &out->surface;
> > +    mfxU64 ts;
> > +    int ret, has_logged = 0;
> > +
> > +    while (1) {
> > +        int start;
> > +        memset(payload.Data, 0, payload.BufSize);
> > +
> > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer), payload.BufSize);
> > +            return 0;
> > +        }
> > +        if (ret != MFX_ERR_NONE)
> > +            return ret;
> > +
> > +        if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > +            break;
> > +
> > +        if (!has_logged) {
> > +            has_logged = 1;
> > +            av_log(avctx, AV_LOG_VERBOSE, "-----------------------
> ------------------\n");
> > +            av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI -
> payload timestamp: %llu - surface timestamp: %llu\n", ts, surface-
> >Data.TimeStamp);
> > +        }
> > +
> > +        if (ts != surface->Data.TimeStamp) {
> > +            av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp
> (%llu) does not match surface timestamp: (%llu)\n", ts, surface-
> >Data.TimeStamp);
> > +        }
> > +
> > +        start = find_start_offset(payload.Data);
> > +
> > +        av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d
> Numbits %3d  Start: %d\n", payload.Type, payload.NumBit, start);
> > +
> > +        switch (payload.Type) {
> > +            case SEI_TYPE_BUFFERING_PERIOD:
> > +            case SEI_TYPE_PIC_TIMING:
> > +                continue;
> > +            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> > +                // There seems to be a bug in MSDK
> > +                payload.NumBit -= 8;
> > +
> > +                break;
> > +            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> > +                // There seems to be a bug in MSDK
> > +                payload.NumBit = 48;
> > +
> > +                break;
> > +            case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> > +                // There seems to be a bug in MSDK
> > +                if (payload.NumBit == 552)
> > +                    payload.NumBit = 528;
> > +                break;
> > +        }
> > +
> > +        if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start * 8) < 0)
> > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader SEI type: %d  Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > +        else {
> > +            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> HEVC_NAL_SEI_PREFIX);
> > +
> > +            if (ret < 0)
> > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit,
> ret);
> > +            else
> > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits %d\n", payload.Type, payload.NumBit);
> > +        }
> > +    }
> > +
> > +    if (has_logged) {
> > +        av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> > +    }
> > +
> > +    if (out && out->frame)
> > +        return ff_hevc_set_side_data(avctx, &sei, NULL, out-
> >frame);
> > +
> > +    return 0;
> > +}
> > +
> > +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q,
> AVFrame* out)
> > +{
> > +    Mpeg1Context *mpeg_ctx = &q->mpeg_ctx;
> > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize = sizeof(q->payload_buffer) };
> > +    mfxU64 ts;
> > +    int ret;
> > +
> > +    while (1) {
> > +        int start;
> > +
> > +        memset(payload.Data, 0, payload.BufSize);
> > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer), payload.BufSize);
> > +            return 0;
> > +        }
> > +        if (ret != MFX_ERR_NONE)
> > +            return ret;
> > +
> > +        if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > +            break;
> > +
> > +        start = find_start_offset(payload.Data);
> > +
> > +        start++;
> > +
> > +        ff_mpeg_decode_user_data(avctx, mpeg_ctx,
> &payload.Data[start], (int)((payload.NumBit + 7) / 8) - start);
> > +
> > +        av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits
> %d start %d -> %.s\n", payload.Type, payload.NumBit, start, (char
> *)(&payload.Data[start]));
> > +    }
> > +
> > +    if (!out)
> > +        return 0;
> > +
> > +    if (mpeg_ctx->a53_buf_ref) {
> > +
> > +        AVFrameSideData *sd = av_frame_new_side_data_from_buf(out,
> AV_FRAME_DATA_A53_CC, mpeg_ctx->a53_buf_ref);
> > +        if (!sd)
> > +            av_buffer_unref(&mpeg_ctx->a53_buf_ref);
> > +        mpeg_ctx->a53_buf_ref = NULL;
> > +    }
> > +
> > +    if (mpeg_ctx->has_stereo3d) {
> > +        AVStereo3D *stereo = av_stereo3d_create_side_data(out);
> > +        if (!stereo)
> > +            return AVERROR(ENOMEM);
> > +
> > +        *stereo = mpeg_ctx->stereo3d;
> > +        mpeg_ctx->has_stereo3d = 0;
> > +    }
> > +
> > +    if (mpeg_ctx->has_afd) {
> > +        AVFrameSideData *sd = av_frame_new_side_data(out,
> AV_FRAME_DATA_AFD, 1);
> > +        if (!sd)
> > +            return AVERROR(ENOMEM);
> > +
> > +        *sd->data   = mpeg_ctx->afd;
> > +        mpeg_ctx->has_afd = 0;
> > +    }
> > +
> > +    return 0;
> > +}
> >
> >  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> >                        AVFrame *frame, int *got_frame,
> > @@ -636,6 +851,8 @@ static int qsv_decode(AVCodecContext *avctx,
> QSVContext *q,
> >                                                insurf, &outsurf,
> sync);
> >          if (ret == MFX_WRN_DEVICE_BUSY)
> >              av_usleep(500);
> > +        else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
> > +            parse_sei_mpeg12(avctx, q, NULL);
> >
> >      } while (ret == MFX_WRN_DEVICE_BUSY || ret ==
> MFX_ERR_MORE_SURFACE);
> >
> > @@ -677,6 +894,23 @@ static int qsv_decode(AVCodecContext *avctx,
> QSVContext *q,
> >              return AVERROR_BUG;
> >          }
> >
> > +        switch (avctx->codec_id) {
> > +        case AV_CODEC_ID_MPEG2VIDEO:
> > +            ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
> > +            break;
> > +        case AV_CODEC_ID_H264:
> > +            ret = parse_sei_h264(avctx, q, out_frame->frame);
> > +            break;
> > +        case AV_CODEC_ID_HEVC:
> > +            ret = parse_sei_hevc(avctx, q, out_frame);
> > +            break;
> > +        default:
> > +            ret = 0;
> > +        }
> > +
> > +        if (ret < 0)
> > +            av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data:
> %d\n", ret);
> > +
> >          out_frame->queued += 1;
> >
> >          aframe = (QSVAsyncFrame){ sync, out_frame };
> 
> You completely forgot necessary changes to configure/the Makefile.
> The
> way you are doing it here means that you basically have the qsv
> decoders
> to rely on the H.264/HEVC/MPEG-1/2 decoders which is way too much.

You are referring to the hypothetical case where one would disable
one of the sw decoders while having a qsv decoder enabled, right?

The SEI parsing code is not trivial and tied to those decoders 
(means using these contexts). It would be not a straightforward
task to extract/separate those parts, that's why I preferred to
just make that functionality accessible.
I wouldn't mind when the QSV decoders would be dependent on those
decoders being included in compilation, even more when considering
that so many other hwaccel decoders have the same dependencies;
DXVA2, D3D11VA, NVDEC, VAAPI.

The question would be whether to not build the qsv decoders when
the sw decoders are deselected or whether to build the sw decoder
code even these are disabled. AFAIU, both would be possible?

Or would you have a better idea?

Thanks,
sw







_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".