Re: [FFmpeg-devel] [PATCH v4 6/6] avcodec/qsvdec: Implement SEI parsing for QSV decoders

[Andreas Rheinhardt <andreas.rheinhardt@outlook.com>]

softworkz:
> From: softworkz <softworkz@hotmail.com>
> 
> Signed-off-by: softworkz <softworkz@hotmail.com>
> ---
>  libavcodec/qsvdec.c | 234 ++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 234 insertions(+)
> 
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 5fc5bed4c8..e854f363ec 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -49,6 +49,12 @@
>  #include "hwconfig.h"
>  #include "qsv.h"
>  #include "qsv_internal.h"
> +#include "h264dec.h"
> +#include "h264_sei.h"
> +#include "hevcdec.h"
> +#include "hevc_ps.h"
> +#include "hevc_sei.h"
> +#include "mpeg12.h"
>  
>  static const AVRational mfx_tb = { 1, 90000 };
>  
> @@ -60,6 +66,8 @@ static const AVRational mfx_tb = { 1, 90000 };
>      AV_NOPTS_VALUE : pts_tb.num ? \
>      av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
>  
> +#define PAYLOAD_BUFFER_SIZE 65535
> +
>  typedef struct QSVAsyncFrame {
>      mfxSyncPoint *sync;
>      QSVFrame     *frame;
> @@ -101,6 +109,9 @@ typedef struct QSVContext {
>  
>      mfxExtBuffer **ext_buffers;
>      int         nb_ext_buffers;
> +
> +    mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> +    Mpeg1Context mpeg_ctx;
>  } QSVContext;
>  
>  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -599,6 +610,210 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
>      return 0;
>  }
>  #endif
> +static int find_start_offset(mfxU8 data[4])
> +{
> +    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> +        return 3;
> +
> +    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
> +        return 4;
> +
> +    return 0;
> +}
> +
> +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
> +{
> +    H264SEIContext sei = { 0 };
> +    GetBitContext gb = { 0 };
> +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) };
> +    mfxU64 ts;
> +    int ret;
> +
> +    while (1) {
> +        int start;
> +        memset(payload.Data, 0, payload.BufSize);
> +
> +        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
> +            return 0;
> +        }
> +        if (ret != MFX_ERR_NONE)
> +            return ret;
> +
> +        if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> +            break;
> +
> +        start = find_start_offset(payload.Data);
> +
> +        switch (payload.Type) {
> +            case SEI_TYPE_BUFFERING_PERIOD:
> +            case SEI_TYPE_PIC_TIMING:
> +                continue;
> +        }
> +
> +        if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
> +            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +        else {
> +            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> +
> +            if (ret < 0)
> +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +            else
> +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d\n", payload.Type, payload.NumBit);
> +        }
> +    }
> +
> +    if (out)
> +        return ff_h264_export_frame_props(avctx, &sei, NULL, out);
> +
> +    return 0;
> +}
> +
> +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame* out)
> +{
> +    HEVCSEI sei = { 0 };
> +    HEVCParamSets ps = { 0 };
> +    GetBitContext gb = { 0 };
> +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) };
> +    mfxFrameSurface1 *surface = &out->surface;
> +    mfxU64 ts;
> +    int ret, has_logged = 0;
> +
> +    while (1) {
> +        int start;
> +        memset(payload.Data, 0, payload.BufSize);
> +
> +        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
> +            return 0;
> +        }
> +        if (ret != MFX_ERR_NONE)
> +            return ret;
> +
> +        if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> +            break;
> +
> +        if (!has_logged) {
> +            has_logged = 1;
> +            av_log(avctx, AV_LOG_VERBOSE, "-----------------------------------------\n");
> +            av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp);
> +        }
> +
> +        if (ts != surface->Data.TimeStamp) {
> +            av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp);
> +        }
> +
> +        start = find_start_offset(payload.Data);
> +
> +        av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d  Numbits %3d  Start: %d\n", payload.Type, payload.NumBit, start);
> +
> +        switch (payload.Type) {
> +            case SEI_TYPE_BUFFERING_PERIOD:
> +            case SEI_TYPE_PIC_TIMING:
> +                continue;
> +            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> +                // There seems to be a bug in MSDK
> +                payload.NumBit -= 8;
> +
> +                break;
> +            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> +                // There seems to be a bug in MSDK
> +                payload.NumBit = 48;
> +
> +                break;
> +            case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> +                // There seems to be a bug in MSDK
> +                if (payload.NumBit == 552)
> +                    payload.NumBit = 528;
> +                break;
> +        }
> +
> +        if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
> +            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +        else {
> +            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, HEVC_NAL_SEI_PREFIX);
> +
> +            if (ret < 0)
> +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +            else
> +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d\n", payload.Type, payload.NumBit);
> +        }
> +    }
> +
> +    if (has_logged) {
> +        av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> +    }
> +
> +    if (out && out->frame)
> +        return ff_hevc_set_side_data(avctx, &sei, NULL, out->frame);
> +
> +    return 0;
> +}
> +
> +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
> +{
> +    Mpeg1Context *mpeg_ctx = &q->mpeg_ctx;
> +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) };
> +    mfxU64 ts;
> +    int ret;
> +
> +    while (1) {
> +        int start;
> +
> +        memset(payload.Data, 0, payload.BufSize);
> +        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
> +            return 0;
> +        }
> +        if (ret != MFX_ERR_NONE)
> +            return ret;
> +
> +        if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> +            break;
> +
> +        start = find_start_offset(payload.Data);
> +
> +        start++;
> +
> +        ff_mpeg_decode_user_data(avctx, mpeg_ctx, &payload.Data[start], (int)((payload.NumBit + 7) / 8) - start);
> +
> +        av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d start %d -> %.s\n", payload.Type, payload.NumBit, start, (char *)(&payload.Data[start]));
> +    }
> +
> +    if (!out)
> +        return 0;
> +
> +    if (mpeg_ctx->a53_buf_ref) {
> +
> +        AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, mpeg_ctx->a53_buf_ref);
> +        if (!sd)
> +            av_buffer_unref(&mpeg_ctx->a53_buf_ref);
> +        mpeg_ctx->a53_buf_ref = NULL;
> +    }
> +
> +    if (mpeg_ctx->has_stereo3d) {
> +        AVStereo3D *stereo = av_stereo3d_create_side_data(out);
> +        if (!stereo)
> +            return AVERROR(ENOMEM);
> +
> +        *stereo = mpeg_ctx->stereo3d;
> +        mpeg_ctx->has_stereo3d = 0;
> +    }
> +
> +    if (mpeg_ctx->has_afd) {
> +        AVFrameSideData *sd = av_frame_new_side_data(out, AV_FRAME_DATA_AFD, 1);
> +        if (!sd)
> +            return AVERROR(ENOMEM);
> +
> +        *sd->data   = mpeg_ctx->afd;
> +        mpeg_ctx->has_afd = 0;
> +    }
> +
> +    return 0;
> +}
>  
>  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>                        AVFrame *frame, int *got_frame,
> @@ -636,6 +851,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>                                                insurf, &outsurf, sync);
>          if (ret == MFX_WRN_DEVICE_BUSY)
>              av_usleep(500);
> +        else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
> +            parse_sei_mpeg12(avctx, q, NULL);
>  
>      } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
>  
> @@ -677,6 +894,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>              return AVERROR_BUG;
>          }
>  
> +        switch (avctx->codec_id) {
> +        case AV_CODEC_ID_MPEG2VIDEO:
> +            ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
> +            break;
> +        case AV_CODEC_ID_H264:
> +            ret = parse_sei_h264(avctx, q, out_frame->frame);
> +            break;
> +        case AV_CODEC_ID_HEVC:
> +            ret = parse_sei_hevc(avctx, q, out_frame);
> +            break;
> +        default:
> +            ret = 0;
> +        }
> +
> +        if (ret < 0)
> +            av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret);
> +
>          out_frame->queued += 1;
>  
>          aframe = (QSVAsyncFrame){ sync, out_frame };

You completely forgot necessary changes to configure/the Makefile. The
way you are doing it here means that you basically have the qsv decoders
to rely on the H.264/HEVC/MPEG-1/2 decoders which is way too much.

- Andreas
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".