Re: [FFmpeg-devel] [PATCH] libavcodec/qsvenc: add ROI support to qsv encoder

["Chen, Wenbin" <wenbin.chen-at-intel.com@ffmpeg.org>]

> On Fri, 2022-05-06 at 13:49 +0800, Wenbin Chen wrote:
> > Use the mfxEncoderCtrl parameter to enable ROI. Get side data
> > "AVRegionOfInterest"
> > from filter "addroi" and use it to configure "mfxExtEncoderROI" which is
> > the MediaSDK's ROI configuration.
> >
> > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> > ---
> >  libavcodec/qsv_internal.h |  4 ++
> >  libavcodec/qsvenc.c       | 88
> +++++++++++++++++++++++++++++++++++++++
> >  2 files changed, 92 insertions(+)
> >
> > diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
> > index e2aecdcbd6..8131acdae9 100644
> > --- a/libavcodec/qsv_internal.h
> > +++ b/libavcodec/qsv_internal.h
> > @@ -51,6 +51,9 @@
> >  #define ASYNC_DEPTH_DEFAULT 4       // internal parallelism
> >
> >  #define QSV_MAX_ENC_PAYLOAD 2       // # of mfxEncodeCtrl payloads
> supported
> > +#define QSV_MAX_ENC_EXTPARAM 2
> > +
> > +#define QSV_MAX_ROI_NUM 256
> >
> >  #define QSV_MAX_FRAME_EXT_PARAMS 4
> >
> > @@ -83,6 +86,7 @@ typedef struct QSVFrame {
> >      int num_ext_params;
> >
> >      mfxPayload *payloads[QSV_MAX_ENC_PAYLOAD]; ///< used for
> enc_ctrl.Payload
> > +    mfxExtBuffer *extparam[QSV_MAX_ENC_EXTPARAM]; ///< used for
> > enc_ctrl.ExtParam
> >
> >      int queued;
> >      int used;
> > diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
> > index fbb22ca436..9abead604b 100644
> > --- a/libavcodec/qsvenc.c
> > +++ b/libavcodec/qsvenc.c
> > @@ -1501,15 +1501,29 @@ static void
> free_encoder_ctrl_payloads(mfxEncodeCtrl*
> > enc_ctrl)
> >      }
> >  }
> >
> > +static void free_encoder_ctrl_extparam(mfxEncodeCtrl* enc_ctrl)
> > +{
> > +    if (enc_ctrl) {
> > +        int i;
> > +        for (i = 0; i < enc_ctrl->NumExtParam && i <
> QSV_MAX_ENC_EXTPARAM;
> > i++) {
> > +            if (enc_ctrl->ExtParam[i])
> > +                av_freep(&(enc_ctrl->ExtParam[i]));
> > +        }
> > +        enc_ctrl->NumExtParam = 0;
> > +    }
> > +}
> > +
> >  static void clear_unused_frames(QSVEncContext *q)
> >  {
> >      QSVFrame *cur = q->work_frames;
> >      while (cur) {
> >          if (cur->used && !cur->surface.Data.Locked) {
> >              free_encoder_ctrl_payloads(&cur->enc_ctrl);
> > +            free_encoder_ctrl_extparam(&cur->enc_ctrl);
> >              //do not reuse enc_ctrl from previous frame
> >              memset(&cur->enc_ctrl, 0, sizeof(cur->enc_ctrl));
> >              cur->enc_ctrl.Payload = cur->payloads;
> > +            cur->enc_ctrl.ExtParam = cur->extparam;
> >              if (cur->frame->format == AV_PIX_FMT_QSV) {
> >                  av_frame_unref(cur->frame);
> >              }
> > @@ -1547,6 +1561,7 @@ static int get_free_frame(QSVEncContext *q,
> QSVFrame
> > **f)
> >          return AVERROR(ENOMEM);
> >      }
> >      frame->enc_ctrl.Payload = frame->payloads;
> > +    frame->enc_ctrl.ExtParam = frame->extparam;
> >      *last = frame;
> >
> >      *f = frame;
> > @@ -1648,6 +1663,70 @@ static void print_interlace_msg(AVCodecContext
> *avctx,
> > QSVEncContext *q)
> >      }
> >  }
> >
> > +static int set_roi_encode_ctrl(AVCodecContext *avctx, const AVFrame
> *frame,
> > +                               mfxEncodeCtrl *enc_ctrl)
> > +{
> > +#if QSV_VERSION_ATLEAST(1, 22)
> 
> This check can be removed now because the minimal version is 1.28
> 
> > +    AVFrameSideData *sd = NULL;
> > +    int mb_size;
> > +
> > +    if (avctx->codec_id == AV_CODEC_ID_H264) {
> > +        mb_size = 16;
> > +    } else if (avctx->codec_id == AV_CODEC_ID_H265) {
> > +        mb_size = 32;
> > +    } else {
> > +        return 0;
> > +    }
> > +
> > +    if (frame)
> > +        sd = av_frame_get_side_data(frame,
> > AV_FRAME_DATA_REGIONS_OF_INTEREST);
> > +
> > +    if (sd) {
> > +        mfxExtEncoderROI *enc_roi = NULL;
> > +        AVRegionOfInterest *roi;
> > +        uint32_t roi_size;
> > +        int nb_roi, i;
> > +
> > +        roi = (AVRegionOfInterest *)sd->data;
> > +        roi_size = roi->self_size;
> > +        if (!roi_size || sd->size % roi_size) {
> > +            av_log(avctx, AV_LOG_ERROR, "Invalid ROI Data.\n");
> > +            return AVERROR(EINVAL);
> > +        }
> > +        nb_roi = sd->size / roi_size;
> > +        if (nb_roi > QSV_MAX_ROI_NUM) {
> > +            av_log(avctx, AV_LOG_WARNING, "More ROIs set than "
> > +                    "supported by driver (%d > %d).\n",
> > +                    nb_roi, QSV_MAX_ROI_NUM);
> > +            nb_roi = QSV_MAX_ROI_NUM;
> > +        }
> > +
> > +        enc_roi = av_mallocz(sizeof(*enc_roi));
> > +        if (!enc_roi)
> > +            return AVERROR(ENOMEM);
> > +        enc_roi->Header.BufferId = MFX_EXTBUFF_ENCODER_ROI;
> > +        enc_roi->Header.BufferSz = sizeof(*enc_roi);
> > +        enc_roi->NumROI  = nb_roi;
> > +        enc_roi->ROIMode = MFX_ROI_MODE_QP_DELTA;
> > +        for (i = 0; i < nb_roi; i++) {
> > +            roi = (AVRegionOfInterest *)(sd->data + roi_size * i);
> > +            enc_roi->ROI[i].Top    = FFALIGN(roi->top, mb_size);
> > +            enc_roi->ROI[i].Bottom = FFALIGN(roi->bottom, mb_size);
> > +            enc_roi->ROI[i].Left   = FFALIGN(roi->left, mb_size);
> > +            enc_roi->ROI[i].Right  = FFALIGN(roi->right, mb_size);
> > +            enc_roi->ROI[i].DeltaQP =
> > +                roi->qoffset.num * 51 / roi->qoffset.den;
> > +            av_log(avctx, AV_LOG_DEBUG, "ROI: (%d,%d)-(%d,%d) -> %+d.\n",
> > +                   roi->top, roi->left, roi->bottom, roi->right,
> > +                   enc_roi->ROI[i].DeltaQP);
> > +        }
> > +        enc_ctrl->ExtParam[enc_ctrl->NumExtParam] = (mfxExtBuffer
> *)enc_roi;
> 
> 
> You should ensure the runtime supports MFX_EXTBUFF_ENCODER_ROI
> when adding
> MFX_EXTBUFF_ENCODER_ROI buffer to enc_ctrl->ExtParam[].
> 
> Thanks
> Haihao

Thanks for review. I will update the PR.
I will add runtime check before call set_roi_encode_ctrl().

Thanks
Wenbin
> 
> > +        enc_ctrl->NumExtParam++;
> > +    }
> > +#endif
> > +    return 0;
> > +}
> > +
> >  static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
> >                          const AVFrame *frame)
> >  {
> > @@ -1714,6 +1793,14 @@ static int encode_frame(AVCodecContext
> *avctx,
> > QSVEncContext *q,
> >          q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl);
> >      }
> >
> > +    if ((avctx->codec_id == AV_CODEC_ID_H264 ||
> > +         avctx->codec_id == AV_CODEC_ID_H265) &&
> > +         enc_ctrl) {
> > +        ret = set_roi_encode_ctrl(avctx, frame, enc_ctrl);
> > +        if (ret < 0)
> > +            goto free;
> > +    }
> > +
> >      pkt.sync = av_mallocz(sizeof(*pkt.sync));
> >      if (!pkt.sync)
> >          goto nomem;
> > @@ -1842,6 +1929,7 @@ int ff_qsv_enc_close(AVCodecContext *avctx,
> > QSVEncContext *q)
> >      while (cur) {
> >          q->work_frames = cur->next;
> >          av_frame_free(&cur->frame);
> > +        free_encoder_ctrl_extparam(&cur->enc_ctrl);
> >          free_encoder_ctrl_payloads(&cur->enc_ctrl);
> >          av_freep(&cur);
> >          cur = q->work_frames;
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".