* [FFmpeg-devel] [PATCH] libavcodec/qsvdec.c: extract frame packing arrangement data
@ 2022-04-26 10:00 Wenbin Chen
2022-04-27 9:41 ` Xiang, Haihao
2022-04-28 1:14 ` Andreas Rheinhardt
0 siblings, 2 replies; 4+ messages in thread
From: Wenbin Chen @ 2022-04-26 10:00 UTC (permalink / raw)
To: ffmpeg-devel
Use h264_sei to parse SEI data got from MediaSDK. Extract frame
packing arrangement information from SEI data and config AVStereo3D
side data for decoded frame.
Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
Signed-off-by: Tong Wu <tong1.wu@intel.com>
---
libavcodec/qsv_internal.h | 2 +
libavcodec/qsvdec.c | 160 ++++++++++++++++++++++++++++++++++++++
2 files changed, 162 insertions(+)
diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
index e2aecdcbd6..a804c392c1 100644
--- a/libavcodec/qsv_internal.h
+++ b/libavcodec/qsv_internal.h
@@ -54,6 +54,8 @@
#define QSV_MAX_FRAME_EXT_PARAMS 4
+#define QSV_PAYLOAD_SIZE 1024
+
#define QSV_VERSION_ATLEAST(MAJOR, MINOR) \
(MFX_VERSION_MAJOR > (MAJOR) || \
MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR))
diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index 5fc5bed4c8..26fa178b4d 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -41,13 +41,16 @@
#include "libavutil/time.h"
#include "libavutil/imgutils.h"
#include "libavutil/film_grain_params.h"
+#include "libavutil/stereo3d.h"
#include "avcodec.h"
#include "codec_internal.h"
#include "internal.h"
#include "decode.h"
#include "hwconfig.h"
+#include "get_bits.h"
#include "qsv.h"
+#include "h264_sei.h"
#include "qsv_internal.h"
static const AVRational mfx_tb = { 1, 90000 };
@@ -101,6 +104,10 @@ typedef struct QSVContext {
mfxExtBuffer **ext_buffers;
int nb_ext_buffers;
+
+ mfxPayload payload;
+ H264SEIContext sei;
+ H264ParamSets ps;
} QSVContext;
static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
@@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
}
#endif
+static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame)
+{
+ if (!fpa || !frame)
+ return AVERROR(EINVAL);
+
+ if (!fpa->arrangement_cancel_flag &&
+ fpa->arrangement_type <= 6 &&
+ fpa->content_interpretation_type > 0 &&
+ fpa->content_interpretation_type < 3) {
+ AVStereo3D *stereo = av_stereo3d_create_side_data(frame);
+ if (stereo) {
+ switch (fpa->arrangement_type) {
+ case 0:
+ stereo->type = AV_STEREO3D_CHECKERBOARD;
+ break;
+ case 1:
+ stereo->type = AV_STEREO3D_COLUMNS;
+ break;
+ case 2:
+ stereo->type = AV_STEREO3D_LINES;
+ break;
+ case 3:
+ if (fpa->quincunx_sampling_flag)
+ stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
+ else
+ stereo->type = AV_STEREO3D_SIDEBYSIDE;
+ break;
+ case 4:
+ stereo->type = AV_STEREO3D_TOPBOTTOM;
+ break;
+ case 5:
+ stereo->type = AV_STEREO3D_FRAMESEQUENCE;
+ if (fpa->current_frame_is_frame0_flag)
+ stereo->view = AV_STEREO3D_VIEW_LEFT;
+ else
+ stereo->view = AV_STEREO3D_VIEW_RIGHT;
+ break;
+ case 6:
+ stereo->type = AV_STEREO3D_2D;
+ break;
+ }
+
+ if (fpa->content_interpretation_type == 2)
+ stereo->flags = AV_STEREO3D_FLAG_INVERT;
+ }
+ }
+ return 0;
+}
+
+static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame)
+{
+ GetBitContext gb_payload;
+ uint8_t *sei_buffer;
+ int sei_buffer_index;
+ int ret;
+
+ if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT)
+ return 0;
+
+ sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8);
+ if (!sei_buffer) {
+ av_freep(&sei_buffer);
+ return AVERROR(ENOMEM);
+ }
+ /* remove emulation prevention bytes */
+ sei_buffer_index = 0;
+ for (int i = 0; i < q->payload.NumBit / 8; i++) {
+ if (q->payload.Data[i] == 3)
+ i++;
+ sei_buffer[sei_buffer_index] = q->payload.Data[i];
+ sei_buffer_index += 1;
+ }
+
+ ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1);
+ if (ret < 0) {
+ av_freep(&sei_buffer);
+ return ret;
+ }
+
+ ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx);
+ if (ret < 0) {
+ av_freep(&sei_buffer);
+ return ret;
+ }
+
+ switch (q->payload.Type) {
+ case SEI_TYPE_FRAME_PACKING_ARRANGEMENT:
+ ret = h264_decode_fpa(&q->sei.frame_packing, frame);
+ break;
+ default:
+ break;
+ }
+
+ av_freep(&sei_buffer);
+ return ret;
+}
+
+static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame)
+{
+ mfxU64 ts;
+ mfxStatus sts;
+ int ret = 0;
+
+ if (q->payload.BufSize == 0) {
+ q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE);
+ if (!q->payload.Data)
+ return AVERROR(ENOMEM);
+ q->payload.BufSize = QSV_PAYLOAD_SIZE;
+ }
+
+ sts = MFX_ERR_NONE;
+ while (sts == MFX_ERR_NONE) {
+
+ sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload);
+ if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) {
+ av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough."
+ "Realloc buffer\n");
+ if (q->payload.BufSize >= INT16_MAX / 2)
+ return AVERROR(ENOMEM);
+ q->payload.BufSize = q->payload.BufSize * 2;
+ av_freep(&q->payload.Data);
+ q->payload.Data = av_mallocz(q->payload.BufSize);
+ if (!q->payload.Data)
+ return AVERROR(ENOMEM);
+ continue;
+ } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) {
+ break;
+ }
+
+ switch (avctx->codec_id) {
+ case AV_CODEC_ID_H264:
+ ret = h264_parse_side_data(avctx, q, frame);
+ break;
+ default:
+ break;
+ }
+ if (ret < 0) {
+ av_log(avctx, AV_LOG_WARNING, "parse side data failed\n");
+ break;
+ }
+ }
+ return ret;
+}
+
static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
AVFrame *frame, int *got_frame,
const AVPacket *avpkt)
@@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
outsurf = &aframe.frame->surface;
+ ret = extract_frame_side_data(avctx, q, frame);
+ if (ret == AVERROR_INVALIDDATA)
+ av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n");
+ else if (ret < 0) {
+ av_log(avctx, AV_LOG_ERROR, "Error extracting side data from packet\n");
+ return ret;
+ }
+
frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx->pkt_timebase);
#if QSV_VERSION_ATLEAST(1, 34)
if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
@@ -770,6 +929,7 @@ static void qsv_decode_close_qsvcontext(QSVContext *q)
av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
av_buffer_unref(&q->frames_ctx.mids_buf);
av_buffer_pool_uninit(&q->pool);
+ av_freep(&q->payload.Data);
}
static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,
--
2.32.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 4+ messages in thread* Re: [FFmpeg-devel] [PATCH] libavcodec/qsvdec.c: extract frame packing arrangement data
2022-04-26 10:00 [FFmpeg-devel] [PATCH] libavcodec/qsvdec.c: extract frame packing arrangement data Wenbin Chen
@ 2022-04-27 9:41 ` Xiang, Haihao
2022-04-28 1:37 ` Chen, Wenbin
2022-04-28 1:14 ` Andreas Rheinhardt
1 sibling, 1 reply; 4+ messages in thread
From: Xiang, Haihao @ 2022-04-27 9:41 UTC (permalink / raw)
To: ffmpeg-devel
On Tue, 2022-04-26 at 18:00 +0800, Wenbin Chen wrote:
> Use h264_sei to parse SEI data got from MediaSDK. Extract frame
> packing arrangement information from SEI data and config AVStereo3D
> side data for decoded frame.
>
> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> Signed-off-by: Tong Wu <tong1.wu@intel.com>
> ---
> libavcodec/qsv_internal.h | 2 +
> libavcodec/qsvdec.c | 160 ++++++++++++++++++++++++++++++++++++++
> 2 files changed, 162 insertions(+)
>
> diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
> index e2aecdcbd6..a804c392c1 100644
> --- a/libavcodec/qsv_internal.h
> +++ b/libavcodec/qsv_internal.h
> @@ -54,6 +54,8 @@
>
> #define QSV_MAX_FRAME_EXT_PARAMS 4
>
> +#define QSV_PAYLOAD_SIZE 1024
> +
> #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \
> (MFX_VERSION_MAJOR > (MAJOR) || \
> MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR))
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 5fc5bed4c8..26fa178b4d 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -41,13 +41,16 @@
> #include "libavutil/time.h"
> #include "libavutil/imgutils.h"
> #include "libavutil/film_grain_params.h"
> +#include "libavutil/stereo3d.h"
>
> #include "avcodec.h"
> #include "codec_internal.h"
> #include "internal.h"
> #include "decode.h"
> #include "hwconfig.h"
> +#include "get_bits.h"
> #include "qsv.h"
> +#include "h264_sei.h"
> #include "qsv_internal.h"
>
> static const AVRational mfx_tb = { 1, 90000 };
> @@ -101,6 +104,10 @@ typedef struct QSVContext {
>
> mfxExtBuffer **ext_buffers;
> int nb_ext_buffers;
> +
> + mfxPayload payload;
> + H264SEIContext sei;
> + H264ParamSets ps;
> } QSVContext;
>
> static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext *avctx,
> mfxExtAV1FilmGrainParam
> }
> #endif
>
> +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame)
> +{
> + if (!fpa || !frame)
> + return AVERROR(EINVAL);
> +
> + if (!fpa->arrangement_cancel_flag &&
> + fpa->arrangement_type <= 6 &&
> + fpa->content_interpretation_type > 0 &&
> + fpa->content_interpretation_type < 3) {
> + AVStereo3D *stereo = av_stereo3d_create_side_data(frame);
> + if (stereo) {
> + switch (fpa->arrangement_type) {
> + case 0:
> + stereo->type = AV_STEREO3D_CHECKERBOARD;
> + break;
> + case 1:
> + stereo->type = AV_STEREO3D_COLUMNS;
> + break;
> + case 2:
> + stereo->type = AV_STEREO3D_LINES;
> + break;
> + case 3:
> + if (fpa->quincunx_sampling_flag)
> + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
> + else
> + stereo->type = AV_STEREO3D_SIDEBYSIDE;
> + break;
> + case 4:
> + stereo->type = AV_STEREO3D_TOPBOTTOM;
> + break;
> + case 5:
> + stereo->type = AV_STEREO3D_FRAMESEQUENCE;
> + if (fpa->current_frame_is_frame0_flag)
> + stereo->view = AV_STEREO3D_VIEW_LEFT;
> + else
> + stereo->view = AV_STEREO3D_VIEW_RIGHT;
> + break;
> + case 6:
> + stereo->type = AV_STEREO3D_2D;
> + break;
> + }
> +
> + if (fpa->content_interpretation_type == 2)
> + stereo->flags = AV_STEREO3D_FLAG_INVERT;
> + }
> + }
> + return 0;
> +}
> +
> +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame
> *frame)
> +{
> + GetBitContext gb_payload;
> + uint8_t *sei_buffer;
> + int sei_buffer_index;
> + int ret;
> +
> + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT)
> + return 0;
> +
> + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8);
> + if (!sei_buffer) {
> + av_freep(&sei_buffer);
> + return AVERROR(ENOMEM);
> + }
> + /* remove emulation prevention bytes */
> + sei_buffer_index = 0;
> + for (int i = 0; i < q->payload.NumBit / 8; i++) {
> + if (q->payload.Data[i] == 3)
> + i++;
> + sei_buffer[sei_buffer_index] = q->payload.Data[i];
> + sei_buffer_index += 1;
> + }
> +
> + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1);
> + if (ret < 0) {
> + av_freep(&sei_buffer);
> + return ret;
> + }
> +
> + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx);
> + if (ret < 0) {
> + av_freep(&sei_buffer);
> + return ret;
> + }
> +
> + switch (q->payload.Type) {
> + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT:
> + ret = h264_decode_fpa(&q->sei.frame_packing, frame);
> + break;
> + default:
> + break;
> + }
> +
> + av_freep(&sei_buffer);
> + return ret;
> +}
> +
> +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q,
> AVFrame *frame)
> +{
> + mfxU64 ts;
> + mfxStatus sts;
> + int ret = 0;
> +
> + if (q->payload.BufSize == 0) {
> + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE);
> + if (!q->payload.Data)
> + return AVERROR(ENOMEM);
> + q->payload.BufSize = QSV_PAYLOAD_SIZE;
> + }
> +
> + sts = MFX_ERR_NONE;
> + while (sts == MFX_ERR_NONE) {
> +
> + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload);
> + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough."
> + "Realloc buffer\n");
> + if (q->payload.BufSize >= INT16_MAX / 2)
> + return AVERROR(ENOMEM);
> + q->payload.BufSize = q->payload.BufSize * 2;
> + av_freep(&q->payload.Data);
> + q->payload.Data = av_mallocz(q->payload.BufSize);
> + if (!q->payload.Data)
> + return AVERROR(ENOMEM);
> + continue;
> + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) {
> + break;
> + }
> +
> + switch (avctx->codec_id) {
> + case AV_CODEC_ID_H264:
> + ret = h264_parse_side_data(avctx, q, frame);
> + break;
> + default:
> + break;
> + }
> + if (ret < 0) {
> + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n");
> + break;
> + }
> + }
> + return ret;
> +}
> +
> static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> AVFrame *frame, int *got_frame,
> const AVPacket *avpkt)
> @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
>
> outsurf = &aframe.frame->surface;
>
> + ret = extract_frame_side_data(avctx, q, frame);
> + if (ret == AVERROR_INVALIDDATA)
> + av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n");
> + else if (ret < 0) {
> + av_log(avctx, AV_LOG_ERROR, "Error extracting side data from
> packet\n");
> + return ret;
> + }
> +
The payload got from the SDK is in encoding order, however the frame returned
from MFXVideoDECODE_DecodeFrameAsync() is in display order, see the discussion
about MFXVideoDECODE_GetPayload() in
https://github.com/Intel-Media-SDK/MediaSDK/issues/2597 . I think it would be
better to ask the SDK returns the corresponding info, like what they did in
https://github.com/Intel-Media-SDK/MediaSDK/pull/2726 .
Thanks
Haihao
> frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx-
> >pkt_timebase);
> #if QSV_VERSION_ATLEAST(1, 34)
> if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
> @@ -770,6 +929,7 @@ static void qsv_decode_close_qsvcontext(QSVContext *q)
> av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
> av_buffer_unref(&q->frames_ctx.mids_buf);
> av_buffer_pool_uninit(&q->pool);
> + av_freep(&q->payload.Data);
> }
>
> static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 4+ messages in thread* Re: [FFmpeg-devel] [PATCH] libavcodec/qsvdec.c: extract frame packing arrangement data
2022-04-27 9:41 ` Xiang, Haihao
@ 2022-04-28 1:37 ` Chen, Wenbin
0 siblings, 0 replies; 4+ messages in thread
From: Chen, Wenbin @ 2022-04-28 1:37 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> On Tue, 2022-04-26 at 18:00 +0800, Wenbin Chen wrote:
> > Use h264_sei to parse SEI data got from MediaSDK. Extract frame
> > packing arrangement information from SEI data and config AVStereo3D
> > side data for decoded frame.
> >
> > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> > Signed-off-by: Tong Wu <tong1.wu@intel.com>
> > ---
> > libavcodec/qsv_internal.h | 2 +
> > libavcodec/qsvdec.c | 160
> ++++++++++++++++++++++++++++++++++++++
> > 2 files changed, 162 insertions(+)
> >
> > diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
> > index e2aecdcbd6..a804c392c1 100644
> > --- a/libavcodec/qsv_internal.h
> > +++ b/libavcodec/qsv_internal.h
> > @@ -54,6 +54,8 @@
> >
> > #define QSV_MAX_FRAME_EXT_PARAMS 4
> >
> > +#define QSV_PAYLOAD_SIZE 1024
> > +
> > #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \
> > (MFX_VERSION_MAJOR > (MAJOR) || \
> > MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >=
> (MINOR))
> > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > index 5fc5bed4c8..26fa178b4d 100644
> > --- a/libavcodec/qsvdec.c
> > +++ b/libavcodec/qsvdec.c
> > @@ -41,13 +41,16 @@
> > #include "libavutil/time.h"
> > #include "libavutil/imgutils.h"
> > #include "libavutil/film_grain_params.h"
> > +#include "libavutil/stereo3d.h"
> >
> > #include "avcodec.h"
> > #include "codec_internal.h"
> > #include "internal.h"
> > #include "decode.h"
> > #include "hwconfig.h"
> > +#include "get_bits.h"
> > #include "qsv.h"
> > +#include "h264_sei.h"
> > #include "qsv_internal.h"
> >
> > static const AVRational mfx_tb = { 1, 90000 };
> > @@ -101,6 +104,10 @@ typedef struct QSVContext {
> >
> > mfxExtBuffer **ext_buffers;
> > int nb_ext_buffers;
> > +
> > + mfxPayload payload;
> > + H264SEIContext sei;
> > + H264ParamSets ps;
> > } QSVContext;
> >
> > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext
> *avctx,
> > mfxExtAV1FilmGrainParam
> > }
> > #endif
> >
> > +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame)
> > +{
> > + if (!fpa || !frame)
> > + return AVERROR(EINVAL);
> > +
> > + if (!fpa->arrangement_cancel_flag &&
> > + fpa->arrangement_type <= 6 &&
> > + fpa->content_interpretation_type > 0 &&
> > + fpa->content_interpretation_type < 3) {
> > + AVStereo3D *stereo = av_stereo3d_create_side_data(frame);
> > + if (stereo) {
> > + switch (fpa->arrangement_type) {
> > + case 0:
> > + stereo->type = AV_STEREO3D_CHECKERBOARD;
> > + break;
> > + case 1:
> > + stereo->type = AV_STEREO3D_COLUMNS;
> > + break;
> > + case 2:
> > + stereo->type = AV_STEREO3D_LINES;
> > + break;
> > + case 3:
> > + if (fpa->quincunx_sampling_flag)
> > + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
> > + else
> > + stereo->type = AV_STEREO3D_SIDEBYSIDE;
> > + break;
> > + case 4:
> > + stereo->type = AV_STEREO3D_TOPBOTTOM;
> > + break;
> > + case 5:
> > + stereo->type = AV_STEREO3D_FRAMESEQUENCE;
> > + if (fpa->current_frame_is_frame0_flag)
> > + stereo->view = AV_STEREO3D_VIEW_LEFT;
> > + else
> > + stereo->view = AV_STEREO3D_VIEW_RIGHT;
> > + break;
> > + case 6:
> > + stereo->type = AV_STEREO3D_2D;
> > + break;
> > + }
> > +
> > + if (fpa->content_interpretation_type == 2)
> > + stereo->flags = AV_STEREO3D_FLAG_INVERT;
> > + }
> > + }
> > + return 0;
> > +}
> > +
> > +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q,
> AVFrame
> > *frame)
> > +{
> > + GetBitContext gb_payload;
> > + uint8_t *sei_buffer;
> > + int sei_buffer_index;
> > + int ret;
> > +
> > + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT)
> > + return 0;
> > +
> > + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8);
> > + if (!sei_buffer) {
> > + av_freep(&sei_buffer);
> > + return AVERROR(ENOMEM);
> > + }
> > + /* remove emulation prevention bytes */
> > + sei_buffer_index = 0;
> > + for (int i = 0; i < q->payload.NumBit / 8; i++) {
> > + if (q->payload.Data[i] == 3)
> > + i++;
> > + sei_buffer[sei_buffer_index] = q->payload.Data[i];
> > + sei_buffer_index += 1;
> > + }
> > +
> > + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1);
> > + if (ret < 0) {
> > + av_freep(&sei_buffer);
> > + return ret;
> > + }
> > +
> > + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx);
> > + if (ret < 0) {
> > + av_freep(&sei_buffer);
> > + return ret;
> > + }
> > +
> > + switch (q->payload.Type) {
> > + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT:
> > + ret = h264_decode_fpa(&q->sei.frame_packing, frame);
> > + break;
> > + default:
> > + break;
> > + }
> > +
> > + av_freep(&sei_buffer);
> > + return ret;
> > +}
> > +
> > +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext
> *q,
> > AVFrame *frame)
> > +{
> > + mfxU64 ts;
> > + mfxStatus sts;
> > + int ret = 0;
> > +
> > + if (q->payload.BufSize == 0) {
> > + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE);
> > + if (!q->payload.Data)
> > + return AVERROR(ENOMEM);
> > + q->payload.BufSize = QSV_PAYLOAD_SIZE;
> > + }
> > +
> > + sts = MFX_ERR_NONE;
> > + while (sts == MFX_ERR_NONE) {
> > +
> > + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload);
> > + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough."
> > + "Realloc buffer\n");
> > + if (q->payload.BufSize >= INT16_MAX / 2)
> > + return AVERROR(ENOMEM);
> > + q->payload.BufSize = q->payload.BufSize * 2;
> > + av_freep(&q->payload.Data);
> > + q->payload.Data = av_mallocz(q->payload.BufSize);
> > + if (!q->payload.Data)
> > + return AVERROR(ENOMEM);
> > + continue;
> > + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) {
> > + break;
> > + }
> > +
> > + switch (avctx->codec_id) {
> > + case AV_CODEC_ID_H264:
> > + ret = h264_parse_side_data(avctx, q, frame);
> > + break;
> > + default:
> > + break;
> > + }
> > + if (ret < 0) {
> > + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n");
> > + break;
> > + }
> > + }
> > + return ret;
> > +}
> > +
> > static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> > AVFrame *frame, int *got_frame,
> > const AVPacket *avpkt)
> > @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx,
> QSVContext
> > *q,
> >
> > outsurf = &aframe.frame->surface;
> >
> > + ret = extract_frame_side_data(avctx, q, frame);
> > + if (ret == AVERROR_INVALIDDATA)
> > + av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n");
> > + else if (ret < 0) {
> > + av_log(avctx, AV_LOG_ERROR, "Error extracting side data from
> > packet\n");
> > + return ret;
> > + }
> > +
>
> The payload got from the SDK is in encoding order, however the frame
> returned
> from MFXVideoDECODE_DecodeFrameAsync() is in display order, see the
> discussion
> about MFXVideoDECODE_GetPayload() in
> https://github.com/Intel-Media-SDK/MediaSDK/issues/2597 . I think it would
> be
> better to ask the SDK returns the corresponding info, like what they did in
> https://github.com/Intel-Media-SDK/MediaSDK/pull/2726 .
>
> Thanks
> Haihao
Thanks for your information. I will try to fix this problem.
Thanks
Wenbin
>
>
>
> > frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx-
> > >pkt_timebase);
> > #if QSV_VERSION_ATLEAST(1, 34)
> > if ((avctx->export_side_data &
> AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
> > @@ -770,6 +929,7 @@ static void
> qsv_decode_close_qsvcontext(QSVContext *q)
> > av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
> > av_buffer_unref(&q->frames_ctx.mids_buf);
> > av_buffer_pool_uninit(&q->pool);
> > + av_freep(&q->payload.Data);
> > }
> >
> > static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [FFmpeg-devel] [PATCH] libavcodec/qsvdec.c: extract frame packing arrangement data
2022-04-26 10:00 [FFmpeg-devel] [PATCH] libavcodec/qsvdec.c: extract frame packing arrangement data Wenbin Chen
2022-04-27 9:41 ` Xiang, Haihao
@ 2022-04-28 1:14 ` Andreas Rheinhardt
1 sibling, 0 replies; 4+ messages in thread
From: Andreas Rheinhardt @ 2022-04-28 1:14 UTC (permalink / raw)
To: ffmpeg-devel
Wenbin Chen:
> Use h264_sei to parse SEI data got from MediaSDK. Extract frame
> packing arrangement information from SEI data and config AVStereo3D
> side data for decoded frame.
>
> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> Signed-off-by: Tong Wu <tong1.wu@intel.com>
> ---
> libavcodec/qsv_internal.h | 2 +
> libavcodec/qsvdec.c | 160 ++++++++++++++++++++++++++++++++++++++
> 2 files changed, 162 insertions(+)
>
> diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
> index e2aecdcbd6..a804c392c1 100644
> --- a/libavcodec/qsv_internal.h
> +++ b/libavcodec/qsv_internal.h
> @@ -54,6 +54,8 @@
>
> #define QSV_MAX_FRAME_EXT_PARAMS 4
>
> +#define QSV_PAYLOAD_SIZE 1024
> +
> #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \
> (MFX_VERSION_MAJOR > (MAJOR) || \
> MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR))
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 5fc5bed4c8..26fa178b4d 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -41,13 +41,16 @@
> #include "libavutil/time.h"
> #include "libavutil/imgutils.h"
> #include "libavutil/film_grain_params.h"
> +#include "libavutil/stereo3d.h"
>
> #include "avcodec.h"
> #include "codec_internal.h"
> #include "internal.h"
> #include "decode.h"
> #include "hwconfig.h"
> +#include "get_bits.h"
> #include "qsv.h"
> +#include "h264_sei.h"
> #include "qsv_internal.h"
>
> static const AVRational mfx_tb = { 1, 90000 };
> @@ -101,6 +104,10 @@ typedef struct QSVContext {
>
> mfxExtBuffer **ext_buffers;
> int nb_ext_buffers;
> +
> + mfxPayload payload;
> + H264SEIContext sei;
> + H264ParamSets ps;
> } QSVContext;
>
> static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
> }
> #endif
>
> +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame)
> +{
> + if (!fpa || !frame)
> + return AVERROR(EINVAL);
> +
> + if (!fpa->arrangement_cancel_flag &&
> + fpa->arrangement_type <= 6 &&
> + fpa->content_interpretation_type > 0 &&
> + fpa->content_interpretation_type < 3) {
> + AVStereo3D *stereo = av_stereo3d_create_side_data(frame);
> + if (stereo) {
> + switch (fpa->arrangement_type) {
> + case 0:
> + stereo->type = AV_STEREO3D_CHECKERBOARD;
> + break;
> + case 1:
> + stereo->type = AV_STEREO3D_COLUMNS;
> + break;
> + case 2:
> + stereo->type = AV_STEREO3D_LINES;
> + break;
> + case 3:
> + if (fpa->quincunx_sampling_flag)
> + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
> + else
> + stereo->type = AV_STEREO3D_SIDEBYSIDE;
> + break;
> + case 4:
> + stereo->type = AV_STEREO3D_TOPBOTTOM;
> + break;
> + case 5:
> + stereo->type = AV_STEREO3D_FRAMESEQUENCE;
> + if (fpa->current_frame_is_frame0_flag)
> + stereo->view = AV_STEREO3D_VIEW_LEFT;
> + else
> + stereo->view = AV_STEREO3D_VIEW_RIGHT;
> + break;
> + case 6:
> + stereo->type = AV_STEREO3D_2D;
> + break;
> + }
> +
> + if (fpa->content_interpretation_type == 2)
> + stereo->flags = AV_STEREO3D_FLAG_INVERT;
> + }
> + }
> + return 0;
> +}
> +
> +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame)
> +{
> + GetBitContext gb_payload;
> + uint8_t *sei_buffer;
> + int sei_buffer_index;
> + int ret;
> +
> + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT)
> + return 0;
> +
> + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8);
> + if (!sei_buffer) {
> + av_freep(&sei_buffer);
> + return AVERROR(ENOMEM);
> + }
> + /* remove emulation prevention bytes */
> + sei_buffer_index = 0;
> + for (int i = 0; i < q->payload.NumBit / 8; i++) {
> + if (q->payload.Data[i] == 3)
> + i++;
> + sei_buffer[sei_buffer_index] = q->payload.Data[i];
> + sei_buffer_index += 1;
Not every 0x03 is an emulation prevention byte.
> + }
> +
> + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1);
Buffers used with the get-bits API need to be padded. Best to use
av_fast_padded_malloc and ff_h2645_extract_rbsp.
> + if (ret < 0) {
> + av_freep(&sei_buffer);
> + return ret;
> + }
> +
> + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx);
> + if (ret < 0) {
> + av_freep(&sei_buffer);
> + return ret;
> + }
> +
> + switch (q->payload.Type) {
> + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT:
> + ret = h264_decode_fpa(&q->sei.frame_packing, frame);
> + break;
> + default:
> + break;
> + }
> +
> + av_freep(&sei_buffer);
> + return ret;
> +}
> +
> +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame)
> +{
> + mfxU64 ts;
> + mfxStatus sts;
> + int ret = 0;
> +
> + if (q->payload.BufSize == 0) {
> + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE);
> + if (!q->payload.Data)
> + return AVERROR(ENOMEM);
> + q->payload.BufSize = QSV_PAYLOAD_SIZE;
> + }
> +
> + sts = MFX_ERR_NONE;
> + while (sts == MFX_ERR_NONE) {
> +
> + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload);
> + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough."
> + "Realloc buffer\n");
Why this warning? Why a log-message at all?
> + if (q->payload.BufSize >= INT16_MAX / 2)
Where does this limit come from?
> + return AVERROR(ENOMEM);
> + q->payload.BufSize = q->payload.BufSize * 2;
> + av_freep(&q->payload.Data);
> + q->payload.Data = av_mallocz(q->payload.BufSize);
> + if (!q->payload.Data)
> + return AVERROR(ENOMEM);
q->payload.Data is NULL if this happens, but q->payload.BufSize will be
> 0, so that q->payload used in the next MFXVideoDECODE_GetPayload call
will be inconsistent.
> + continue;
> + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) {
> + break;
> + }
> +
> + switch (avctx->codec_id) {
> + case AV_CODEC_ID_H264:
> + ret = h264_parse_side_data(avctx, q, frame);
> + break;
> + default:
> + break;
> + }
> + if (ret < 0) {
> + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n");
> + break;
> + }
> + }
> + return ret;
> +}
> +
> static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> AVFrame *frame, int *got_frame,
> const AVPacket *avpkt)
> @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>
> outsurf = &aframe.frame->surface;
>
> + ret = extract_frame_side_data(avctx, q, frame);
> + if (ret == AVERROR_INVALIDDATA)
> + av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n");
> + else if (ret < 0) {
> + av_log(avctx, AV_LOG_ERROR, "Error extracting side data from packet\n");
> + return ret;
> + }
> +
> frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx->pkt_timebase);
> #if QSV_VERSION_ATLEAST(1, 34)
> if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
> @@ -770,6 +929,7 @@ static void qsv_decode_close_qsvcontext(QSVContext *q)
> av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
> av_buffer_unref(&q->frames_ctx.mids_buf);
> av_buffer_pool_uninit(&q->pool);
> + av_freep(&q->payload.Data);
> }
>
> static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2022-04-28 1:37 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-04-26 10:00 [FFmpeg-devel] [PATCH] libavcodec/qsvdec.c: extract frame packing arrangement data Wenbin Chen
2022-04-27 9:41 ` Xiang, Haihao
2022-04-28 1:37 ` Chen, Wenbin
2022-04-28 1:14 ` Andreas Rheinhardt
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git