From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by master.gitmailbox.com (Postfix) with ESMTP id 7612B42B6E for ; Tue, 26 Apr 2022 10:01:09 +0000 (UTC) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 01D5E68B2F7; Tue, 26 Apr 2022 13:01:06 +0300 (EEST) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id E942168AACC for ; Tue, 26 Apr 2022 13:00:59 +0300 (EEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1650967265; x=1682503265; h=from:to:subject:date:message-id:mime-version: content-transfer-encoding; bh=JCWvT+wdLF9eR6O5TAoHT9vFhbnFbPGYNhSUe3VKudk=; b=aYpxBI9lKcKacdEK3vWeAcGirDZ3k74Tkypv6JV+tPqZaCNmG1Xk99eo LuMFx7fMMdOA7GWptCqZdza424GIQXQEFparpS/FTlyPwxnEfLmjtXAu4 ePgp12qZR+X/2faSswWg5KytRvIp4XIFvkw5wJVIqNnI4sVKr8Hzkb4AG 5EEZXgZXe/6xJis931seqSiw6HHa4sr0rqyPuTOVof6zjSFoFClq9vPzi hFj6QkEk4skZxjimVfG258PkJDW1U3KCB8/xjKX1WWBY8ZC4On4EdF9b6 v5uv70e0jkXzRx/vfZvtIN06R47PWVSY07O9A79WVzLa5gHBxRpJCaLqG w==; X-IronPort-AV: E=McAfee;i="6400,9594,10328"; a="328467780" X-IronPort-AV: E=Sophos;i="5.90,290,1643702400"; d="scan'208";a="328467780" Received: from orsmga008.jf.intel.com ([10.7.209.65]) by orsmga105.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 26 Apr 2022 03:00:56 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.90,290,1643702400"; d="scan'208";a="579806599" Received: from wenbin-z390-aorus-ultra.sh.intel.com ([10.239.35.4]) by orsmga008.jf.intel.com with ESMTP; 26 Apr 2022 03:00:55 -0700 From: Wenbin Chen To: ffmpeg-devel@ffmpeg.org Date: Tue, 26 Apr 2022 18:00:35 +0800 Message-Id: <20220426100035.2026406-1-wenbin.chen@intel.com> X-Mailer: git-send-email 2.32.0 MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH] libavcodec/qsvdec.c: extract frame packing arrangement data X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" Archived-At: List-Archive: List-Post: Use h264_sei to parse SEI data got from MediaSDK. Extract frame packing arrangement information from SEI data and config AVStereo3D side data for decoded frame. Signed-off-by: Wenbin Chen Signed-off-by: Tong Wu --- libavcodec/qsv_internal.h | 2 + libavcodec/qsvdec.c | 160 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h index e2aecdcbd6..a804c392c1 100644 --- a/libavcodec/qsv_internal.h +++ b/libavcodec/qsv_internal.h @@ -54,6 +54,8 @@ #define QSV_MAX_FRAME_EXT_PARAMS 4 +#define QSV_PAYLOAD_SIZE 1024 + #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \ (MFX_VERSION_MAJOR > (MAJOR) || \ MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR)) diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c index 5fc5bed4c8..26fa178b4d 100644 --- a/libavcodec/qsvdec.c +++ b/libavcodec/qsvdec.c @@ -41,13 +41,16 @@ #include "libavutil/time.h" #include "libavutil/imgutils.h" #include "libavutil/film_grain_params.h" +#include "libavutil/stereo3d.h" #include "avcodec.h" #include "codec_internal.h" #include "internal.h" #include "decode.h" #include "hwconfig.h" +#include "get_bits.h" #include "qsv.h" +#include "h264_sei.h" #include "qsv_internal.h" static const AVRational mfx_tb = { 1, 90000 }; @@ -101,6 +104,10 @@ typedef struct QSVContext { mfxExtBuffer **ext_buffers; int nb_ext_buffers; + + mfxPayload payload; + H264SEIContext sei; + H264ParamSets ps; } QSVContext; static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam } #endif +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame) +{ + if (!fpa || !frame) + return AVERROR(EINVAL); + + if (!fpa->arrangement_cancel_flag && + fpa->arrangement_type <= 6 && + fpa->content_interpretation_type > 0 && + fpa->content_interpretation_type < 3) { + AVStereo3D *stereo = av_stereo3d_create_side_data(frame); + if (stereo) { + switch (fpa->arrangement_type) { + case 0: + stereo->type = AV_STEREO3D_CHECKERBOARD; + break; + case 1: + stereo->type = AV_STEREO3D_COLUMNS; + break; + case 2: + stereo->type = AV_STEREO3D_LINES; + break; + case 3: + if (fpa->quincunx_sampling_flag) + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX; + else + stereo->type = AV_STEREO3D_SIDEBYSIDE; + break; + case 4: + stereo->type = AV_STEREO3D_TOPBOTTOM; + break; + case 5: + stereo->type = AV_STEREO3D_FRAMESEQUENCE; + if (fpa->current_frame_is_frame0_flag) + stereo->view = AV_STEREO3D_VIEW_LEFT; + else + stereo->view = AV_STEREO3D_VIEW_RIGHT; + break; + case 6: + stereo->type = AV_STEREO3D_2D; + break; + } + + if (fpa->content_interpretation_type == 2) + stereo->flags = AV_STEREO3D_FLAG_INVERT; + } + } + return 0; +} + +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) +{ + GetBitContext gb_payload; + uint8_t *sei_buffer; + int sei_buffer_index; + int ret; + + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT) + return 0; + + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8); + if (!sei_buffer) { + av_freep(&sei_buffer); + return AVERROR(ENOMEM); + } + /* remove emulation prevention bytes */ + sei_buffer_index = 0; + for (int i = 0; i < q->payload.NumBit / 8; i++) { + if (q->payload.Data[i] == 3) + i++; + sei_buffer[sei_buffer_index] = q->payload.Data[i]; + sei_buffer_index += 1; + } + + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1); + if (ret < 0) { + av_freep(&sei_buffer); + return ret; + } + + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx); + if (ret < 0) { + av_freep(&sei_buffer); + return ret; + } + + switch (q->payload.Type) { + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT: + ret = h264_decode_fpa(&q->sei.frame_packing, frame); + break; + default: + break; + } + + av_freep(&sei_buffer); + return ret; +} + +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) +{ + mfxU64 ts; + mfxStatus sts; + int ret = 0; + + if (q->payload.BufSize == 0) { + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE); + if (!q->payload.Data) + return AVERROR(ENOMEM); + q->payload.BufSize = QSV_PAYLOAD_SIZE; + } + + sts = MFX_ERR_NONE; + while (sts == MFX_ERR_NONE) { + + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload); + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) { + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough." + "Realloc buffer\n"); + if (q->payload.BufSize >= INT16_MAX / 2) + return AVERROR(ENOMEM); + q->payload.BufSize = q->payload.BufSize * 2; + av_freep(&q->payload.Data); + q->payload.Data = av_mallocz(q->payload.BufSize); + if (!q->payload.Data) + return AVERROR(ENOMEM); + continue; + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) { + break; + } + + switch (avctx->codec_id) { + case AV_CODEC_ID_H264: + ret = h264_parse_side_data(avctx, q, frame); + break; + default: + break; + } + if (ret < 0) { + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n"); + break; + } + } + return ret; +} + static int qsv_decode(AVCodecContext *avctx, QSVContext *q, AVFrame *frame, int *got_frame, const AVPacket *avpkt) @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q, outsurf = &aframe.frame->surface; + ret = extract_frame_side_data(avctx, q, frame); + if (ret == AVERROR_INVALIDDATA) + av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n"); + else if (ret < 0) { + av_log(avctx, AV_LOG_ERROR, "Error extracting side data from packet\n"); + return ret; + } + frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx->pkt_timebase); #if QSV_VERSION_ATLEAST(1, 34) if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) && @@ -770,6 +929,7 @@ static void qsv_decode_close_qsvcontext(QSVContext *q) av_buffer_unref(&q->frames_ctx.hw_frames_ctx); av_buffer_unref(&q->frames_ctx.mids_buf); av_buffer_pool_uninit(&q->pool); + av_freep(&q->payload.Data); } static int qsv_process_data(AVCodecContext *avctx, QSVContext *q, -- 2.32.0 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".