* [FFmpeg-devel] [PATCH 2/2] avcodec/mfenc: add support for D3D11 input surfaces
@ 2025-05-22 13:21 Dash Santosh Sathyanarayanan
0 siblings, 0 replies; only message in thread
From: Dash Santosh Sathyanarayanan @ 2025-05-22 13:21 UTC (permalink / raw)
To: FFmpeg development discussions and patches
[-- Attachment #1: Type: text/plain, Size: 13203 bytes --]
Adds D3D11 input surface support to the MediaFoundation encoder (mfenc),
allowing direct encoding of GPU frames without readback to system memory.
This improves performance and compatibility when used alongside scale_d3d11.
---
Changelog | 1 +
libavcodec/mf_utils.h | 7 ++
libavcodec/mfenc.c | 204 ++++++++++++++++++++++++++++++++++++------
3 files changed, 183 insertions(+), 29 deletions(-)
diff --git a/Changelog b/Changelog
index 68610a63d0..194f09121f 100644
--- a/Changelog
+++ b/Changelog
@@ -19,6 +19,7 @@ version <next>:
- VVC decoder supports all content of SCC (Screen Content Coding):
IBC (Inter Block Copy), Palette Mode and ACT (Adaptive Color Transform
- vf_scale_d3d11 filter
+- mfenc supports d3d11 input surfaces
version 7.1:
diff --git a/libavcodec/mf_utils.h b/libavcodec/mf_utils.h
index a59b36d015..ecebb6fcdf 100644
--- a/libavcodec/mf_utils.h
+++ b/libavcodec/mf_utils.h
@@ -53,6 +53,13 @@ typedef struct MFFunctions {
IMFMediaBuffer **ppBuffer);
HRESULT (WINAPI *MFCreateSample) (IMFSample **ppIMFSample);
HRESULT (WINAPI *MFCreateMediaType) (IMFMediaType **ppMFType);
+ HRESULT (WINAPI *MFCreateDXGISurfaceBuffer) (REFIID riid,
+ IUnknown* punkSurface,
+ UINT uSubresourceIndex,
+ BOOL fBottomUpWhenLinear,
+ IMFMediaBuffer** ppBuffer);
+ HRESULT (WINAPI *MFCreateDXGIDeviceManager) (UINT* resetToken,
+ IMFDXGIDeviceManager** ppDeviceManager);
// MFTEnumEx is missing in Windows Vista's mfplat.dll.
HRESULT (WINAPI *MFTEnumEx)(GUID guidCategory, UINT32 Flags,
const MFT_REGISTER_TYPE_INFO *pInputType,
diff --git a/libavcodec/mfenc.c b/libavcodec/mfenc.c
index c9e2191fde..7ddf918c9a 100644
--- a/libavcodec/mfenc.c
+++ b/libavcodec/mfenc.c
@@ -31,10 +31,17 @@
#include "codec_internal.h"
#include "internal.h"
#include "compat/w32dlfcn.h"
+#if CONFIG_D3D11VA
+#include "libavutil/hwcontext_d3d11va.h"
+#endif
typedef struct MFContext {
AVClass *av_class;
HMODULE library;
+ HMODULE d3d_dll;
+ IMFDXGIDeviceManager *dxgiManager;
+ int resetToken;
+
MFFunctions functions;
AVFrame *frame;
int is_video, is_audio;
@@ -47,6 +54,7 @@ typedef struct MFContext {
int out_stream_provides_samples;
int draining, draining_done;
int sample_sent;
+ int stream_started;
int async_need_input, async_have_output, async_marker;
int64_t reorder_delay;
ICodecAPI *codec_api;
@@ -55,6 +63,7 @@ typedef struct MFContext {
int opt_enc_quality;
int opt_enc_scenario;
int opt_enc_hw;
+ AVD3D11VADeviceContext* device_hwctx;
} MFContext;
static int mf_choose_output_type(AVCodecContext *avctx);
@@ -303,36 +312,118 @@ static IMFSample *mf_a_avframe_to_sample(AVCodecContext *avctx, const AVFrame *f
return sample;
}
-static IMFSample *mf_v_avframe_to_sample(AVCodecContext *avctx, const AVFrame *frame)
+static int initialize_dxgi_manager(AVCodecContext *avctx)
{
MFContext *c = avctx->priv_data;
- IMFSample *sample;
- IMFMediaBuffer *buffer;
- BYTE *data;
+ MFFunctions *func = &c->functions;
HRESULT hr;
- int ret;
- int size;
+
+ hr = func->MFCreateDXGIDeviceManager(&c->resetToken, &c->dxgiManager);
+ if (FAILED(hr)) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to create DXGI device manager: %s\n", ff_hr_str(hr));
+ return AVERROR_EXTERNAL;
+ }
+
+ hr = IMFDXGIDeviceManager_ResetDevice(c->dxgiManager, c->device_hwctx->device, c->resetToken);
+ if (FAILED(hr)) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to reset device: %s\n", ff_hr_str(hr));
+ return AVERROR_EXTERNAL;
+ }
+
+ hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_SET_D3D_MANAGER, (ULONG_PTR)c->dxgiManager);
+ if (FAILED(hr)) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to set D3D manager: %s\n", ff_hr_str(hr));
+ return AVERROR_EXTERNAL;
+ }
+
+ return 0;
+}
+
+static int process_d3d11_frame(AVCodecContext *avctx, const AVFrame *frame, IMFSample **out_sample)
+{
+ MFContext *c = avctx->priv_data;
+ MFFunctions *func = &c->functions;
+ AVHWFramesContext *frames_ctx = NULL;
+ ID3D11Texture2D *d3d11_texture = NULL;
+ IMFSample *sample = NULL;
+ IMFMediaBuffer *buffer = NULL;
+ int subIdx = 0;
+ HRESULT hr;
+
+ frames_ctx = (AVHWFramesContext*)frame->hw_frames_ctx->data;
+ c->device_hwctx = (AVD3D11VADeviceContext*)frames_ctx->device_ctx->hwctx;
+
+ if (!c->dxgiManager) {
+ hr = initialize_dxgi_manager(avctx);
+ if (FAILED(hr)) {
+ return AVERROR_EXTERNAL;
+ }
+ }
+
+ d3d11_texture = (ID3D11Texture2D*)frame->data[0];
+ subIdx = (int)(intptr_t)frame->data[1];
+
+ if (!d3d11_texture) {
+ av_log(avctx, AV_LOG_ERROR, "D3D11 texture not found\n");
+ return AVERROR(EINVAL);
+ }
+
+ hr = func->MFCreateSample(&sample);
+ if (FAILED(hr)) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to create MF sample: %s\n", ff_hr_str(hr));
+ return AVERROR_EXTERNAL;
+ }
+
+ hr = func->MFCreateDXGISurfaceBuffer(&IID_ID3D11Texture2D, d3d11_texture, subIdx, 0, &buffer);
+ if (FAILED(hr)) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to create DXGI surface buffer: %s\n", ff_hr_str(hr));
+ IMFSample_Release(sample);
+ return AVERROR_EXTERNAL;
+ }
+
+ hr = IMFSample_AddBuffer(sample, buffer);
+ if (FAILED(hr)) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to add buffer to sample: %s\n", ff_hr_str(hr));
+ IMFMediaBuffer_Release(buffer);
+ IMFSample_Release(sample);
+ return AVERROR_EXTERNAL;
+ }
+
+ IMFMediaBuffer_Release(buffer);
+
+ *out_sample = sample;
+ return 0;
+}
+
+static int process_software_frame(AVCodecContext *avctx, const AVFrame *frame, IMFSample **out_sample)
+{
+ MFContext *c = avctx->priv_data;
+ IMFSample *sample = NULL;
+ IMFMediaBuffer *buffer = NULL;
+ BYTE *data = NULL;
+ HRESULT hr;
+ int size, ret;
size = av_image_get_buffer_size(avctx->pix_fmt, avctx->width, avctx->height, 1);
if (size < 0)
- return NULL;
+ return AVERROR_EXTERNAL;
sample = ff_create_memory_sample(&c->functions, NULL, size,
c->in_info.cbAlignment);
if (!sample)
- return NULL;
+ return AVERROR_EXTERNAL;
hr = IMFSample_GetBufferByIndex(sample, 0, &buffer);
if (FAILED(hr)) {
IMFSample_Release(sample);
- return NULL;
+ return AVERROR_EXTERNAL;
}
hr = IMFMediaBuffer_Lock(buffer, &data, NULL, NULL);
if (FAILED(hr)) {
IMFMediaBuffer_Release(buffer);
IMFSample_Release(sample);
- return NULL;
+ return AVERROR_EXTERNAL;
}
ret = av_image_copy_to_buffer((uint8_t *)data, size, (void *)frame->data, frame->linesize,
@@ -342,10 +433,43 @@ static IMFSample *mf_v_avframe_to_sample(AVCodecContext *avctx, const AVFrame *f
IMFMediaBuffer_Release(buffer);
if (ret < 0) {
IMFSample_Release(sample);
- return NULL;
+ return AVERROR_EXTERNAL;
}
IMFSample_SetSampleDuration(sample, mf_to_mf_time(avctx, frame->duration));
+ *out_sample = sample;
+
+ return 0;
+}
+
+static IMFSample *mf_v_avframe_to_sample(AVCodecContext *avctx, const AVFrame *frame)
+{
+ MFContext *c = avctx->priv_data;
+ MFFunctions *func = &c->functions;
+ IMFSample *sample = NULL;
+ IMFMediaBuffer *buffer = NULL;
+ HRESULT hr;
+ int ret;
+
+ if (frame->format == AV_PIX_FMT_D3D11) {
+ // Handle D3D11 hardware frames
+ ret = process_d3d11_frame(avctx, frame, &sample);
+ if (ret < 0) {
+ return NULL;
+ }
+ } else {
+ // Handle software frames
+ ret = process_software_frame(avctx, frame, &sample);
+ if (ret < 0) {
+ return NULL;
+ }
+ }
+
+ // Set sample duration
+ hr = IMFSample_SetSampleDuration(sample, mf_to_mf_time(avctx, frame->duration));
+ if (FAILED(hr)) {
+ av_log(avctx, AV_LOG_WARNING, "Failed to set sample duration: %s\n", ff_hr_str(hr));
+ }
return sample;
}
@@ -511,6 +635,22 @@ static int mf_receive_packet(AVCodecContext *avctx, AVPacket *avpkt)
}
}
+ if (!c->stream_started) {
+ HRESULT hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
+ if (FAILED(hr)) {
+ av_log(avctx, AV_LOG_ERROR, "could not start streaming (%s)\n", ff_hr_str(hr));
+ return AVERROR(EBADMSG);
+ }
+
+ hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
+ if (FAILED(hr)) {
+ av_log(avctx, AV_LOG_ERROR, "could not start stream (%s)\n", ff_hr_str(hr));
+ return AVERROR(EBADMSG);
+ }
+
+ c->stream_started = 1;
+ }
+
ret = mf_send_sample(avctx, sample);
if (sample)
IMFSample_Release(sample);
@@ -727,8 +867,15 @@ static int mf_encv_output_adjust(AVCodecContext *avctx, IMFMediaType *type)
static int64_t mf_encv_input_score(AVCodecContext *avctx, IMFMediaType *type)
{
enum AVPixelFormat pix_fmt = ff_media_type_to_pix_fmt((IMFAttributes *)type);
- if (pix_fmt != avctx->pix_fmt)
- return -1; // can not use
+
+ if (avctx->pix_fmt == AV_PIX_FMT_D3D11) {
+ if (pix_fmt != AV_PIX_FMT_NV12) {
+ return -1; // can not use
+ }
+ } else {
+ if (pix_fmt != avctx->pix_fmt)
+ return -1; // can not use
+ }
return 0;
}
@@ -736,9 +883,16 @@ static int64_t mf_encv_input_score(AVCodecContext *avctx, IMFMediaType *type)
static int mf_encv_input_adjust(AVCodecContext *avctx, IMFMediaType *type)
{
enum AVPixelFormat pix_fmt = ff_media_type_to_pix_fmt((IMFAttributes *)type);
- if (pix_fmt != avctx->pix_fmt) {
- av_log(avctx, AV_LOG_ERROR, "unsupported input pixel format set\n");
- return AVERROR(EINVAL);
+ if (avctx->pix_fmt == AV_PIX_FMT_D3D11) {
+ if (pix_fmt != AV_PIX_FMT_NV12 && pix_fmt != AV_PIX_FMT_D3D11) {
+ av_log(avctx, AV_LOG_ERROR, "unsupported input pixel format set\n");
+ return AVERROR(EINVAL);
+ }
+ } else {
+ if (pix_fmt != avctx->pix_fmt) {
+ av_log(avctx, AV_LOG_ERROR, "unsupported input pixel format set\n");
+ return AVERROR(EINVAL);
+ }
}
//ff_MFSetAttributeSize((IMFAttributes *)type, &MF_MT_FRAME_SIZE, avctx->width, avctx->height);
@@ -1106,18 +1260,6 @@ static int mf_init_encoder(AVCodecContext *avctx)
if ((ret = mf_setup_context(avctx)) < 0)
return ret;
- hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
- if (FAILED(hr)) {
- av_log(avctx, AV_LOG_ERROR, "could not start streaming (%s)\n", ff_hr_str(hr));
- return AVERROR_EXTERNAL;
- }
-
- hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
- if (FAILED(hr)) {
- av_log(avctx, AV_LOG_ERROR, "could not start stream (%s)\n", ff_hr_str(hr));
- return AVERROR_EXTERNAL;
- }
-
if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER && c->async_events &&
c->is_video && !avctx->extradata) {
int sleep = 10000, total = 0;
@@ -1175,6 +1317,7 @@ static int mf_load_library(AVCodecContext *avctx)
#if !HAVE_UWP
c->library = dlopen("mfplat.dll", 0);
+ c->d3d_dll = dlopen("D3D11.dll", 0);
if (!c->library) {
av_log(c, AV_LOG_ERROR, "DLL mfplat.dll failed to open\n");
@@ -1187,6 +1330,8 @@ static int mf_load_library(AVCodecContext *avctx)
LOAD_MF_FUNCTION(c, MFCreateAlignedMemoryBuffer);
LOAD_MF_FUNCTION(c, MFCreateSample);
LOAD_MF_FUNCTION(c, MFCreateMediaType);
+ LOAD_MF_FUNCTION(c, MFCreateDXGISurfaceBuffer);
+ LOAD_MF_FUNCTION(c, MFCreateDXGIDeviceManager);
// MFTEnumEx is missing in Windows Vista's mfplat.dll.
LOAD_MF_FUNCTION(c, MFTEnumEx);
@@ -1208,6 +1353,7 @@ static int mf_close(AVCodecContext *avctx)
ff_free_mf(&c->functions, &c->mft);
dlclose(c->library);
+ dlclose(c->d3d_dll);
c->library = NULL;
#else
ff_free_mf(&c->functions, &c->mft);
@@ -1300,7 +1446,7 @@ static const FFCodecDefault defaults[] = {
};
#define VFMTS \
- CODEC_PIXFMTS(AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P),
+ CODEC_PIXFMTS(AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P, AV_PIX_FMT_D3D11),
#define VCAPS \
.p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HYBRID | \
AV_CODEC_CAP_DR1,
--
2.34.1
[-- Attachment #2: 0002-avcodec-mfenc-add-support-for-D3D11-input-surfaces.patch --]
[-- Type: application/octet-stream, Size: 13081 bytes --]
[-- Attachment #3: Type: text/plain, Size: 251 bytes --]
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2025-05-22 13:22 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2025-05-22 13:21 [FFmpeg-devel] [PATCH 2/2] avcodec/mfenc: add support for D3D11 input surfaces Dash Santosh Sathyanarayanan
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git