Re: [FFmpeg-devel] [PATCH v2 1/8] avutil/hwcontext: Add hwdevice type for V4L2 Request API

[Lynne via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

[-- Attachment #1.1.1.1: Type: text/plain, Size: 16520 bytes --]

On 06/08/2024 11:06, Jonas Karlman wrote:
> Add a hwdevice type for V4L2 Request API with transfer_data_from support
> for AV_PIX_FMT_DRM_PRIME, based on AV_HWDEVICE_TYPE_DRM.
> 
> AVV4L2RequestDeviceContext.media_fd can be set by the application or a
> media device path can be supplied when hwdevice is created. When none
> is supplied it default to -1 and hwaccel will auto-detect a media device
> with a capable video device.
> 
> Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
> ---
>   configure                         |   7 +
>   libavutil/Makefile                |   3 +
>   libavutil/hwcontext.c             |   4 +
>   libavutil/hwcontext.h             |   1 +
>   libavutil/hwcontext_internal.h    |   1 +
>   libavutil/hwcontext_v4l2request.c | 261 ++++++++++++++++++++++++++++++
>   libavutil/hwcontext_v4l2request.h |  41 +++++
>   7 files changed, 318 insertions(+)
>   create mode 100644 libavutil/hwcontext_v4l2request.c
>   create mode 100644 libavutil/hwcontext_v4l2request.h
> 
> diff --git a/configure b/configure
> index 37178d7d81..23d00edc48 100755
> --- a/configure
> +++ b/configure
> @@ -358,6 +358,7 @@ External library support:
>     --enable-omx-rpi         enable OpenMAX IL code for Raspberry Pi [no]
>     --enable-rkmpp           enable Rockchip Media Process Platform code [no]
>     --disable-v4l2-m2m       disable V4L2 mem2mem code [autodetect]
> +  --enable-v4l2-request    enable V4L2 Request API code [no]
>     --disable-vaapi          disable Video Acceleration API (mainly Unix/Intel) code [autodetect]
>     --disable-vdpau          disable Nvidia Video Decode and Presentation API for Unix code [autodetect]
>     --disable-videotoolbox   disable VideoToolbox code [autodetect]
> @@ -2023,6 +2024,7 @@ HWACCEL_LIBRARY_LIST="
>       mmal
>       omx
>       opencl
> +    v4l2_request
>   "
>   
>   DOCUMENT_LIST="
> @@ -3148,6 +3150,7 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32"
>   ffnvcodec_deps_any="libdl LoadLibrary"
>   mediacodec_deps="android mediandk"
>   nvdec_deps="ffnvcodec"
> +v4l2_request_deps="linux_media_h v4l2_timeval_to_ns"
>   vaapi_x11_deps="xlib_x11"
>   videotoolbox_hwaccel_deps="videotoolbox pthreads"
>   videotoolbox_hwaccel_extralibs="-framework QuartzCore"
> @@ -7172,6 +7175,10 @@ if enabled v4l2_m2m; then
>       check_cc vp9_v4l2_m2m linux/videodev2.h "int i = V4L2_PIX_FMT_VP9;"
>   fi
>   
> +if enabled v4l2_request; then
> +    check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
> +fi
> +
>   check_headers sys/videoio.h
>   test_code cc sys/videoio.h "struct v4l2_frmsizeenum vfse; vfse.discrete.width = 0;" && enable_sanitized struct_v4l2_frmivalenum_discrete
>   
> diff --git a/libavutil/Makefile b/libavutil/Makefile
> index 6e6fa8d800..1ce46157dd 100644
> --- a/libavutil/Makefile
> +++ b/libavutil/Makefile
> @@ -48,6 +48,7 @@ HEADERS = adler32.h                                                     \
>             hwcontext_qsv.h                                               \
>             hwcontext_mediacodec.h                                        \
>             hwcontext_opencl.h                                            \
> +          hwcontext_v4l2request.h                                       \
>             hwcontext_vaapi.h                                             \
>             hwcontext_videotoolbox.h                                      \
>             hwcontext_vdpau.h                                             \
> @@ -201,6 +202,7 @@ OBJS-$(CONFIG_MACOS_KPERF)              += macos_kperf.o
>   OBJS-$(CONFIG_MEDIACODEC)               += hwcontext_mediacodec.o
>   OBJS-$(CONFIG_OPENCL)                   += hwcontext_opencl.o
>   OBJS-$(CONFIG_QSV)                      += hwcontext_qsv.o
> +OBJS-$(CONFIG_V4L2_REQUEST)             += hwcontext_v4l2request.o
>   OBJS-$(CONFIG_VAAPI)                    += hwcontext_vaapi.o
>   OBJS-$(CONFIG_VIDEOTOOLBOX)             += hwcontext_videotoolbox.o
>   OBJS-$(CONFIG_VDPAU)                    += hwcontext_vdpau.o
> @@ -222,6 +224,7 @@ SKIPHEADERS-$(CONFIG_D3D12VA)          += hwcontext_d3d12va.h
>   SKIPHEADERS-$(CONFIG_DXVA2)            += hwcontext_dxva2.h
>   SKIPHEADERS-$(CONFIG_QSV)              += hwcontext_qsv.h
>   SKIPHEADERS-$(CONFIG_OPENCL)           += hwcontext_opencl.h
> +SKIPHEADERS-$(CONFIG_V4L2_REQUEST)     += hwcontext_v4l2request.h
>   SKIPHEADERS-$(CONFIG_VAAPI)            += hwcontext_vaapi.h
>   SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX)     += hwcontext_videotoolbox.h
>   SKIPHEADERS-$(CONFIG_VDPAU)            += hwcontext_vdpau.h
> diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c
> index fa99a0d8a4..7fae9381da 100644
> --- a/libavutil/hwcontext.c
> +++ b/libavutil/hwcontext.c
> @@ -65,6 +65,9 @@ static const HWContextType * const hw_table[] = {
>   #endif
>   #if CONFIG_VULKAN
>       &ff_hwcontext_type_vulkan,
> +#endif
> +#if CONFIG_V4L2_REQUEST
> +    &ff_hwcontext_type_v4l2request,
>   #endif
>       NULL,
>   };
> @@ -77,6 +80,7 @@ static const char *const hw_type_names[] = {
>       [AV_HWDEVICE_TYPE_D3D12VA] = "d3d12va",
>       [AV_HWDEVICE_TYPE_OPENCL] = "opencl",
>       [AV_HWDEVICE_TYPE_QSV]    = "qsv",
> +    [AV_HWDEVICE_TYPE_V4L2REQUEST] = "v4l2request",
>       [AV_HWDEVICE_TYPE_VAAPI]  = "vaapi",
>       [AV_HWDEVICE_TYPE_VDPAU]  = "vdpau",
>       [AV_HWDEVICE_TYPE_VIDEOTOOLBOX] = "videotoolbox",
> diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h
> index bac30debae..8cf50ddbd0 100644
> --- a/libavutil/hwcontext.h
> +++ b/libavutil/hwcontext.h
> @@ -38,6 +38,7 @@ enum AVHWDeviceType {
>       AV_HWDEVICE_TYPE_MEDIACODEC,
>       AV_HWDEVICE_TYPE_VULKAN,
>       AV_HWDEVICE_TYPE_D3D12VA,
> +    AV_HWDEVICE_TYPE_V4L2REQUEST,
>   };
>   
>   /**
> diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h
> index e32b786238..fd0cf29c6e 100644
> --- a/libavutil/hwcontext_internal.h
> +++ b/libavutil/hwcontext_internal.h
> @@ -158,6 +158,7 @@ extern const HWContextType ff_hwcontext_type_drm;
>   extern const HWContextType ff_hwcontext_type_dxva2;
>   extern const HWContextType ff_hwcontext_type_opencl;
>   extern const HWContextType ff_hwcontext_type_qsv;
> +extern const HWContextType ff_hwcontext_type_v4l2request;
>   extern const HWContextType ff_hwcontext_type_vaapi;
>   extern const HWContextType ff_hwcontext_type_vdpau;
>   extern const HWContextType ff_hwcontext_type_videotoolbox;
> diff --git a/libavutil/hwcontext_v4l2request.c b/libavutil/hwcontext_v4l2request.c
> new file mode 100644
> index 0000000000..833fbf9f40
> --- /dev/null
> +++ b/libavutil/hwcontext_v4l2request.c
> @@ -0,0 +1,261 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "config.h"
> +
> +#include <fcntl.h>
> +#include <linux/dma-buf.h>
> +#include <linux/media.h>
> +#include <sys/ioctl.h>
> +#include <sys/mman.h>
> +#include <unistd.h>
> +
> +#include "avassert.h"
> +#include "hwcontext_drm.h"
> +#include "hwcontext_internal.h"
> +#include "hwcontext_v4l2request.h"
> +#include "mem.h"
> +
> +static void v4l2request_device_free(AVHWDeviceContext *hwdev)
> +{
> +    AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx;
> +
> +    if (hwctx->media_fd >= 0) {
> +        close(hwctx->media_fd);
> +        hwctx->media_fd = -1;
> +    }
> +}
> +
> +static int v4l2request_device_create(AVHWDeviceContext *hwdev, const char *device,
> +                                     AVDictionary *opts, int flags)
> +{
> +    AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx;
> +
> +    hwctx->media_fd = -1;
> +    hwdev->free = v4l2request_device_free;
> +
> +    // Use auto-detect
> +    if (!device || !device[0])
> +        return 0;
> +
> +    hwctx->media_fd = open(device, O_RDWR);
> +    if (hwctx->media_fd < 0)
> +        return AVERROR(errno);
> +
> +    return 0;
> +}
> +
> +static int v4l2request_device_init(AVHWDeviceContext *hwdev)
> +{
> +    AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx;
> +    struct media_device_info device_info;
> +
> +    // Use auto-detect
> +    if (hwctx->media_fd < 0)
> +        return 0;
> +
> +    if (ioctl(hwctx->media_fd, MEDIA_IOC_DEVICE_INFO, &device_info) < 0)
> +        return AVERROR(errno);
> +
> +    av_log(hwdev, AV_LOG_VERBOSE, "Using V4L2 media driver %s (%u.%u.%u)\n",
> +           device_info.driver,
> +           device_info.driver_version >> 16,
> +           (device_info.driver_version >> 8) & 0xff,
> +           device_info.driver_version & 0xff);
> +
> +    return 0;
> +}
> +
> +static int v4l2request_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
> +{
> +    frame->buf[0] = av_buffer_pool_get(hwfc->pool);
> +    if (!frame->buf[0])
> +        return AVERROR(ENOMEM);
> +
> +    frame->data[0] = (uint8_t *)frame->buf[0]->data;
> +
> +    frame->format = AV_PIX_FMT_DRM_PRIME;
> +    frame->width  = hwfc->width;
> +    frame->height = hwfc->height;
> +
> +    return 0;
> +}
> +
> +typedef struct DRMMapping {
> +    // Address and length of each mmap()ed region.
> +    int nb_regions;
> +    int object[AV_DRM_MAX_PLANES];
> +    void *address[AV_DRM_MAX_PLANES];
> +    size_t length[AV_DRM_MAX_PLANES];
> +} DRMMapping;
> +
> +static void v4l2request_unmap_frame(AVHWFramesContext *hwfc,
> +                                    HWMapDescriptor *hwmap)
> +{
> +    DRMMapping *map = hwmap->priv;
> +
> +    for (int i = 0; i < map->nb_regions; i++) {
> +        struct dma_buf_sync sync = {
> +            .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ,
> +        };
> +        ioctl(map->object[i], DMA_BUF_IOCTL_SYNC, &sync);
> +        munmap(map->address[i], map->length[i]);
> +    }
> +
> +    av_free(map);
> +}
> +
> +static int v4l2request_map_frame(AVHWFramesContext *hwfc,
> +                                 AVFrame *dst, const AVFrame *src)
> +{
> +    const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
> +    struct dma_buf_sync sync = {
> +        .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ,
> +    };
> +    DRMMapping *map;
> +    int ret, i, p, plane;
> +    void *addr;
> +
> +    map = av_mallocz(sizeof(*map));
> +    if (!map)
> +        return AVERROR(ENOMEM);
> +
> +    av_assert0(desc->nb_objects <= AV_DRM_MAX_PLANES);
> +    for (i = 0; i < desc->nb_objects; i++) {
> +        addr = mmap(NULL, desc->objects[i].size, AV_HWFRAME_MAP_READ, MAP_SHARED,
> +                    desc->objects[i].fd, 0);
> +        if (addr == MAP_FAILED) {
> +            av_log(hwfc, AV_LOG_ERROR, "Failed to map DRM object %d to memory: %s (%d)\n",
> +                   desc->objects[i].fd, strerror(errno), errno);
> +            ret = AVERROR(errno);
> +            goto fail;
> +        }
> +
> +        map->address[i] = addr;
> +        map->length[i]  = desc->objects[i].size;
> +        map->object[i]  = desc->objects[i].fd;
> +
> +        /*
> +         * We're not checking for errors here because the kernel may not
> +         * support the ioctl, in which case its okay to carry on
> +         */
> +        ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_SYNC, &sync);
> +    }
> +    map->nb_regions = i;
> +
> +    plane = 0;
> +    for (i = 0; i < desc->nb_layers; i++) {
> +        const AVDRMLayerDescriptor *layer = &desc->layers[i];
> +        for (p = 0; p < layer->nb_planes; p++) {
> +            dst->data[plane] =
> +                (uint8_t *)map->address[layer->planes[p].object_index] +
> +                                        layer->planes[p].offset;
> +            dst->linesize[plane] =      layer->planes[p].pitch;
> +            ++plane;
> +        }
> +    }
> +    av_assert0(plane <= AV_DRM_MAX_PLANES);
> +
> +    dst->width  = src->width;
> +    dst->height = src->height;
> +
> +    ret = ff_hwframe_map_create(src->hw_frames_ctx, dst, src,
> +                                v4l2request_unmap_frame, map);
> +    if (ret < 0)
> +        goto fail;
> +
> +    return 0;
> +
> +fail:
> +    for (i = 0; i < desc->nb_objects; i++) {
> +        if (map->address[i])
> +            munmap(map->address[i], map->length[i]);
> +    }
> +    av_free(map);
> +    return ret;
> +}
> +
> +static int v4l2request_transfer_get_formats(AVHWFramesContext *hwfc,
> +                                            enum AVHWFrameTransferDirection dir,
> +                                            enum AVPixelFormat **formats)
> +{
> +    enum AVPixelFormat *pix_fmts;
> +
> +    if (dir == AV_HWFRAME_TRANSFER_DIRECTION_TO)
> +        return AVERROR(ENOSYS);
> +
> +    pix_fmts = av_malloc_array(2, sizeof(*pix_fmts));
> +    if (!pix_fmts)
> +        return AVERROR(ENOMEM);
> +
> +    pix_fmts[0] = hwfc->sw_format;
> +    pix_fmts[1] = AV_PIX_FMT_NONE;
> +
> +    *formats = pix_fmts;
> +    return 0;
> +}
> +
> +static int v4l2request_transfer_data_from(AVHWFramesContext *hwfc,
> +                                          AVFrame *dst, const AVFrame *src)
> +{
> +    AVFrame *map;
> +    int ret;
> +
> +    if (dst->width > hwfc->width || dst->height > hwfc->height)
> +        return AVERROR(EINVAL);
> +
> +    map = av_frame_alloc();
> +    if (!map)
> +        return AVERROR(ENOMEM);
> +    map->format = dst->format;
> +
> +    ret = v4l2request_map_frame(hwfc, map, src);
> +    if (ret)
> +        goto fail;
> +
> +    map->width  = dst->width;
> +    map->height = dst->height;
> +
> +    ret = av_frame_copy(dst, map);
> +    if (ret)
> +        goto fail;
> +
> +    ret = 0;
> +fail:
> +    av_frame_free(&map);
> +    return ret;
> +}
> +
> +const HWContextType ff_hwcontext_type_v4l2request = {
> +    .type                   = AV_HWDEVICE_TYPE_V4L2REQUEST,
> +    .name                   = "V4L2 Request API",
> +
> +    .device_hwctx_size      = sizeof(AVV4L2RequestDeviceContext),
> +    .device_create          = v4l2request_device_create,
> +    .device_init            = v4l2request_device_init,
> +
> +    .frames_get_buffer      = v4l2request_get_buffer,
> +
> +    .transfer_get_formats   = v4l2request_transfer_get_formats,
> +    .transfer_data_from     = v4l2request_transfer_data_from,
> +
> +    .pix_fmts = (const enum AVPixelFormat[]) {
> +        AV_PIX_FMT_DRM_PRIME,
> +        AV_PIX_FMT_NONE
> +    },
> +};
> diff --git a/libavutil/hwcontext_v4l2request.h b/libavutil/hwcontext_v4l2request.h
> new file mode 100644
> index 0000000000..0fe42f97b4
> --- /dev/null
> +++ b/libavutil/hwcontext_v4l2request.h
> @@ -0,0 +1,41 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#ifndef AVUTIL_HWCONTEXT_V4L2REQUEST_H
> +#define AVUTIL_HWCONTEXT_V4L2REQUEST_H
> +
> +/**
> + * @file
> + * An API-specific header for AV_HWDEVICE_TYPE_V4L2REQUEST.
> + */
> +
> +/**
> + * V4L2 Request API device details.
> + *
> + * Allocated as AVHWDeviceContext.hwctx
> + */
> +typedef struct AVV4L2RequestDeviceContext {
> +    /**
> +     * File descriptor of media device.
> +     *
> +     * Defaults to -1 for auto-detect.
> +     */
> +    int media_fd;
> +} AVV4L2RequestDeviceContext;
> +
> +#endif /* AVUTIL_HWCONTEXT_V4L2REQUEST_H */

Any reason you're not using hwcontext_drm.h?

[-- Attachment #1.1.1.2: OpenPGP public key --]
[-- Type: application/pgp-keys, Size: 624 bytes --]

[-- Attachment #1.2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 236 bytes --]

[-- Attachment #2: Type: text/plain, Size: 251 bytes --]

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".