[FFmpeg-devel] [PATCH] avdevice/pipewire_dec: pipewire video capture

* [FFmpeg-devel] [PATCH] avdevice/pipewire_dec: pipewire video capture
@ 2023-04-23 12:24 metamuffin
  2023-04-24  6:36 ` Anton Khirnov
  0 siblings, 1 reply; 4+ messages in thread
From: metamuffin @ 2023-04-23 12:24 UTC (permalink / raw)
  To: ffmpeg-devel


[-- Attachment #1.1.1.1: Type: text/plain, Size: 15935 bytes --]

Added a minimal (and partial) input device for pipewire.
The implementation lacks audio support for now since alsa and pulse can 
do that too
Video output is not included yet. The patch requires _XOPEN_SOURCE=700
to work.

Signed-off-by: metamuffin <metamuffin@disroot.org>
---
  Changelog                  |   1 +
  configure                  |   4 +
  doc/indevs.texi            |  23 +++
  libavdevice/Makefile       |   1 +
  libavdevice/alldevices.c   |   1 +
  libavdevice/pipewire_dec.c | 288 +++++++++++++++++++++++++++++++++++++
  libavdevice/version.h      |   2 +-
  7 files changed, 319 insertions(+), 1 deletion(-)
  create mode 100644 libavdevice/pipewire_dec.c

diff --git a/Changelog b/Changelog
index 4284a250a2..176a90b393 100644
--- a/Changelog
+++ b/Changelog
@@ -4,6 +4,7 @@ releases are sorted from youngest to oldest.
  version <next>:
  - libaribcaption decoder
  - Playdate video decoder and demuxer
+- pipewire video input device
   version 6.0:
  - Radiance HDR image support
diff --git a/configure b/configure
index 549ed1401c..944eff687a 100755
--- a/configure
+++ b/configure
@@ -257,6 +257,7 @@ External library support:
    --enable-libopenvino     enable OpenVINO as a DNN module backend
                             for DNN based filters like dnn_processing [no]
    --enable-libopus         enable Opus de/encoding via libopus [no]
+  --enable-libpipewire     enable Pipewire input via libpipewire [no]
    --enable-libplacebo      enable libplacebo library [no]
    --enable-libpulse        enable Pulseaudio input via libpulse [no]
    --enable-librabbitmq     enable RabbitMQ library [no]
@@ -1838,6 +1839,7 @@ EXTERNAL_LIBRARY_LIST="
      libopenmpt
      libopenvino
      libopus
+    libpipewire
      libplacebo
      libpulse
      librabbitmq
@@ -3541,6 +3543,7 @@ opengl_outdev_deps="opengl"
  opengl_outdev_suggest="sdl2"
  oss_indev_deps_any="sys_soundcard_h"
  oss_outdev_deps_any="sys_soundcard_h"
+pipewire_indev_deps="libpipewire"
  pulse_indev_deps="libpulse"
  pulse_outdev_deps="libpulse"
  sdl2_outdev_deps="sdl2"
@@ -6669,6 +6672,7 @@ enabled libopus           && {
          require_pkg_config libopus opus opus_multistream.h 
opus_multistream_surround_encoder_create
      }
  }
+enabled libpipewire       && require_pkg_config libpipewire 
libpipewire-0.3 pipewire/pipewire.h pw_init -D_XOPEN_SOURCE=700
  enabled libplacebo        && require_pkg_config libplacebo "libplacebo 
 >= 4.192.0" libplacebo/vulkan.h pl_vulkan_create
  enabled libpulse          && require_pkg_config libpulse libpulse 
pulse/pulseaudio.h pa_context_new
  enabled librabbitmq       && require_pkg_config librabbitmq 
"librabbitmq >= 0.7.1" amqp.h amqp_new_connection
diff --git a/doc/indevs.texi b/doc/indevs.texi
index 8a198c4b44..64707bd74d 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -1254,6 +1254,29 @@ Set the number of channels. Default is 2.
   @end table
  +@section pipewire
+
+Pipewire video input.
+
+The node's target object is set to the URL.
+
+More information about Pipewire can be found on @url{https://pipewire.org}.
+
+@subsection Options
+
+@table @option
+
+@item framerate
+Sets the average framerate (in Hz) reported from the demuxer, the 
actual framerate can differ. Default is 60 FPS.
+
+@end table
+
+@subsection Examples
+Caputure screen from an already present xdg-desktop-portal node.
+@example
+ffmpeg_g -f pipewire -i xdg-desktop-portal-wlr /tmp/recording.webm
+@end example
+
  @section pulse
   PulseAudio input device.
diff --git a/libavdevice/Makefile b/libavdevice/Makefile
index 8a62822b69..4b180d9b28 100644
--- a/libavdevice/Makefile
+++ b/libavdevice/Makefile
@@ -38,6 +38,7 @@ OBJS-$(CONFIG_OPENAL_INDEV)              += openal-dec.o
  OBJS-$(CONFIG_OPENGL_OUTDEV)             += opengl_enc.o
  OBJS-$(CONFIG_OSS_INDEV)                 += oss_dec.o oss.o
  OBJS-$(CONFIG_OSS_OUTDEV)                += oss_enc.o oss.o
+OBJS-$(CONFIG_PIPEWIRE_INDEV)            += pipewire_dec.o
  OBJS-$(CONFIG_PULSE_INDEV)               += pulse_audio_dec.o \
                                              pulse_audio_common.o 
timefilter.o
  OBJS-$(CONFIG_PULSE_OUTDEV)              += pulse_audio_enc.o \
diff --git a/libavdevice/alldevices.c b/libavdevice/alldevices.c
index 8a90fcb5d7..4937a9994f 100644
--- a/libavdevice/alldevices.c
+++ b/libavdevice/alldevices.c
@@ -44,6 +44,7 @@ extern const AVInputFormat  ff_openal_demuxer;
  extern const FFOutputFormat ff_opengl_muxer;
  extern const AVInputFormat  ff_oss_demuxer;
  extern const FFOutputFormat ff_oss_muxer;
+extern const AVInputFormat  ff_pipewire_demuxer;
  extern const AVInputFormat  ff_pulse_demuxer;
  extern const FFOutputFormat ff_pulse_muxer;
  extern const FFOutputFormat ff_sdl2_muxer;
diff --git a/libavdevice/pipewire_dec.c b/libavdevice/pipewire_dec.c
new file mode 100644
index 0000000000..f9d1fc80a8
--- /dev/null
+++ b/libavdevice/pipewire_dec.c
@@ -0,0 +1,288 @@
+/*
+ * Pipewire video capture
+ * Copyright (c) 2023 metamuffin <metamuffin@disroot.org>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 
02110-1301 USA
+ */
+
+#undef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 700 // required for uselocale() in pipewire headers
+
+#include <pipewire/pipewire.h>
+#include <spa/param/video/format-utils.h>
+#include <spa/param/video/type-info.h>
+#include <spa/debug/types.h>
+#include <stdatomic.h>
+
+#include "libavutil/parseutils.h"
+#include "libavutil/internal.h"
+#include "libavutil/opt.h"
+#include "libavutil/time.h"
+#include "libavformat/avformat.h"
+#include "libavformat/internal.h"
+
+struct pipewire_state {
+    AVClass* class;
+
+    const char* framerate;
+
+    struct pw_thread_loop* loop;
+    struct pw_stream* stream;
+
+    int ready;
+    int width;
+    int height;
+    struct spa_video_info format;
+
+    int buffer_len;
+    _Atomic(void*) buffer;
+};
+
+static enum AVPixelFormat pixelformat_spa_to_av(enum spa_video_format 
format) {
+    switch (format) {
+    case SPA_VIDEO_FORMAT_I420:
+        return AV_PIX_FMT_YUV420P;
+    case SPA_VIDEO_FORMAT_YUY2:
+        return AV_PIX_FMT_YUYV422;
+    case SPA_VIDEO_FORMAT_RGBx:
+        return AV_PIX_FMT_RGB0;
+    case SPA_VIDEO_FORMAT_BGRx:
+        return AV_PIX_FMT_BGR0;
+    case SPA_VIDEO_FORMAT_RGBA:
+        return AV_PIX_FMT_RGBA;
+    case SPA_VIDEO_FORMAT_BGRA:
+        return AV_PIX_FMT_BGRA;
+    case SPA_VIDEO_FORMAT_RGB:
+        return AV_PIX_FMT_RGB8;
+    case SPA_VIDEO_FORMAT_BGR:
+        return AV_PIX_FMT_BGR8;
+    default:
+        return -1;
+    }
+}
+
+static void on_param_changed(void* s, uint32_t id, const struct 
spa_pod* param) {
+    struct pipewire_state* state = s;
+
+    if (id != SPA_PARAM_Format || param == NULL)
+        return;
+
+    if (spa_format_parse(param, &state->format.media_type, 
&state->format.media_subtype) < 0)
+        return;
+
+    if (state->format.media_type != SPA_MEDIA_TYPE_video
+        || state->format.media_subtype != SPA_MEDIA_SUBTYPE_raw)
+        return;
+
+    if (spa_format_video_raw_parse(param, &state->format.info.raw) < 0)
+        return;
+
+    state->width = state->format.info.raw.size.width;
+    state->height = state->format.info.raw.size.height;
+    state->ready = 1;
+
+    av_log(state, AV_LOG_INFO, "got video format:\n");
+    av_log(state, AV_LOG_INFO, "  format: %d (%s)\n", 
state->format.info.raw.format,
+           spa_debug_type_find_name(spa_type_video_format, 
state->format.info.raw.format));
+    av_log(state, AV_LOG_INFO, "  size: %dx%d\n", 
state->format.info.raw.size.width,
+           state->format.info.raw.size.height);
+    av_log(state, AV_LOG_INFO, "  framerate: %d/%d\n", 
state->format.info.raw.framerate.num,
+           state->format.info.raw.framerate.denom);
+}
+
+static void on_process(void* s) {
+    struct pipewire_state* state = s;
+    struct pw_buffer* b;
+    struct spa_buffer* pw_buffer;
+    void* buffer;
+
+    av_log(state, AV_LOG_DEBUG, "process\n");
+
+    if ((b = pw_stream_dequeue_buffer(state->stream)) == NULL) {
+        pw_log_warn("out of buffers: %m");
+        return;
+    }
+
+    pw_buffer = b->buffer;
+    if (pw_buffer->datas[0].data == NULL)
+        return;
+
+    buffer = av_malloc(pw_buffer->datas[0].chunk->size);
+    if (!buffer)
+        return;
+    memcpy(buffer, pw_buffer->datas[0].data, 
pw_buffer->datas[0].chunk->size);
+
+    if (!state->buffer_len)
+        state->buffer_len = pw_buffer->datas[0].chunk->size;
+
+    pw_stream_queue_buffer(state->stream, b);
+
+    // swap in the new buffer and free the old one
+    buffer = atomic_exchange(&state->buffer, buffer);
+    if (buffer)
+        av_free(buffer);
+}
+
+static void on_state_changed(void* s, enum pw_stream_state old, enum 
pw_stream_state new,
+                             const char* error) {
+    struct pipewire_state* state = s;
+    av_log(state, AV_LOG_DEBUG, "stream state changed: %s -> %s\n",
+           pw_stream_state_as_string(old), pw_stream_state_as_string(new));
+}
+
+static const struct pw_stream_events stream_events = {
+    PW_VERSION_STREAM_EVENTS,
+    .state_changed = on_state_changed,
+    .param_changed = on_param_changed,
+    .process = on_process,
+};
+
+static av_cold int pwdec_read_header(AVFormatContext* s) {
+    struct pipewire_state* state = s->priv_data;
+    struct pw_properties* props;
+    AVStream* avstream;
+    uint8_t spa_pod_buffer[1024];
+    const struct spa_pod* params[1];
+    struct spa_pod_builder b = SPA_POD_BUILDER_INIT(spa_pod_buffer, 
sizeof(spa_pod_buffer));
+    int ret;
+    enum AVPixelFormat format;
+
+    avstream = avformat_new_stream(s, NULL);
+
+    ret = av_parse_video_rate(&avstream->avg_frame_rate, state->framerate);
+    if (ret < 0)
+        return ret;
+
+    state->ready = 0;
+    state->buffer_len = 0;
+    atomic_init(&state->buffer, NULL);
+
+    pw_init(0, NULL);
+    state->loop = pw_thread_loop_new("ffmpeg pipewire loop", NULL);
+
+    props = pw_properties_new(PW_KEY_MEDIA_TYPE, "Video", 
PW_KEY_MEDIA_CATEGORY, "Capture",
+                              PW_KEY_MEDIA_ROLE, "Camera", 
PW_KEY_APP_ID, "ffmpeg",
+                              PW_KEY_APP_NAME, "ffmpeg video capture", 
NULL);
+    if (s->url != NULL)
+        pw_properties_set(props, PW_KEY_TARGET_OBJECT, s->url);
+
+    state->stream = 
pw_stream_new_simple(pw_thread_loop_get_loop(state->loop), "ffmpeg-capture",
+                                         props, &stream_events, state);
+
+    params[0] = spa_pod_builder_add_object(
+            &b, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat, 
SPA_FORMAT_mediaType,
+            SPA_POD_Id(SPA_MEDIA_TYPE_video), SPA_FORMAT_mediaSubtype,
+            SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), SPA_FORMAT_VIDEO_format,
+            SPA_POD_CHOICE_ENUM_Id(8, SPA_VIDEO_FORMAT_RGB, 
SPA_VIDEO_FORMAT_BGR,
+                                   SPA_VIDEO_FORMAT_RGBA, 
SPA_VIDEO_FORMAT_BGRA,
+                                   SPA_VIDEO_FORMAT_RGBx, 
SPA_VIDEO_FORMAT_BGRx,
+                                   SPA_VIDEO_FORMAT_YUY2, 
SPA_VIDEO_FORMAT_I420),
+            SPA_FORMAT_VIDEO_size,
+            SPA_POD_CHOICE_RANGE_Rectangle(&SPA_RECTANGLE(320, 240), 
&SPA_RECTANGLE(1, 1),
+                                           &SPA_RECTANGLE(4096, 4096)),
+            SPA_FORMAT_VIDEO_framerate,
+            SPA_POD_CHOICE_RANGE_Fraction(&SPA_FRACTION(60, 1), 
&SPA_FRACTION(0, 1),
+                                          &SPA_FRACTION(1000, 1)));
+
+    pw_stream_connect(state->stream, PW_DIRECTION_INPUT, PW_ID_ANY,
+                      PW_STREAM_FLAG_AUTOCONNECT | 
PW_STREAM_FLAG_MAP_BUFFERS, params, 1);
+
+    pw_thread_loop_start(state->loop);
+    av_log(state, AV_LOG_INFO, "waiting for the stream to start… \n");
+    while (!state->ready) {
+        av_usleep(1000);
+    }
+    av_log(state, AV_LOG_INFO, "starting stream\n");
+
+    format = pixelformat_spa_to_av(state->format.info.raw.format);
+    if (format < 0) {
+        av_log(state, AV_LOG_ERROR, "pixel format not expected nor 
implemented\n");
+        return AVERROR(EINVAL);
+    }
+
+    avstream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
+    avstream->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
+    avstream->codecpar->format = format;
+    avstream->codecpar->width = state->width;
+    avstream->codecpar->height = state->height;
+    avpriv_set_pts_info(avstream, 64, 1, 1000000);
+
+    return 0;
+}
+
+static int pwdec_read_packet(AVFormatContext* s, AVPacket* pkt) {
+    struct pipewire_state* state = s->priv_data;
+    void* buffer = NULL;
+
+    buffer = atomic_exchange(&state->buffer, buffer);
+    if (!buffer)
+        return AVERROR(EWOULDBLOCK);
+
+    if (av_new_packet(pkt, state->buffer_len) < 0)
+        return AVERROR(ENOMEM);
+    memcpy(pkt->data, buffer, state->buffer_len);
+    av_free(buffer);
+
+    pkt->pts = pkt->dts = av_gettime();
+    av_log(state, AV_LOG_DEBUG, "pts=%li size=%i\n", pkt->pts, 
state->buffer_len);
+
+    return 0;
+}
+
+static av_cold int pwdec_close(AVFormatContext* s) {
+    struct pipewire_state* state = s->priv_data;
+
+    av_free(state->buffer);
+    pw_thread_loop_stop(state->loop);
+    pw_thread_loop_destroy(state->loop);
+    pw_deinit();
+
+    return 0;
+}
+
+static int pwdec_get_device_list(AVFormatContext* s, struct 
AVDeviceInfoList* device_list) {
+    struct pipewire_state* state = s->priv_data;
+    av_log(state, AV_LOG_ERROR, "device list not implemented");
+    return AVERROR(ENOTSUP);
+}
+
+#define OFFSET(x) offsetof(struct pipewire_state, x)
+#define DEC AV_OPT_FLAG_DECODING_PARAM
+static const AVOption options[] = {
+    { "framerate", "", OFFSET(framerate), AV_OPT_TYPE_STRING, { .str = 
"60" }, 0, 0, DEC },
+    { NULL },
+};
+
+static const AVClass pipewire_demuxer_class = {
+    .class_name = "pipewire input",
+    .item_name = av_default_item_name,
+    .option = options,
+    .version = LIBAVUTIL_VERSION_INT,
+    .category = AV_CLASS_CATEGORY_DEVICE_INPUT,
+};
+
+const AVInputFormat ff_pipewire_demuxer = {
+    .name = "pipewire",
+    .long_name = NULL_IF_CONFIG_SMALL("pipewire input"),
+    .priv_data_size = sizeof(struct pipewire_state),
+    .read_header = pwdec_read_header,
+    .read_packet = pwdec_read_packet,
+    .read_close = pwdec_close,
+    .get_device_list = pwdec_get_device_list,
+    .flags = AVFMT_NOFILE,
+    .priv_class = &pipewire_demuxer_class,
+};
diff --git a/libavdevice/version.h b/libavdevice/version.h
index 5cd01a1672..7608a8602c 100644
--- a/libavdevice/version.h
+++ b/libavdevice/version.h
@@ -29,7 +29,7 @@
   #include "version_major.h"
  -#define LIBAVDEVICE_VERSION_MINOR   2
+#define LIBAVDEVICE_VERSION_MINOR   3
  #define LIBAVDEVICE_VERSION_MICRO 100
   #define LIBAVDEVICE_VERSION_INT 
AV_VERSION_INT(LIBAVDEVICE_VERSION_MAJOR, \
-- 
2.40.0


[-- Attachment #1.1.1.2: OpenPGP public key --]
[-- Type: application/pgp-keys, Size: 677 bytes --]

[-- Attachment #1.2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 236 bytes --]

[-- Attachment #2: Type: text/plain, Size: 251 bytes --]

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

^ permalink raw reply	[flat|nested] 4+ messages in thread