[FFmpeg-devel] [PATCH v2 5/6] libavformat/webrtc_demux: add WebRTC-HTTP egress protocol (WHEP) demuxer

[Michael Riedl <michael.riedl@nativewaves.com>]

Signed-off-by: Michael Riedl <michael.riedl@nativewaves.com>
---
 Changelog                        |   4 +
 configure                        |   2 +
 doc/demuxers.texi                |  22 +++
 libavformat/Makefile             |   1 +
 libavformat/allformats.c         |   1 +
 libavformat/webrtc_demux.c (new) | 246 +++++++++++++++++++++++++++++++
 6 files changed, 276 insertions(+)

diff --git a/Changelog b/Changelog
index 8f0606fc267..45c6c752a06 100644
--- a/Changelog
+++ b/Changelog
@@ -1,6 +1,10 @@
 Entries are sorted chronologically from oldest to youngest within each release,
 releases are sorted from youngest to oldest.
 
+version <next>:
+- WHEP demuxer
+
+
 version 6.1:
 - libaribcaption decoder
 - Playdate video decoder and demuxer
diff --git a/configure b/configure
index 187f16b425d..02c6f7f2c5d 100755
--- a/configure
+++ b/configure
@@ -3557,6 +3557,8 @@ wav_demuxer_select="riffdec"
 wav_muxer_select="riffenc"
 webm_chunk_muxer_select="webm_muxer"
 webm_dash_manifest_demuxer_select="matroska_demuxer"
+whep_demuxer_deps="libdatachannel sdp_demuxer"
+whep_demuxer_select="http_protocol"
 wtv_demuxer_select="mpegts_demuxer riffdec"
 wtv_muxer_select="mpegts_muxer riffenc"
 xmv_demuxer_select="riffdec"
diff --git a/doc/demuxers.texi b/doc/demuxers.texi
index ca1563abb03..81940b8ece7 100644
--- a/doc/demuxers.texi
+++ b/doc/demuxers.texi
@@ -943,4 +943,26 @@ which in turn, acts as a ceiling for the size of scripts that can be read.
 Default is 1 MiB.
 @end table
 
+@section whep
+
+WebRTC-HTTP egress protocol (WHEP) demuxer.
+
+This demuxers allows reading media from a remote media server using the
+WebRTC-HTTP egress protocol (WHEP) as defined in
+@url{https://datatracker.ietf.org/doc/draft-murillo-whep/02}. Currently, only a
+single video (H.264) and a single audio stream (OPUS) are supported.
+
+This demuxer accepts the following options:
+@table @option
+@item bearer_token
+Optional bearer token for authentication and authorization to the HTTP server.
+Default is @code{NULL}.
+@item connection_timeout
+Timeout for establishing a connection to the media server.
+Default is 10 seconds.
+@item rw_timeout
+Timeout for receiving/writing data from/to the media server.
+Default is 1 second.
+@end table
+
 @c man end DEMUXERS
diff --git a/libavformat/Makefile b/libavformat/Makefile
index 329055ccfd9..f790fa8cae4 100644
--- a/libavformat/Makefile
+++ b/libavformat/Makefile
@@ -621,6 +621,7 @@ OBJS-$(CONFIG_WEBM_CHUNK_MUXER)          += webm_chunk.o
 OBJS-$(CONFIG_WEBP_MUXER)                += webpenc.o
 OBJS-$(CONFIG_WEBVTT_DEMUXER)            += webvttdec.o subtitles.o
 OBJS-$(CONFIG_WEBVTT_MUXER)              += webvttenc.o
+OBJS-$(CONFIG_WHEP_DEMUXER)              += webrtc.o webrtc_demux.o
 OBJS-$(CONFIG_WSAUD_DEMUXER)             += westwood_aud.o
 OBJS-$(CONFIG_WSAUD_MUXER)               += westwood_audenc.o
 OBJS-$(CONFIG_WSD_DEMUXER)               += wsddec.o rawdec.o
diff --git a/libavformat/allformats.c b/libavformat/allformats.c
index d4b505a5a32..7acb05634c8 100644
--- a/libavformat/allformats.c
+++ b/libavformat/allformats.c
@@ -504,6 +504,7 @@ extern const FFOutputFormat ff_webm_chunk_muxer;
 extern const FFOutputFormat ff_webp_muxer;
 extern const AVInputFormat  ff_webvtt_demuxer;
 extern const FFOutputFormat ff_webvtt_muxer;
+extern const AVInputFormat  ff_whep_demuxer;
 extern const AVInputFormat  ff_wsaud_demuxer;
 extern const FFOutputFormat ff_wsaud_muxer;
 extern const AVInputFormat  ff_wsd_demuxer;
diff --git a/libavformat/webrtc_demux.c b/libavformat/webrtc_demux.c
new file mode 100644
index 00000000000..391eea6d654
--- /dev/null
+++ b/libavformat/webrtc_demux.c
@@ -0,0 +1,246 @@
+/*
+ * WebRTC-HTTP egress protocol (WHEP) demuxer using libdatachannel
+ *
+ * Copyright (C) 2023 NativeWaves GmbH <contact@nativewaves.com>
+ * This work is supported by FFG project 47168763.
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "internal.h"
+#include "libavutil/avstring.h"
+#include "libavutil/time.h"
+#include "libavutil/random_seed.h"
+#include "version.h"
+#include "rtsp.h"
+#include "webrtc.h"
+
+typedef struct WHEPContext {
+    const AVClass *av_class;
+    WebRTCContext webrtc_ctx;
+} WHEPContext;
+
+static int whep_read_header(AVFormatContext* avctx)
+{
+    WHEPContext*const ctx = (WHEPContext*const)avctx->priv_data;
+    int ret, i;
+    char media_stream_id[37] = { 0 };
+    rtcTrackInit track_init;
+    AVDictionary* options = NULL;
+    const AVInputFormat* infmt;
+    AVStream* stream;
+    FFIOContext sdp_pb;
+    int64_t timeout;
+
+    ff_webrtc_init_logger();
+    ret = ff_webrtc_init_connection(&ctx->webrtc_ctx);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to initialize connection\n");
+        goto fail;
+    }
+
+    /* configure audio and video track */
+    ret = ff_webrtc_generate_media_stream_id(media_stream_id);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to generate media stream id\n");
+        goto fail;
+    }
+    ctx->webrtc_ctx.tracks = av_mallocz(2 * sizeof(WebRTCTrack));
+    ctx->webrtc_ctx.nb_tracks = 2;
+    ctx->webrtc_ctx.avctx = avctx;
+    if (!ctx->webrtc_ctx.tracks) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+    for (i=0; i < ctx->webrtc_ctx.nb_tracks; i++) {
+        ctx->webrtc_ctx.tracks[i].avctx = avctx;
+    }
+
+    /* configure video track */
+    memset(&track_init, 0, sizeof(rtcTrackInit));
+    track_init.direction = RTC_DIRECTION_RECVONLY;
+    track_init.codec = RTC_CODEC_H264; // TODO: support more codecs once libdatachannel C api supports them
+    track_init.payloadType = 96;
+    track_init.ssrc = av_get_random_seed();
+    track_init.mid = "0";
+    track_init.name = LIBAVFORMAT_IDENT;
+    track_init.msid = media_stream_id;
+    track_init.trackId = av_asprintf("%s-video", media_stream_id);
+    track_init.profile = "profile-level-id=42e01f;packetization-mode=1;level-asymmetry-allowed=1";
+
+    ctx->webrtc_ctx.tracks[0].track_id = rtcAddTrackEx(ctx->webrtc_ctx.peer_connection, &track_init);
+    if (!ctx->webrtc_ctx.tracks[0].track_id) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to add track\n");
+        ret = AVERROR_EXTERNAL;
+        goto fail;
+    }
+
+    /* configure audio track */
+    memset(&track_init, 0, sizeof(rtcTrackInit));
+    track_init.direction = RTC_DIRECTION_RECVONLY;
+    track_init.codec = RTC_CODEC_OPUS; // TODO: support more codecs once libdatachannel C api supports them
+    track_init.payloadType = 97;
+    track_init.ssrc = av_get_random_seed();
+    track_init.mid = "1";
+    track_init.name = LIBAVFORMAT_IDENT;
+    track_init.msid = media_stream_id;
+    track_init.trackId = av_asprintf("%s-audio", media_stream_id);
+    track_init.profile = "minptime=10;maxaveragebitrate=96000;stereo=1;sprop-stereo=1;useinbandfec=1";
+
+    ctx->webrtc_ctx.tracks[1].track_id = rtcAddTrackEx(ctx->webrtc_ctx.peer_connection, &track_init);
+    if (!ctx->webrtc_ctx.tracks[1].track_id) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to add track\n");
+        ret = AVERROR_EXTERNAL;
+        goto fail;
+    }
+
+    /* create resource */
+    ret = ff_webrtc_create_resource(&ctx->webrtc_ctx);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR, "webrtc_create_resource failed\n");
+        goto fail;
+    }
+
+    /* wait for connection to be established */
+    timeout = av_gettime_relative() + ctx->webrtc_ctx.connection_timeout;
+    while (ctx->webrtc_ctx.state != RTC_CONNECTED) {
+        if (ctx->webrtc_ctx.state == RTC_FAILED || ctx->webrtc_ctx.state == RTC_CLOSED || av_gettime_relative() > timeout) {
+            av_log(avctx, AV_LOG_ERROR, "Failed to open connection\n");
+            ret = AVERROR_EXTERNAL;
+            goto fail;
+        }
+
+        av_log(avctx, AV_LOG_VERBOSE, "Waiting for PeerConnection to open\n");
+        av_usleep(1000);
+    }
+
+    /* initialize SDP muxer per track */
+    for (int i = 0; i < ctx->webrtc_ctx.nb_tracks; i++) {
+        char sdp_track[SDP_MAX_SIZE] = { 0 };
+        ret = rtcGetTrackDescription(ctx->webrtc_ctx.tracks[i].track_id, sdp_track, sizeof(sdp_track));
+        if (ret < 0) {
+            av_log(avctx, AV_LOG_ERROR, "rtcGetTrackDescription failed\n");
+            goto fail;
+        }
+
+        ffio_init_read_context(&sdp_pb, (uint8_t*)sdp_track, strlen(sdp_track));
+
+        infmt = av_find_input_format("sdp");
+        if (!infmt)
+            goto fail;
+        ctx->webrtc_ctx.tracks[i].rtp_ctx = avformat_alloc_context();
+        if (!ctx->webrtc_ctx.tracks[i].rtp_ctx) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
+        ctx->webrtc_ctx.tracks[i].rtp_ctx->max_delay = avctx->max_delay;
+        ctx->webrtc_ctx.tracks[i].rtp_ctx->pb = &sdp_pb.pub;
+        ctx->webrtc_ctx.tracks[i].rtp_ctx->interrupt_callback = avctx->interrupt_callback;
+
+        if ((ret = ff_copy_whiteblacklists(ctx->webrtc_ctx.tracks[i].rtp_ctx, avctx)) < 0)
+            goto fail;
+
+        av_dict_set(&options, "sdp_flags", "custom_io", 0);
+
+        ret = avformat_open_input(&ctx->webrtc_ctx.tracks[i].rtp_ctx, "temp.sdp", infmt, &options);
+        if (ret < 0) {
+            av_log(avctx, AV_LOG_ERROR, "avformat_open_input failed\n");
+            goto fail;
+        }
+
+        ret = ff_webrtc_init_urlcontext(&ctx->webrtc_ctx, i);
+        if (ret < 0) {
+            av_log(avctx, AV_LOG_ERROR, "webrtc_init_urlcontext failed\n");
+            goto fail;
+        }
+        ret = ffio_fdopen(&ctx->webrtc_ctx.tracks[i].rtp_ctx->pb, ctx->webrtc_ctx.tracks[i].rtp_url_context);
+        if (ret < 0) {
+            av_log(avctx, AV_LOG_ERROR, "ffio_fdopen failed\n");
+            goto fail;
+        }
+
+        /* copy codec parameters */
+        stream = avformat_new_stream(avctx, NULL);
+        if (!stream) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
+
+        ret = avcodec_parameters_copy(stream->codecpar, ctx->webrtc_ctx.tracks[i].rtp_ctx->streams[0]->codecpar);
+        if (ret < 0) {
+            av_log(avctx, AV_LOG_ERROR, "avcodec_parameters_copy failed\n");
+            goto fail;
+        }
+        stream->time_base = ctx->webrtc_ctx.tracks[i].rtp_ctx->streams[0]->time_base;
+    }
+
+    return 0;
+
+fail:
+    ff_webrtc_deinit(&ctx->webrtc_ctx);
+    return ret;
+}
+
+static int whep_read_close(AVFormatContext* avctx)
+{
+    WHEPContext*const ctx = (WHEPContext*const)avctx->priv_data;
+    int ret = 0;
+
+    /* close resource */
+    ret = ff_webrtc_close_resource(&ctx->webrtc_ctx);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR, "webrtc_close_resource failed\n");
+    }
+
+    ff_webrtc_deinit(&ctx->webrtc_ctx);
+
+    return ret;
+}
+
+static int whep_read_packet(AVFormatContext* avctx, AVPacket* pkt)
+{
+    const WHEPContext*const s = (const WHEPContext*const)avctx->priv_data;
+    const WebRTCTrack*const track = &s->webrtc_ctx.tracks[pkt->stream_index];
+    pkt->stream_index = 0;
+    return av_read_frame(track->rtp_ctx, pkt);
+}
+
+
+#define OFFSET(x) offsetof(WHEPContext, x)
+#define FLAGS AV_OPT_FLAG_DECODING_PARAM
+static const AVOption options[] = {
+    FF_WEBRTC_COMMON_OPTIONS,
+    { NULL },
+};
+
+static const AVClass whep_demuxer_class = {
+    .class_name = "WHEP demuxer",
+    .item_name  = av_default_item_name,
+    .option     = options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+const AVInputFormat ff_whep_demuxer = {
+    .name             = "whep",
+    .long_name        = NULL_IF_CONFIG_SMALL("WebRTC-HTTP egress protocol (WHEP) demuxer"),
+    .flags            = AVFMT_NOFILE | AVFMT_EXPERIMENTAL,
+    .priv_class       = &whep_demuxer_class,
+    .priv_data_size   = sizeof(WHEPContext),
+    .read_header      = whep_read_header,
+    .read_packet      = whep_read_packet,
+    .read_close       = whep_read_close,
+};
-- 
2.39.2

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".