[FFmpeg-devel] [PATCH] avformat/whip: add rtp history store and find method (PR #20703)

[Jack Lau via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

PR #20703 opened by Jack Lau (JackLau)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20703
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20703.patch

This patch aims to enable rtp history store for RTX

TODO:
handle the rtx send to make rtx really work


>From 90b863a2f54964009cff4dcf367e38267a0ad01b Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222@qq.com>
Date: Fri, 3 Oct 2025 10:33:55 +0800
Subject: [PATCH 1/2] avformat/whip: add rtp history store and find method

This patch aims to enable rtp history store for RTX

TODO:
handle the rtx send to make rtx really work

Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
 doc/muxers.texi    |  4 +++
 libavformat/whip.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)

diff --git a/doc/muxers.texi b/doc/muxers.texi
index 9889bd2ff6..91dc90008c 100644
--- a/doc/muxers.texi
+++ b/doc/muxers.texi
@@ -3959,6 +3959,10 @@ Default value is 1200.
 Set the buffer size, in bytes, of underlying protocol.
 Default value is -1(auto). The UDP auto selects a reasonable value.
 
+@item rtp_history @var{integer}
+Set the number of RTP history items to store.
+Default value is 512.
+
 @item authorization @var{string}
 The optional Bearer token for WHIP Authorization.
 
diff --git a/libavformat/whip.c b/libavformat/whip.c
index e809075643..0f00a6652b 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -168,6 +168,19 @@
 #define WHIP_ICE_CONSENT_CHECK_INTERVAL 5000
 #define WHIP_ICE_CONSENT_EXPIRED_TIMER 30000
 
+/**
+ * RTP history packet size.
+ * Target: hold 1000ms of RTP traffic.
+ *
+ * Formula:
+ * bandwidth_bps = (RTP payload bytes) * (RTP history size) * 8
+ *
+ * Assumes average RTP payload is 1184 bytes (MTU - SRTP_CHECKSUM_LEN).
+ */
+#define WHIP_RTP_HISTORY_MIN 64 /* around 0.61 Mbps */
+#define WHIP_RTP_HISTORY_DEFAULT 512 /* around 4.85 Mbps */
+#define WHIP_RTP_HISTORY_MAX 2048 /* around 19.40 Mbps */
+
 /* Calculate the elapsed time from starttime to endtime in milliseconds. */
 #define ELAPSED(starttime, endtime) ((float)(endtime - starttime) / 1000)
 
@@ -211,6 +224,12 @@ enum WHIPState {
     WHIP_STATE_FAILED,
 };
 
+typedef struct RtpHistoryItem {
+    uint16_t seq;
+    int size;
+    uint8_t *buf;
+} RtpHistoryItem;
+
 typedef struct WHIPContext {
     AVClass *av_class;
 
@@ -329,6 +348,11 @@ typedef struct WHIPContext {
     /* The certificate and private key used for DTLS handshake. */
     char* cert_file;
     char* key_file;
+
+    int hist_sz;
+    RtpHistoryItem *hist;
+    uint8_t *hist_pool;
+    int hist_head;
 } WHIPContext;
 
 /**
@@ -418,6 +442,17 @@ static av_cold int initialize(AVFormatContext *s)
     whip->audio_first_seq = av_lfg_get(&whip->rnd) & 0x0fff;
     whip->video_first_seq = whip->audio_first_seq + 1;
 
+    whip->hist = av_calloc(whip->hist_sz, sizeof(*whip->hist));
+    if (!whip->hist)
+        return AVERROR(ENOMEM);
+
+    whip->hist_pool = av_calloc(whip->hist_sz, whip->pkt_size - DTLS_SRTP_CHECKSUM_LEN);
+    if (!whip->hist_pool)
+        return AVERROR(ENOMEM);
+
+    for (int i = 0; i < whip->hist_sz; i++)
+        whip->hist[i].buf = whip->hist_pool + i * (whip->pkt_size - DTLS_SRTP_CHECKSUM_LEN);
+
     if (whip->pkt_size < ideal_pkt_size)
         av_log(whip, AV_LOG_WARNING, "pkt_size=%d(<%d) is too small, may cause packet loss\n",
                whip->pkt_size, ideal_pkt_size);
@@ -1473,6 +1508,28 @@ end:
     return ret;
 }
 
+static int rtp_history_store(WHIPContext *whip, const uint8_t *buf, int size)
+{
+    uint16_t seq = AV_RB16(buf + 2);
+    uint32_t pos = ((uint32_t)seq - (uint32_t)whip->video_first_seq) % (uint32_t)whip->hist_sz;
+    RtpHistoryItem *it = &whip->hist[pos];
+    if (size > whip->pkt_size - DTLS_SRTP_CHECKSUM_LEN)
+        return AVERROR_INVALIDDATA;
+    memcpy(it->buf, buf, size);
+    it->size = size;
+    it->seq = seq;
+
+    whip->hist_head = ++pos;
+    return 0;
+}
+
+static const RtpHistoryItem *rtp_history_find(WHIPContext *whip, uint16_t seq)
+{
+    uint32_t pos = ((uint32_t)seq - (uint32_t)whip->video_first_seq) % (uint32_t)whip->hist_sz;
+    const RtpHistoryItem *it = &whip->hist[pos];
+    return it->seq == seq ? it : NULL;
+}
+
 /**
  * Callback triggered by the RTP muxer when it creates and sends out an RTP packet.
  *
@@ -1509,6 +1566,12 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size)
         return 0;
     }
 
+    if (is_video) {
+        ret = rtp_history_store(whip, buf, buf_size);
+        if (ret < 0)
+            return ret;
+    }
+
     ret = ffurl_write(whip->udp, whip->buf, cipher_size);
     if (ret < 0) {
         av_log(whip, AV_LOG_ERROR, "Failed to write packet=%dB, ret=%d\n", cipher_size, ret);
@@ -1997,6 +2060,8 @@ static av_cold void whip_deinit(AVFormatContext *s)
     ff_srtp_free(&whip->srtp_recv);
     ffurl_close(whip->dtls_uc);
     ffurl_closep(&whip->udp);
+    av_freep(&whip->hist_pool);
+    av_freep(&whip->hist);
 }
 
 static int whip_check_bitstream(AVFormatContext *s, AVStream *st, const AVPacket *pkt)
@@ -2024,6 +2089,7 @@ static const AVOption options[] = {
     { "handshake_timeout",  "Timeout in milliseconds for ICE and DTLS handshake.",      OFFSET(handshake_timeout),  AV_OPT_TYPE_INT,    { .i64 = 5000 },    -1, INT_MAX, ENC },
     { "pkt_size",           "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size),           AV_OPT_TYPE_INT,    { .i64 = 1200 },    -1, INT_MAX, ENC },
     { "buffer_size",        "The buffer size, in bytes, of underlying protocol",        OFFSET(buffer_size),        AV_OPT_TYPE_INT,    { .i64 = -1 },      -1, INT_MAX, ENC },
+    { "rtp_history",        "The number of RTP history items to store",                 OFFSET(hist_sz),            AV_OPT_TYPE_INT,    { .i64 = WHIP_RTP_HISTORY_DEFAULT }, WHIP_RTP_HISTORY_MIN, WHIP_RTP_HISTORY_MAX, ENC },
     { "authorization",      "The optional Bearer token for WHIP Authorization",         OFFSET(authorization),      AV_OPT_TYPE_STRING, { .str = NULL },     0,       0, ENC },
     { "cert_file",          "The optional certificate file path for DTLS",              OFFSET(cert_file),          AV_OPT_TYPE_STRING, { .str = NULL },     0,       0, ENC },
     { "key_file",           "The optional private key file path for DTLS",              OFFSET(key_file),      AV_OPT_TYPE_STRING, { .str = NULL },     0,       0, ENC },
-- 
2.49.1


>From dc5fc6315374ca8d1d113a6af4448f74fb9b55c8 Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222@qq.com>
Date: Tue, 14 Oct 2025 17:24:10 +0800
Subject: [PATCH 2/2] avformat/whip: reindent the options

Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
 libavformat/whip.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libavformat/whip.c b/libavformat/whip.c
index 0f00a6652b..8000bad3a4 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -2092,7 +2092,7 @@ static const AVOption options[] = {
     { "rtp_history",        "The number of RTP history items to store",                 OFFSET(hist_sz),            AV_OPT_TYPE_INT,    { .i64 = WHIP_RTP_HISTORY_DEFAULT }, WHIP_RTP_HISTORY_MIN, WHIP_RTP_HISTORY_MAX, ENC },
     { "authorization",      "The optional Bearer token for WHIP Authorization",         OFFSET(authorization),      AV_OPT_TYPE_STRING, { .str = NULL },     0,       0, ENC },
     { "cert_file",          "The optional certificate file path for DTLS",              OFFSET(cert_file),          AV_OPT_TYPE_STRING, { .str = NULL },     0,       0, ENC },
-    { "key_file",           "The optional private key file path for DTLS",              OFFSET(key_file),      AV_OPT_TYPE_STRING, { .str = NULL },     0,       0, ENC },
+    { "key_file",           "The optional private key file path for DTLS",              OFFSET(key_file),           AV_OPT_TYPE_STRING, { .str = NULL },     0,       0, ENC },
     { NULL },
 };
 
-- 
2.49.1

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org