* [FFmpeg-devel] [PATCH v3 1/6] avformat/whip: add whip_flags ignore_ipv6 to skip IPv6 ICE candidates
2025-07-12 13:29 [FFmpeg-devel] [PATCH v3 0/6] avformat/whip: Add NACK and RTX support Jack Lau
@ 2025-07-12 13:29 ` Jack Lau
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 2/6] avformat/whip: fix typos Jack Lau
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Jack Lau @ 2025-07-12 13:29 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Jack Lau
mark this ignore_ipv6 flag could ignore any IPv6 ICE candidate,
preventing “No route to host” errors on devices without IPv6 connectivity.
Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
libavformat/whip.c | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/libavformat/whip.c b/libavformat/whip.c
index e272254a6f..e27a948b76 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -193,9 +193,14 @@ enum WHIPState {
WHIP_STATE_FAILED,
};
+typedef enum WHIPFlags {
+ WHIP_FLAG_IGNORE_IPV6 = (1 << 0) // Ignore ipv6 candidate
+} WHIPFlags;
+
typedef struct WHIPContext {
AVClass *av_class;
+ uint32_t flags; // enum WHIPFlags
/* The state of the RTC connection. */
enum WHIPState state;
/* The callback return value for DTLS. */
@@ -884,6 +889,9 @@ static int parse_answer(AVFormatContext *s)
if (ptr && av_stristr(ptr, "host")) {
char protocol[17], host[129];
int priority, port;
+#if HAVE_STRUCT_SOCKADDR_IN6
+ struct in6_addr addr6;
+#endif
ret = sscanf(ptr, "%16s %d %128s %d typ host", protocol, &priority, host, &port);
if (ret != 4) {
av_log(whip, AV_LOG_ERROR, "Failed %d to parse line %d %s from %s\n",
@@ -891,7 +899,12 @@ static int parse_answer(AVFormatContext *s)
ret = AVERROR(EIO);
goto end;
}
-
+#if HAVE_STRUCT_SOCKADDR_IN6
+ if (whip->flags & WHIP_FLAG_IGNORE_IPV6 && inet_pton(AF_INET6, host, &addr6) == 1) {
+ av_log(whip, AV_LOG_DEBUG, "Ignoring IPv6 ICE candidates %s, line %d %s \n", host, i, line);
+ continue;
+ }
+#endif
if (av_strcasecmp(protocol, "udp")) {
av_log(whip, AV_LOG_ERROR, "Protocol %s is not supported by RTC, choose udp, line %d %s of %s\n",
protocol, i, line, whip->sdp_answer);
@@ -1900,6 +1913,8 @@ static const AVOption options[] = {
{ "authorization", "The optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
{ "cert_file", "The optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
{ "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
+ { "whip_flags", "Set flags affecting WHIP connection behavior", OFFSET(flags), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, 0, UINT_MAX, ENC, .unit = "flags" },
+ { "ignore_ipv6", "(Optional) Ignore any IPv6 ICE candidate", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_IGNORE_IPV6 }, 0, UINT_MAX, ENC, .unit = "flags" },
{ NULL },
};
--
2.49.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread* [FFmpeg-devel] [PATCH v3 2/6] avformat/whip: fix typos
2025-07-12 13:29 [FFmpeg-devel] [PATCH v3 0/6] avformat/whip: Add NACK and RTX support Jack Lau
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 1/6] avformat/whip: add whip_flags ignore_ipv6 to skip IPv6 ICE candidates Jack Lau
@ 2025-07-12 13:29 ` Jack Lau
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 3/6] avformat/whip: fix H264 profile_iop bit map for SDP Jack Lau
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Jack Lau @ 2025-07-12 13:29 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Jack Lau
Remove redundant "WHIP: " prefix in log context
since it already add whip context.
Fix grammers in whip options descriptions
Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
libavformat/whip.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/libavformat/whip.c b/libavformat/whip.c
index e27a948b76..0be976edc6 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -1910,11 +1910,11 @@ static int whip_check_bitstream(AVFormatContext *s, AVStream *st, const AVPacket
static const AVOption options[] = {
{ "handshake_timeout", "Timeout in milliseconds for ICE and DTLS handshake.", OFFSET(handshake_timeout), AV_OPT_TYPE_INT, { .i64 = 5000 }, -1, INT_MAX, ENC },
{ "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, ENC },
- { "authorization", "The optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
- { "cert_file", "The optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
- { "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
+ { "authorization", "Optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
+ { "cert_file", "Optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
+ { "key_file", "Optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
{ "whip_flags", "Set flags affecting WHIP connection behavior", OFFSET(flags), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, 0, UINT_MAX, ENC, .unit = "flags" },
- { "ignore_ipv6", "(Optional) Ignore any IPv6 ICE candidate", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_IGNORE_IPV6 }, 0, UINT_MAX, ENC, .unit = "flags" },
+ { "ignore_ipv6", "Ignore any IPv6 ICE candidate", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_IGNORE_IPV6 }, 0, UINT_MAX, ENC, .unit = "flags" },
{ NULL },
};
--
2.49.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread* [FFmpeg-devel] [PATCH v3 3/6] avformat/whip: fix H264 profile_iop bit map for SDP
2025-07-12 13:29 [FFmpeg-devel] [PATCH v3 0/6] avformat/whip: Add NACK and RTX support Jack Lau
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 1/6] avformat/whip: add whip_flags ignore_ipv6 to skip IPv6 ICE candidates Jack Lau
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 2/6] avformat/whip: fix typos Jack Lau
@ 2025-07-12 13:29 ` Jack Lau
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 4/6] WHIP: X509 cert serial number should be positive Jack Lau
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Jack Lau @ 2025-07-12 13:29 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Jack Lau
AVCodecParameters::profile only contains constraint_set1_flag
(AV_PROFILE_H264_CONSTRAINED 1<<9).
So add H264 constraints flag fully parse refer to hlsenc
write_codec_attr
Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
libavformat/whip.c | 47 ++++++++++++++++------------------------------
1 file changed, 16 insertions(+), 31 deletions(-)
diff --git a/libavformat/whip.c b/libavformat/whip.c
index 0be976edc6..f1f8d1b4ad 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -210,6 +210,7 @@ typedef struct WHIPContext {
/* Parameters for the input audio and video codecs. */
AVCodecParameters *audio_par;
AVCodecParameters *video_par;
+ uint8_t constraint_set_flags;
/**
* The h264_mp4toannexb Bitstream Filter (BSF) bypasses the AnnexB packet;
@@ -450,45 +451,30 @@ static av_cold int initialize(AVFormatContext *s)
static int parse_profile_level(AVFormatContext *s, AVCodecParameters *par)
{
int ret = 0;
- const uint8_t *r = par->extradata, *r1, *end = par->extradata + par->extradata_size;
- H264SPS seq, *const sps = &seq;
- uint32_t state;
+ const uint8_t *r = par->extradata;
WHIPContext *whip = s->priv_data;
if (par->codec_id != AV_CODEC_ID_H264)
return ret;
- if (par->profile != AV_PROFILE_UNKNOWN && par->level != AV_LEVEL_UNKNOWN)
- return ret;
-
if (!par->extradata || par->extradata_size <= 0) {
av_log(whip, AV_LOG_ERROR, "Unable to parse profile from empty extradata=%p, size=%d\n",
par->extradata, par->extradata_size);
return AVERROR(EINVAL);
}
- while (1) {
- r = avpriv_find_start_code(r, end, &state);
- if (r >= end)
- break;
-
- r1 = ff_nal_find_startcode(r, end);
- if ((state & 0x1f) == H264_NAL_SPS) {
- ret = ff_avc_decode_sps(sps, r, r1 - r);
- if (ret < 0) {
- av_log(whip, AV_LOG_ERROR, "Failed to decode SPS, state=%x, size=%d\n",
- state, (int)(r1 - r));
- return ret;
- }
-
- av_log(whip, AV_LOG_VERBOSE, "Parse profile=%d, level=%d from SPS\n",
- sps->profile_idc, sps->level_idc);
- par->profile = sps->profile_idc;
- par->level = sps->level_idc;
- }
+ if (AV_RB32(r) == 0x00000001 && (r[4] & 0x1F) == 7)
+ r = &r[5];
+ else if (AV_RB24(r) == 0x000001 && (r[3] & 0x1F) == 7)
+ r = &r[4];
+ else if (r[0] == 0x01) // avcC
+ r = &r[1];
+ else
+ return AVERROR(EINVAL);
- r = r1;
- }
+ if (par->profile == AV_PROFILE_UNKNOWN) par->profile = r[0];
+ whip->constraint_set_flags = r[1];
+ if (par->level == AV_LEVEL_UNKNOWN) par->level = r[2];
return ret;
}
@@ -599,7 +585,7 @@ static int parse_codec(AVFormatContext *s)
*/
static int generate_sdp_offer(AVFormatContext *s)
{
- int ret = 0, profile, level, profile_iop;
+ int ret = 0, profile, level;
const char *acodec_name = NULL, *vcodec_name = NULL;
AVBPrint bp;
WHIPContext *whip = s->priv_data;
@@ -667,11 +653,10 @@ static int generate_sdp_offer(AVFormatContext *s)
}
if (whip->video_par) {
- profile_iop = profile = whip->video_par->profile;
+ profile = whip->video_par->profile;
level = whip->video_par->level;
if (whip->video_par->codec_id == AV_CODEC_ID_H264) {
vcodec_name = "H264";
- profile_iop &= AV_PROFILE_H264_CONSTRAINED;
profile &= (~AV_PROFILE_H264_CONSTRAINED);
}
@@ -699,7 +684,7 @@ static int generate_sdp_offer(AVFormatContext *s)
vcodec_name,
whip->video_payload_type,
profile,
- profile_iop,
+ whip->constraint_set_flags,
level,
whip->video_ssrc,
whip->video_ssrc);
--
2.49.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread* [FFmpeg-devel] [PATCH v3 4/6] WHIP: X509 cert serial number should be positive.
2025-07-12 13:29 [FFmpeg-devel] [PATCH v3 0/6] avformat/whip: Add NACK and RTX support Jack Lau
` (2 preceding siblings ...)
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 3/6] avformat/whip: fix H264 profile_iop bit map for SDP Jack Lau
@ 2025-07-12 13:29 ` Jack Lau
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 5/6] avformat/whip: implement NACK and RTX suppport Jack Lau
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 6/6] avformat/whip: reindent whip options Jack Lau
5 siblings, 0 replies; 7+ messages in thread
From: Jack Lau @ 2025-07-12 13:29 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: winlin, Jack Lau
From: winlin <winlinvip@gmail.com>
See RFC5280 4.1.2.2
Co-authored-by: Jack Lau <jacklau1222@qq.com>
Signed-off-by: winlin <winlinvip@gmail.com>
---
libavformat/tls_openssl.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c
index 2a01fb387d..285ea166ac 100644
--- a/libavformat/tls_openssl.c
+++ b/libavformat/tls_openssl.c
@@ -311,7 +311,8 @@ static int openssl_gen_certificate(EVP_PKEY *pkey, X509 **cert, char **fingerpri
goto enomem_end;
}
- serial = (int)av_get_random_seed();
+ // According to RFC5280 4.1.2.2, The serial number MUST be a positive integer
+ serial = (int)(av_get_random_seed() & 0x7FFFFFFF);
if (ASN1_INTEGER_set(X509_get_serialNumber(*cert), serial) != 1) {
av_log(NULL, AV_LOG_ERROR, "TLS: Failed to set serial, %s\n", ERR_error_string(ERR_get_error(), NULL));
goto einval_end;
--
2.49.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread* [FFmpeg-devel] [PATCH v3 5/6] avformat/whip: implement NACK and RTX suppport
2025-07-12 13:29 [FFmpeg-devel] [PATCH v3 0/6] avformat/whip: Add NACK and RTX support Jack Lau
` (3 preceding siblings ...)
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 4/6] WHIP: X509 cert serial number should be positive Jack Lau
@ 2025-07-12 13:29 ` Jack Lau
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 6/6] avformat/whip: reindent whip options Jack Lau
5 siblings, 0 replies; 7+ messages in thread
From: Jack Lau @ 2025-07-12 13:29 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Sergio Garcia Murillo, Jack Lau
RTP retransmission described in RFC4588 (RTX) is an effective packet
loss recovery technique for real-time applications with relaxed delay bounds.
This patch provides a minimal implementation for RTX and RTCP NACK (RFC3940)
and its associated SDP signaling and negotiation.
Co-authored-by: Sergio Garcia Murillo <sergio.garcia.murillo@gmail.com>
Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
libavformat/whip.c | 206 ++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 202 insertions(+), 4 deletions(-)
diff --git a/libavformat/whip.c b/libavformat/whip.c
index f1f8d1b4ad..d954e80830 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -114,6 +114,7 @@
/* Referring to Chrome's definition of RTP payload types. */
#define WHIP_RTP_PAYLOAD_TYPE_H264 106
#define WHIP_RTP_PAYLOAD_TYPE_OPUS 111
+#define WHIP_RTP_PAYLOAD_TYPE_RTX 105
/**
* The STUN message header, which is 20 bytes long, comprises the
@@ -150,6 +151,11 @@
#define WHIP_SDP_SESSION_ID "4489045141692799359"
#define WHIP_SDP_CREATOR_IP "127.0.0.1"
+/**
+ * Retransmission / NACK support
+*/
+#define HISTORY_SIZE_DEFAULT 512
+
/* Calculate the elapsed time from starttime to endtime in milliseconds. */
#define ELAPSED(starttime, endtime) ((int)(endtime - starttime) / 1000)
@@ -194,9 +200,16 @@ enum WHIPState {
};
typedef enum WHIPFlags {
- WHIP_FLAG_IGNORE_IPV6 = (1 << 0) // Ignore ipv6 candidate
+ WHIP_FLAG_IGNORE_IPV6 = (1 << 0), // Ignore ipv6 candidate
+ WHIP_FLAG_DISABLE_RTX = (1 << 1) // Enable NACK and RTX
} WHIPFlags;
+typedef struct RtpHistoryItem {
+ uint16_t seq; // original RTP seq
+ int size; // length in bytes
+ uint8_t* buf; // malloc-ed copy
+} RtpHistoryItem;
+
typedef struct WHIPContext {
AVClass *av_class;
@@ -285,6 +298,7 @@ typedef struct WHIPContext {
/* The SRTP send context, to encrypt outgoing packets. */
SRTPContext srtp_audio_send;
SRTPContext srtp_video_send;
+ SRTPContext srtp_video_rtx_send;
SRTPContext srtp_rtcp_send;
/* The SRTP receive context, to decrypt incoming packets. */
SRTPContext srtp_recv;
@@ -309,6 +323,14 @@ typedef struct WHIPContext {
/* The certificate and private key used for DTLS handshake. */
char* cert_file;
char* key_file;
+
+ /* RTX and NACK */
+ uint8_t rtx_payload_type;
+ uint32_t video_rtx_ssrc;
+ uint16_t rtx_seq;
+ int history_size;
+ RtpHistoryItem *history; /* ring buffer */
+ int hist_head;
} WHIPContext;
/**
@@ -611,6 +633,16 @@ static int generate_sdp_offer(AVFormatContext *s)
whip->audio_payload_type = WHIP_RTP_PAYLOAD_TYPE_OPUS;
whip->video_payload_type = WHIP_RTP_PAYLOAD_TYPE_H264;
+ /* RTX and NACK init */
+ whip->rtx_payload_type = WHIP_RTP_PAYLOAD_TYPE_RTX;
+ whip->video_rtx_ssrc = av_lfg_get(&whip->rnd);
+ whip->rtx_seq = 0;
+ whip->hist_head = 0;
+ whip->history_size = FFMAX(64, whip->history_size);
+ whip->history = av_calloc(whip->history_size, sizeof(*whip->history));
+ if (!whip->history)
+ return AVERROR(ENOMEM);
+
av_bprintf(&bp, ""
"v=0\r\n"
"o=FFmpeg %s 2 IN IP4 %s\r\n"
@@ -661,7 +693,7 @@ static int generate_sdp_offer(AVFormatContext *s)
}
av_bprintf(&bp, ""
- "m=video 9 UDP/TLS/RTP/SAVPF %u\r\n"
+ "m=video 9 UDP/TLS/RTP/SAVPF %u %u\r\n"
"c=IN IP4 0.0.0.0\r\n"
"a=ice-ufrag:%s\r\n"
"a=ice-pwd:%s\r\n"
@@ -674,9 +706,16 @@ static int generate_sdp_offer(AVFormatContext *s)
"a=rtcp-rsize\r\n"
"a=rtpmap:%u %s/90000\r\n"
"a=fmtp:%u level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=%02x%02x%02x\r\n"
+ "a=rtcp-fb:%u nack\r\n"
+ "a=rtpmap:%u rtx/90000\r\n"
+ "a=fmtp:%u apt=%u\r\n"
+ "a=ssrc-group:FID %u %u\r\n"
+ "a=ssrc:%u cname:FFmpeg\r\n"
+ "a=ssrc:%u msid:FFmpeg video\r\n"
"a=ssrc:%u cname:FFmpeg\r\n"
"a=ssrc:%u msid:FFmpeg video\r\n",
whip->video_payload_type,
+ whip->rtx_payload_type,
whip->ice_ufrag_local,
whip->ice_pwd_local,
whip->dtls_fingerprint,
@@ -686,8 +725,16 @@ static int generate_sdp_offer(AVFormatContext *s)
profile,
whip->constraint_set_flags,
level,
+ whip->video_payload_type,
+ whip->rtx_payload_type,
+ whip->rtx_payload_type,
+ whip->video_payload_type,
+ whip->video_ssrc,
+ whip->video_rtx_ssrc,
whip->video_ssrc,
- whip->video_ssrc);
+ whip->video_ssrc,
+ whip->video_rtx_ssrc,
+ whip->video_rtx_ssrc);
}
if (!av_bprint_is_complete(&bp)) {
@@ -1400,6 +1447,12 @@ static int setup_srtp(AVFormatContext *s)
goto end;
}
+ ret = ff_srtp_set_crypto(&whip->srtp_video_rtx_send, suite, buf);
+ if (ret < 0) {
+ av_log(whip, AV_LOG_ERROR, "Failed to set crypto for video rtx send\n");
+ goto end;
+ }
+
ret = ff_srtp_set_crypto(&whip->srtp_rtcp_send, suite, buf);
if (ret < 0) {
av_log(whip, AV_LOG_ERROR, "Failed to set crypto for rtcp send\n");
@@ -1429,6 +1482,38 @@ end:
return ret;
}
+
+/**
+ * RTX history helpers
+ */
+ static int rtp_history_store(WHIPContext *whip, const uint8_t *buf, int size)
+{
+ int pos = whip->hist_head % whip->history_size;
+ RtpHistoryItem *it = &whip->history[pos];
+ /* free older entry */
+ av_free(it->buf);
+ it->buf = av_malloc(size);
+ if (!it->buf)
+ return AVERROR(ENOMEM);
+
+ memcpy(it->buf, buf, size);
+ it->size = size;
+ it->seq = AV_RB16(buf + 2);
+
+ whip->hist_head = ++pos;
+ return 0;
+}
+
+static const RtpHistoryItem *rtp_history_find(const WHIPContext *whip, uint16_t seq)
+{
+ for (int i = 0; i < whip->history_size; i++) {
+ const RtpHistoryItem *it = &whip->history[i];
+ if (it->buf && it->seq == seq)
+ return it;
+ }
+ return NULL;
+}
+
/**
* Callback triggered by the RTP muxer when it creates and sends out an RTP packet.
*
@@ -1465,6 +1550,12 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size)
return 0;
}
+ /* Store only ORIGINAL video packets (non-RTX, non-RTCP) */
+ if (!is_rtcp && is_video) {
+ ret = rtp_history_store(whip, buf, buf_size);
+ if (ret < 0) return ret;
+ }
+
ret = ffurl_write(whip->udp, whip->buf, cipher_size);
if (ret < 0) {
av_log(whip, AV_LOG_ERROR, "Failed to write packet=%dB, ret=%d\n", cipher_size, ret);
@@ -1473,6 +1564,48 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size)
return ret;
}
+/**
+ * See https://datatracker.ietf.org/doc/html/rfc4588
+ * Build and send a single RTX packet
+ */
+static int send_rtx_packet(AVFormatContext *s, const uint8_t *orig_pkt_buf, int orig_size)
+{
+ int ret;
+ WHIPContext *whip = s->priv_data;
+ int new_size, cipher_size;
+ if (whip->flags & WHIP_FLAG_DISABLE_RTX)
+ return 0;
+
+ /* allocate new buffer: header + 2 + payload */
+ if (orig_size + 2 > sizeof(whip->buf))
+ return 0;
+
+ memcpy(whip->buf, orig_pkt_buf, orig_size);
+
+ uint8_t *hdr = whip->buf;
+ uint16_t orig_seq = AV_RB16(hdr + 2);
+
+ /* rewrite header */
+ hdr[1] = (hdr[1] & 0x80) | whip->rtx_payload_type; /* keep M bit */
+ AV_WB16(hdr + 2, whip->rtx_seq++);
+ AV_WB32(hdr + 8, whip->video_rtx_ssrc);
+
+ /* shift payload 2 bytes */
+ memmove(hdr + 12 + 2, hdr + 12, orig_size - 12);
+ AV_WB16(hdr + 12, orig_seq);
+
+ new_size = orig_size + 2;
+
+ /* Encrypt by SRTP and send out. */
+ cipher_size = ff_srtp_encrypt(&whip->srtp_video_rtx_send, whip->buf, new_size, whip->buf, sizeof(whip->buf));
+ if (cipher_size <= 0 || cipher_size < new_size) {
+ av_log(whip, AV_LOG_WARNING, "Failed to encrypt packet=%dB, cipher=%dB\n", new_size, cipher_size);
+ return 0;
+ }
+ ret = ffurl_write(whip->udp, whip->buf, cipher_size);
+ if (ret <= 0) av_log(whip, AV_LOG_ERROR, "Failed to send RTX packet\n");
+ return ret;
+}
/**
* Creates dedicated RTP muxers for each stream in the AVFormatContext to build RTP
@@ -1780,6 +1913,7 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt)
WHIPContext *whip = s->priv_data;
AVStream *st = s->streams[pkt->stream_index];
AVFormatContext *rtp_ctx = st->priv_data;
+ uint8_t *buf = NULL;
/* TODO: Send binding request every 1s as WebRTC heartbeat. */
@@ -1795,11 +1929,72 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt)
goto end;
}
}
+ /**
+ * Handle RTCP NACK
+ * Refer to RFC 4585, Section 6.2.1
+ * The Generic NACK message is identified by PT=RTPFB and FMT=1.
+ * TODO: disable retransmisstion when "-tune zerolatency"
+ */
+ if (media_is_rtcp(whip->buf, ret)) {
+ int ptr = 0;
+ uint8_t pt = whip->buf[ptr + 1];
+ uint8_t fmt = (whip->buf[ptr] & 0x1f);
+ if (ptr + 4 <= ret && pt == 205 && fmt == 1) {
+ /**
+ * Refer to RFC 3550, Section 6.4.1.
+ * The length of this RTCP packet in 32-bit words minus one,
+ * including the header and any padding.
+ */
+ int rtcp_len = (AV_RB16(&whip->buf[ptr + 2]) + 1) * 4;
+ /* SRTCP index(4 bytes) + HMAC (SRTP_AES128_CM_SHA1_80 10bytes) */
+ int srtcp_len = rtcp_len + 4 + 10;
+ if (srtcp_len == ret && rtcp_len >= 12) {
+ int i = 0;
+ buf = av_malloc(srtcp_len);
+ if (!buf) return AVERROR(ENOMEM);
+ memcpy(buf, whip->buf, srtcp_len);
+ int ret = ff_srtp_decrypt(&whip->srtp_recv, buf, &srtcp_len);
+ if (ret < 0) {
+ av_log(whip, AV_LOG_ERROR, "NACK packet(SRTCP) decrypt failed: %d, Can't send RTX packet\n", ret);
+ goto write_packet;
+ }
+ while (12 + i < rtcp_len) {
+ /**
+ * See https://datatracker.ietf.org/doc/html/rfc4585#section-6.1
+ * Handle multi NACKs in bundled packet.
+ */
+ uint16_t pid = AV_RB16(&buf[ptr + 12 + i]);
+ uint16_t blp = AV_RB16(&buf[ptr + 14 + i]);
+
+ /* retransmit pid + any bit set in blp */
+ for (int bit = -1; bit < 16; bit++) {
+ uint16_t seq = (bit < 0) ? pid : pid + bit + 1;
+ if (bit >= 0 && !(blp & (1 << bit)))
+ continue;
+
+ const RtpHistoryItem *it = rtp_history_find(whip, seq);
+ if (it) {
+ av_log(whip, AV_LOG_VERBOSE,
+ "NACK, packet found: size: %d, seq=%d, rtx size=%d, lateset stored packet seq:%d\n",
+ it->size, seq, ret, whip->history[whip->hist_head-1].seq);
+ send_rtx_packet(s, it->buf, it->size);
+ } else {
+ av_log(whip, AV_LOG_VERBOSE,
+ "NACK, packet not found, seq=%d, latest stored packet seq: %d, latest rtx seq: %d\n",
+ seq, whip->history[whip->hist_head-1].seq, whip->rtx_seq);
+ }
+ }
+ i = i + 4;
+ }
+ av_freep(&buf);
+ }
+ }
+ }
} else if (ret != AVERROR(EAGAIN)) {
av_log(whip, AV_LOG_ERROR, "Failed to read from UDP socket\n");
goto end;
}
-
+write_packet:
if (whip->h264_annexb_insert_sps_pps && st->codecpar->codec_id == AV_CODEC_ID_H264) {
if ((ret = h264_annexb_insert_sps_pps(s, pkt)) < 0) {
av_log(whip, AV_LOG_ERROR, "Failed to insert SPS/PPS before IDR\n");
@@ -1818,6 +2013,7 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt)
}
end:
+ if (buf) av_freep(&buf);
if (ret < 0 && whip->state < WHIP_STATE_FAILED)
whip->state = WHIP_STATE_FAILED;
if (ret >= 0 && whip->state >= WHIP_STATE_FAILED && whip->dtls_ret < 0)
@@ -1900,6 +2096,8 @@ static const AVOption options[] = {
{ "key_file", "Optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
{ "whip_flags", "Set flags affecting WHIP connection behavior", OFFSET(flags), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, 0, UINT_MAX, ENC, .unit = "flags" },
{ "ignore_ipv6", "Ignore any IPv6 ICE candidate", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_IGNORE_IPV6 }, 0, UINT_MAX, ENC, .unit = "flags" },
+ { "disable_rtx", "Disable RFC 4588 RTX", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_DISABLE_RTX }, 0, UINT_MAX, ENC, .unit = "flags" },
+ { "rtx_history_size", "Packet history size", OFFSET(history_size), AV_OPT_TYPE_INT, { .i64 = HISTORY_SIZE_DEFAULT }, 64, 2048, ENC },
{ NULL },
};
--
2.49.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread* [FFmpeg-devel] [PATCH v3 6/6] avformat/whip: reindent whip options
2025-07-12 13:29 [FFmpeg-devel] [PATCH v3 0/6] avformat/whip: Add NACK and RTX support Jack Lau
` (4 preceding siblings ...)
2025-07-12 13:29 ` [FFmpeg-devel] [PATCH v3 5/6] avformat/whip: implement NACK and RTX suppport Jack Lau
@ 2025-07-12 13:29 ` Jack Lau
5 siblings, 0 replies; 7+ messages in thread
From: Jack Lau @ 2025-07-12 13:29 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Jack Lau
Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
libavformat/whip.c | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/libavformat/whip.c b/libavformat/whip.c
index d954e80830..f7eb6e1323 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -2089,13 +2089,13 @@ static int whip_check_bitstream(AVFormatContext *s, AVStream *st, const AVPacket
#define OFFSET(x) offsetof(WHIPContext, x)
#define ENC AV_OPT_FLAG_ENCODING_PARAM
static const AVOption options[] = {
- { "handshake_timeout", "Timeout in milliseconds for ICE and DTLS handshake.", OFFSET(handshake_timeout), AV_OPT_TYPE_INT, { .i64 = 5000 }, -1, INT_MAX, ENC },
- { "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, ENC },
- { "authorization", "Optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
- { "cert_file", "Optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
- { "key_file", "Optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
- { "whip_flags", "Set flags affecting WHIP connection behavior", OFFSET(flags), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, 0, UINT_MAX, ENC, .unit = "flags" },
- { "ignore_ipv6", "Ignore any IPv6 ICE candidate", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_IGNORE_IPV6 }, 0, UINT_MAX, ENC, .unit = "flags" },
+ { "handshake_timeout", "Timeout in milliseconds for ICE and DTLS handshake.", OFFSET(handshake_timeout), AV_OPT_TYPE_INT, { .i64 = 5000 }, -1, INT_MAX, ENC },
+ { "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, ENC },
+ { "authorization", "Optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
+ { "cert_file", "Optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
+ { "key_file", "Optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
+ { "whip_flags", "Set flags affecting WHIP connection behavior", OFFSET(flags), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, 0, 0, ENC, .unit = "flags" },
+ { "ignore_ipv6", "Ignore any IPv6 ICE candidate", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_IGNORE_IPV6 }, 0, UINT_MAX, ENC, .unit = "flags" },
{ "disable_rtx", "Disable RFC 4588 RTX", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_DISABLE_RTX }, 0, UINT_MAX, ENC, .unit = "flags" },
{ "rtx_history_size", "Packet history size", OFFSET(history_size), AV_OPT_TYPE_INT, { .i64 = HISTORY_SIZE_DEFAULT }, 64, 2048, ENC },
{ NULL },
--
2.49.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread