[FFmpeg-devel] [PATCH v4 11/11] avformat/whip: simplify and modularize the ICE and DTLS

[Jack Lau <jacklau1222gm@gmail.com>]

We try to perform dtls handshake when the ICE is totally done.

Refer to RFC8445,
When peer's ICE is lite, the peer won't trigged check so FFmpeg just send
STUN request and receive response, then ICE is done.
When peer's ICE is full, the peer will send STUN request after reponse
FFmpeg's request to ensure candidate pair become valid in both directions.
Then the peer does nomination, and ICE is done.

Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
 libavformat/whip.c | 135 +++++++++++++++++++++------------------------
 1 file changed, 62 insertions(+), 73 deletions(-)

diff --git a/libavformat/whip.c b/libavformat/whip.c
index 4c8ed26cef..cfcb8e8888 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -373,19 +373,6 @@ static av_cold int certificate_key_init(AVFormatContext *s)
     return ret;
 }
 
-static av_cold int dtls_initialize(AVFormatContext *s)
-{
-    WHIPContext *whip = s->priv_data;
-    /* reuse the udp created by whip */
-    ff_tls_set_external_socket(whip->dtls_uc, whip->udp);
-
-    /* Make the socket non-blocking */
-    ff_socket_nonblock(ffurl_get_file_handle(whip->dtls_uc), 1);
-    whip->dtls_uc->flags |= AVIO_FLAG_NONBLOCK;
-
-    return 0;
-}
-
 /**
  * Initialize and check the options for the WebRTC muxer.
  */
@@ -1232,14 +1219,12 @@ end:
     return ret;
 }
 
-static int ice_dtls_handshake(AVFormatContext *s)
+static int ice_handshake(AVFormatContext *s)
 {
     int ret = 0, size, i;
     int64_t starttime = av_gettime(), now;
     WHIPContext *whip = s->priv_data;
     int is_dtls_active = whip->flags & WHIP_FLAG_DTLS_ACTIVE;
-    AVDictionary *opts = NULL;
-    char buf[256], *cert_buf = NULL, *key_buf = NULL;
 
     if (whip->state < WHIP_STATE_UDP_CONNECTED || !whip->udp) {
         av_log(whip, AV_LOG_ERROR, "UDP not connected, state=%d, udp=%p\n", whip->state, whip->udp);
@@ -1261,25 +1246,20 @@ static int ice_dtls_handshake(AVFormatContext *s)
                 goto end;
             }
 
-            if (whip->state < WHIP_STATE_ICE_CONNECTING)
-                whip->state = WHIP_STATE_ICE_CONNECTING;
+            whip->state = WHIP_STATE_ICE_CONNECTING;
         }
 
 next_packet:
-        if (whip->state >= WHIP_STATE_DTLS_FINISHED)
-            /* DTLS handshake is done, exit the loop. */
-            break;
-
         now = av_gettime();
         if (now - starttime >= whip->handshake_timeout * 1000) {
-            av_log(whip, AV_LOG_ERROR, "DTLS handshake timeout=%dms, cost=%dms, elapsed=%dms, state=%d\n",
+            av_log(whip, AV_LOG_ERROR, "ICE handshake timeout=%dms, cost=%dms, elapsed=%dms, state=%d\n",
                 whip->handshake_timeout, ELAPSED(starttime, now), ELAPSED(whip->whip_starttime, now), whip->state);
             ret = AVERROR(ETIMEDOUT);
             goto end;
         }
 
-        /* Read the STUN or DTLS messages from peer. */
-        for (i = 0; i < ICE_DTLS_READ_INTERVAL / 5 && whip->state < WHIP_STATE_ICE_CONNECTED; i++) {
+        /* Read the STUN or DTLS client hello from peer. */
+        for (i = 0; i < ICE_DTLS_READ_INTERVAL / 5; i++) {
             ret = ffurl_read(whip->udp, whip->buf, sizeof(whip->buf));
             if (ret > 0)
                 break;
@@ -1295,35 +1275,8 @@ next_packet:
 
         /* Handle the ICE binding response. */
         if (ice_is_binding_response(whip->buf, ret)) {
-            if (whip->state < WHIP_STATE_ICE_CONNECTED) {
-                if (whip->is_peer_ice_lite)
-                    whip->state = WHIP_STATE_ICE_CONNECTED;
-                whip->whip_ice_time = av_gettime();
-                av_log(whip, AV_LOG_VERBOSE, "ICE STUN ok, state=%d, url=udp://%s:%d, location=%s, username=%s:%s, res=%dB, elapsed=%dms\n",
-                    whip->state, whip->ice_host, whip->ice_port, whip->whip_resource_url ? whip->whip_resource_url : "",
-                    whip->ice_ufrag_remote, whip->ice_ufrag_local, ret, ELAPSED(whip->whip_starttime, av_gettime()));
-
-                ff_url_join(buf, sizeof(buf), "dtls", NULL, whip->ice_host, whip->ice_port, NULL);
-                av_dict_set_int(&opts, "mtu", whip->pkt_size, 0);
-                if (whip->cert_file) {
-                    av_dict_set(&opts, "cert_file", whip->cert_file, 0);
-                } else
-                    av_dict_set(&opts, "cert_pem", whip->cert_buf, 0);
-
-                if (whip->key_file) {
-                    av_dict_set(&opts, "key_file", whip->key_file, 0);
-                } else
-                    av_dict_set(&opts, "key_pem", whip->key_buf, 0);
-                av_dict_set_int(&opts, "external_sock", 1, 0);
-                av_dict_set_int(&opts, "listen", is_dtls_active ? 0 : 1, 0);
-                /* If got the first binding response, start DTLS handshake. */
-                ret = ffurl_open_whitelist(&whip->dtls_uc, buf, AVIO_FLAG_READ_WRITE, &s->interrupt_callback,
-                    &opts, s->protocol_whitelist, s->protocol_blacklist, NULL);
-                av_dict_free(&opts);
-                if (ret < 0)
-                    goto end;
-                dtls_initialize(s);
-            }
+            if (whip->is_peer_ice_lite)
+                whip->state = WHIP_STATE_ICE_CONNECTED;
             goto next_packet;
         }
 
@@ -1334,29 +1287,62 @@ next_packet:
             goto next_packet;
         }
 
-        if ((is_dtls_packet(whip->buf, ret) || is_dtls_active) && whip->state >= WHIP_STATE_ICE_CONNECTED || whip->state == WHIP_STATE_ICE_CONNECTING) {
+        if (is_dtls_packet(whip->buf, ret) || whip->flags & WHIP_FLAG_DTLS_ACTIVE) {
             whip->state = WHIP_STATE_ICE_CONNECTED;
-            ret = ffurl_handshake(whip->dtls_uc);
-            if (ret < 0) {
-                whip->state = WHIP_STATE_FAILED;
-                av_log(whip, AV_LOG_VERBOSE, "DTLS session failed\n");
-                goto end;
-            }
-            if (!ret) {
-                whip->state = WHIP_STATE_DTLS_FINISHED;
-                whip->whip_dtls_time = av_gettime();
-                av_log(whip, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%dms\n",
-                    ELAPSED(whip->whip_starttime, whip->whip_dtls_time));
-            }
-            goto next_packet;
+            ret = 0;
+            whip->whip_ice_time = av_gettime();
+            av_log(whip, AV_LOG_VERBOSE, "ICE STUN ok, state=%d, url=udp://%s:%d, location=%s, username=%s:%s, res=%dB, elapsed=%dms\n",
+                whip->state, whip->ice_host, whip->ice_port, whip->whip_resource_url ? whip->whip_resource_url : "",
+                whip->ice_ufrag_remote, whip->ice_ufrag_local, ret, ELAPSED(whip->whip_starttime, av_gettime()));
+            break;
         }
     }
+end:
+    return ret;
+}
 
+static int dtls_handshake(AVFormatContext *s)
+{
+    int ret = 0;
+    WHIPContext *whip = s->priv_data;
+    AVDictionary *opts = NULL;
+    char buf[256];
+
+    ff_url_join(buf, sizeof(buf), "dtls", NULL, whip->ice_host, whip->ice_port, NULL);
+    av_dict_set_int(&opts, "mtu", whip->pkt_size, 0);
+    if (whip->cert_file) {
+        av_dict_set(&opts, "cert_file", whip->cert_file, 0);
+    } else
+        av_dict_set(&opts, "cert_pem", whip->cert_buf, 0);
+
+    if (whip->key_file) {
+        av_dict_set(&opts, "key_file", whip->key_file, 0);
+    } else
+        av_dict_set(&opts, "key_pem", whip->key_buf, 0);
+    av_dict_set_int(&opts, "external_sock", 1, 0);
+    av_dict_set_int(&opts, "listen", whip->flags & WHIP_FLAG_DTLS_ACTIVE ? 0 : 1, 0);
+    /* If got the first binding response, start DTLS handshake. */
+    ret = ffurl_open_whitelist(&whip->dtls_uc, buf, AVIO_FLAG_READ_WRITE, &s->interrupt_callback,
+        &opts, s->protocol_whitelist, s->protocol_blacklist, NULL);
+    av_dict_free(&opts);
+    if (ret < 0)
+        goto end;
+        
+    /* reuse the udp created by whip */
+    ff_tls_set_external_socket(whip->dtls_uc, whip->udp);
+    
+    ret = ffurl_handshake(whip->dtls_uc);
+    if (ret < 0) {
+        whip->state = WHIP_STATE_FAILED;
+        av_log(whip, AV_LOG_VERBOSE, "DTLS session failed\n");
+    }
+    if (!ret) {
+        whip->state = WHIP_STATE_DTLS_FINISHED;
+        whip->whip_dtls_time = av_gettime();
+        av_log(whip, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%dms\n",
+            ELAPSED(whip->whip_starttime, whip->whip_dtls_time));
+    }
 end:
-    if (cert_buf)
-        av_free(cert_buf);
-    if (key_buf)
-        av_free(key_buf);
     return ret;
 }
 
@@ -1866,7 +1852,10 @@ static av_cold int whip_init(AVFormatContext *s)
     if ((ret = udp_connect(s)) < 0)
         goto end;
 
-    if ((ret = ice_dtls_handshake(s)) < 0)
+    if ((ret = ice_handshake(s)) < 0)
+        goto end;
+    
+    if ((ret = dtls_handshake(s)) < 0)
         goto end;
 
     if ((ret = setup_srtp(s)) < 0)
-- 
2.49.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".