[FFmpeg-devel] [PATCH 0/4] Cleanup avfoundation input

[FFmpeg-devel] [PATCH 0/4] Cleanup avfoundation input

[toots]

This is a series of patches that fix, enhance and cleanup support for audio and video input on macos using avfoundation in libavdevice.


_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 1/4] Use appropriate method for device discovery, fix crash with bogus device index.

[toots]

From: Romain Beauxis <toots@rastageeks.org>

This updates the code for avfoundation to use modern device lookup APIs and also adds a check to avoid querying the video devices array beyound its maximum size.

---
 libavdevice/avfoundation.m | 71 ++++++++++++++++++++++++++++++--------
 1 file changed, 56 insertions(+), 15 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..d8bcd98f81 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -27,6 +27,7 @@
 
 #import <AVFoundation/AVFoundation.h>
 #include <pthread.h>
+#include <Availability.h>
 
 #include "libavutil/channel_layout.h"
 #include "libavutil/pixdesc.h"
@@ -764,8 +765,34 @@ static int avf_read_header(AVFormatContext *s)
     AVCaptureDevice *video_device = nil;
     AVCaptureDevice *audio_device = nil;
     // Find capture device
-    NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
-    NSArray *devices_muxed = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];
+#if defined(__MAC_10_15) || (TARGET_OS_IPHONE && defined(__IPHONE_10_0))
+       AVCaptureDeviceDiscoverySession *discoverySession =
+            [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[
+#if TARGET_OS_IPHONE
+                                                 AVCaptureDeviceTypeBuiltInDualCamera,
+                                                 AVCaptureDeviceTypeBuiltInDualWideCamera,
+                                                 AVCaptureDeviceTypeBuiltInUltraWideCamera,
+                                                 AVCaptureDeviceTypeBuiltInTrueDepthCamera,
+                                                 AVCaptureDeviceTypeBuiltInTelephotoCamera,
+#endif
+                                                 AVCaptureDeviceTypeBuiltInWideAngleCamera,
+                                                 AVCaptureDeviceTypeExternalUnknown
+                                             ]
+                                             mediaType:NULL
+                                             position:AVCaptureDevicePositionUnspecified];
+
+       NSMutableArray *devices       = [NSMutableArray array];
+       NSMutableArray *devices_muxed = [NSMutableArray array];
+       for (AVCaptureDevice *device in [discoverySession devices]) {
+           if ([device hasMediaType:AVMediaTypeVideo])
+               [devices addObject:device];
+           else if ([device hasMediaType:AVMediaTypeMuxed])
+               [devices_muxed addObject:device];
+       }
+#else
+       NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
+       NSArray *devices_muxed = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];
+#endif
 
     ctx->num_video_devices = [devices count] + [devices_muxed count];
 
@@ -775,6 +802,21 @@ static int avf_read_header(AVFormatContext *s)
     CGGetActiveDisplayList(0, NULL, &num_screens);
 #endif
 
+    NSArray *audio_devices;
+#if defined(__MAC_10_15) || (TARGET_OS_IPHONE && defined(__IPHONE_10_0))
+    discoverySession =
+        [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[
+                                             AVCaptureDeviceTypeBuiltInMicrophone,
+                                             AVCaptureDeviceTypeExternalUnknown
+                                         ]
+                                         mediaType:AVMediaTypeAudio
+                                         position:AVCaptureDevicePositionUnspecified];
+
+    audio_devices = [discoverySession devices];
+#else
+    audio_devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
+#endif
+
     // List devices if requested
     if (ctx->list_devices) {
         int index = 0;
@@ -800,8 +842,7 @@ static int avf_read_header(AVFormatContext *s)
 #endif
 
         av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
-        devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
-        for (AVCaptureDevice *device in devices) {
+        for (AVCaptureDevice *device in audio_devices) {
             const char *name = [[device localizedName] UTF8String];
             int index  = [devices indexOfObject:device];
             av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
@@ -824,9 +865,12 @@ static int avf_read_header(AVFormatContext *s)
         if (ctx->video_device_index < ctx->num_video_devices) {
             if (ctx->video_device_index < [devices count]) {
                 video_device = [devices objectAtIndex:ctx->video_device_index];
-            } else {
+            } else if (ctx->video_device_index - [devices count] < [devices_muxed count]) {
                 video_device = [devices_muxed objectAtIndex:(ctx->video_device_index - [devices count])];
                 ctx->video_is_muxed = 1;
+            } else {
+                av_log(ctx, AV_LOG_ERROR, "Invalid video device index\n");
+                goto fail;
             }
         } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) {
 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -922,9 +966,7 @@ static int avf_read_header(AVFormatContext *s)
 
     // get audio device
     if (ctx->audio_device_index >= 0) {
-        NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
-
-        if (ctx->audio_device_index >= [devices count]) {
+        if (ctx->audio_device_index >= [audio_devices count]) {
             av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");
             goto fail;
         }
@@ -935,15 +977,14 @@ static int avf_read_header(AVFormatContext *s)
         if (!strncmp(ctx->audio_filename, "default", 7)) {
             audio_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
         } else {
-        NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
-
-        for (AVCaptureDevice *device in devices) {
-            if (!strncmp(ctx->audio_filename, [[device localizedName] UTF8String], strlen(ctx->audio_filename))) {
-                audio_device = device;
-                break;
+            for (AVCaptureDevice *device in audio_devices) {
+                const char *name = [[device localizedName] UTF8String];
+                if (!strncmp(ctx->audio_filename, name, strlen(ctx->audio_filename))) {
+                    audio_device = device;
+                    break;
+                }
             }
         }
-        }
 
         if (!audio_device) {
             av_log(ctx, AV_LOG_ERROR, "Audio device not found\n");
-- 
2.32.0 (Apple Git-132)

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/4] Use appropriate method for device discovery, fix crash with bogus device index.

[Thilo Borgmann]

Hi,

Am 30.01.22 um 18:30 schrieb toots@rastageeks.org:
> From: Romain Beauxis <toots@rastageeks.org>
> 
> This updates the code for avfoundation to use modern device lookup APIs and also adds a check to avoid querying the video devices array beyound its maximum size.
> 
> ---
>   libavdevice/avfoundation.m | 71 ++++++++++++++++++++++++++++++--------
>   1 file changed, 56 insertions(+), 15 deletions(-)
> 
> diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
> index 0cd6e646d5..d8bcd98f81 100644
> --- a/libavdevice/avfoundation.m
> +++ b/libavdevice/avfoundation.m
> @@ -27,6 +27,7 @@
>   
>   #import <AVFoundation/AVFoundation.h>
>   #include <pthread.h>
> +#include <Availability.h>
>   
>   #include "libavutil/channel_layout.h"
>   #include "libavutil/pixdesc.h"
> @@ -764,8 +765,34 @@ static int avf_read_header(AVFormatContext *s)
[...]

the patch doesn't appear to be broken any more, though it does not apply to current HEAD

Thilos-Mac-mini:FFmpeg borgmann$ git apply ../patches/avfoundation/\[PATCH\ 1_4\]\ Use\ appropriate\ method\ for\ device\ discovery\,\ fix\ crash\ with\ bogus\ device\ index.\ -\ toots\@rastageeks.org\ -\ 2022-01-30\ 1830.eml
error: patch failed: libavdevice/avfoundation.m:27
error: libavdevice/avfoundation.m: patch does not apply

I don't see why it does not apply as the part around line 27 looks sane. Did you try to apply to HEAD?

-Thilo
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/4] Use appropriate method for device discovery, fix crash with bogus device index.

[Romain Beauxis]

Le lun. 31 janv. 2022 à 04:11, Thilo Borgmann <thilo.borgmann@mail.de> a
écrit :
>
> Hi,
>
> Am 30.01.22 um 18:30 schrieb toots@rastageeks.org:
> > From: Romain Beauxis <toots@rastageeks.org>
> >
> > This updates the code for avfoundation to use modern device lookup APIs
and also adds a check to avoid querying the video devices array beyound its
maximum size.
> >
> > ---
> >   libavdevice/avfoundation.m | 71 ++++++++++++++++++++++++++++++--------
> >   1 file changed, 56 insertions(+), 15 deletions(-)
> >
> > diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
> > index 0cd6e646d5..d8bcd98f81 100644
> > --- a/libavdevice/avfoundation.m
> > +++ b/libavdevice/avfoundation.m
> > @@ -27,6 +27,7 @@
> >
> >   #import <AVFoundation/AVFoundation.h>
> >   #include <pthread.h>
> > +#include <Availability.h>
> >
> >   #include "libavutil/channel_layout.h"
> >   #include "libavutil/pixdesc.h"
> > @@ -764,8 +765,34 @@ static int avf_read_header(AVFormatContext *s)
> [...]
>
> the patch doesn't appear to be broken any more, though it does not apply
to current HEAD
>
> Thilos-Mac-mini:FFmpeg borgmann$ git apply
../patches/avfoundation/\[PATCH\ 1_4\]\ Use\ appropriate\ method\ for\
device\ discovery\,\ fix\ crash\ with\ bogus\ device\ index.\ -\ toots\@
rastageeks.org\ -\ 2022-01-30\ 1830.eml
> error: patch failed: libavdevice/avfoundation.m:27
> error: libavdevice/avfoundation.m: patch does not apply
>
> I don't see why it does not apply as the part around line 27 looks sane.
Did you try to apply to HEAD?

It does seem to apply here:
https://patchwork.ffmpeg.org/project/ffmpeg/patch/20220130173045.32690-2-toots@rastageeks.org/
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 2/4] libavdevice/avfoundation.m: Allow to select devices by unique ID

[toots]

From: Romain Beauxis <toots@rastageeks.org>

This adds a backward-compatible method to select devices using a unique ID that should not change accross device reboot or when a device is added or removed.

Signed-off-by: Romain Beauxis <toots@rastageeks.org>
---
 doc/indevs.texi            |  6 ++--
 libavdevice/avfoundation.m | 64 +++++++++++++++++++++++++++++++-------
 2 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 9d8020311a..858c0fa4e4 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following syntax:
 -i "[[VIDEO]:[AUDIO]]"
 @end example
 The first entry selects the video input while the latter selects the audio input.
-The stream has to be specified by the device name or the device index as shown by the device list.
+The stream has to be specified by the device name, index or ID as shown by the device list.
 Alternatively, the video and/or audio input device can be chosen by index using the
 @option{
     -video_device_index <INDEX>
@@ -127,7 +127,9 @@ and/or
 device name or index given in the input filename.
 
 All available devices can be enumerated by using @option{-list_devices true}, listing
-all device names and corresponding indices.
+all device names, corresponding indices and IDs, when available. Device name can be 
+tricky to use when localized and device index can change when devices are plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not change over time.
 
 There are two device name aliases:
 @table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index d8bcd98f81..a837042a6d 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -40,6 +40,8 @@
 #include "libavutil/imgutils.h"
 #include "avdevice.h"
 
+#define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrencesOfString:@":" withString:@"."] UTF8String]
+
 static const int avf_time_base = 1000000;
 
 static const AVRational avf_time_base_q = {
@@ -822,21 +824,23 @@ static int avf_read_header(AVFormatContext *s)
         int index = 0;
         av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
         for (AVCaptureDevice *device in devices) {
-            const char *name = [[device localizedName] UTF8String];
-            index            = [devices indexOfObject:device];
-            av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+            const char *name     = [[device localizedName] UTF8String];
+            const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+            index                = [devices indexOfObject:device];
+            av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, uniqueId);
         }
         for (AVCaptureDevice *device in devices_muxed) {
-            const char *name = [[device localizedName] UTF8String];
-            index            = [devices count] + [devices_muxed indexOfObject:device];
-            av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+            const char *name     = [[device localizedName] UTF8String];
+            const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+            index                = [devices count] + [devices_muxed indexOfObject:device];
+            av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, uniqueId);
         }
 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
         if (num_screens > 0) {
             CGDirectDisplayID screens[num_screens];
             CGGetActiveDisplayList(num_screens, screens, &num_screens);
             for (int i = 0; i < num_screens; i++) {
-                av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", ctx->num_video_devices + i, i);
+                av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d (ID: AvfilterAvfoundationCaptureScreen%d)\n", ctx->num_video_devices + i, i, screens[i]);
             }
         }
 #endif
@@ -844,7 +848,9 @@ static int avf_read_header(AVFormatContext *s)
         av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
         for (AVCaptureDevice *device in audio_devices) {
             const char *name = [[device localizedName] UTF8String];
+            const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
             int index  = [devices indexOfObject:device];
+            av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, uniqueId);
             av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
         }
          goto fail;
@@ -910,14 +916,29 @@ static int avf_read_header(AVFormatContext *s)
         } else {
         // looking for video inputs
         for (AVCaptureDevice *device in devices) {
-            if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
+            const char *name = [[device localizedName] UTF8String];
+            if (!strncmp(ctx->video_filename, name, strlen(ctx->video_filename))) {
+                video_device = device;
+                break;
+            }
+
+            const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+            if (!strncmp(ctx->video_filename, uniqueId, strlen(ctx->video_filename))) {
                 video_device = device;
                 break;
             }
         }
         // looking for muxed inputs
         for (AVCaptureDevice *device in devices_muxed) {
-            if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
+            const char *name = [[device localizedName] UTF8String];
+            if (!strncmp(ctx->video_filename, name, strlen(ctx->video_filename))) {
+                video_device = device;
+                ctx->video_is_muxed = 1;
+                break;
+            }
+
+            const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+            if (!strncmp(ctx->video_filename, uniqueId, strlen(ctx->video_filename))) {
                 video_device = device;
                 ctx->video_is_muxed = 1;
                 break;
@@ -928,10 +949,23 @@ static int avf_read_header(AVFormatContext *s)
         // looking for screen inputs
         if (!video_device) {
             int idx;
+            CGDirectDisplayID screens[num_screens];
+            CGGetActiveDisplayList(num_screens, screens, &num_screens);
+            AVCaptureScreenInput* capture_screen_input = NULL;
+
             if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) {
-                CGDirectDisplayID screens[num_screens];
-                CGGetActiveDisplayList(num_screens, screens, &num_screens);
-                AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];
+                capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];
+            }
+
+            if(sscanf(ctx->video_filename, "AvfilterAvfoundationCaptureScreen%d", &idx)) {
+                for (int i = 0; i < num_screens; i++) {
+                    if (screens[i] == idx) {
+                        capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:idx] autorelease];
+                    }
+                }
+            }
+
+            if (capture_screen_input) {
                 video_device = (AVCaptureDevice*) capture_screen_input;
                 ctx->video_device_index = ctx->num_video_devices + idx;
                 ctx->video_is_screen = 1;
@@ -983,6 +1017,12 @@ static int avf_read_header(AVFormatContext *s)
                     audio_device = device;
                     break;
                 }
+
+                const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+                if (!strncmp(ctx->audio_filename, uniqueId, strlen(ctx->audio_filename))) {
+                    audio_device = device;
+                    break;
+                }
             }
         }
 
-- 
2.32.0 (Apple Git-132)

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 3/4] libavdevice/avfoundation.m: use setAudioSettings, extend supported formats

[toots]

From: Romain Beauxis <toots@rastageeks.org>

This patch reworks the workflow for audio input to extend the range of supported input formats and delegate to the underlying OS the task of converting audio input format.

Previous version of these changes used the AudioConverter API to perform audio conversion explicitly however, it was found to be bug prone with issues seemingly coming from the underlying OS.

This fixes: https://trac.ffmpeg.org/ticket/9502

Signed-off-by: Romain Beauxis <toots@rastageeks.org>
---
 libavdevice/avfoundation.m | 206 ++++++++++++-------------------------
 1 file changed, 63 insertions(+), 143 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index a837042a6d..db9501445d 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -96,6 +96,11 @@
     AVRational      framerate;
     int             width, height;
 
+    int             channels;
+    int             big_endian;
+    int             sample_rate;
+    enum AVSampleFormat sample_format;
+
     int             capture_cursor;
     int             capture_mouse_clicks;
     int             capture_raw_data;
@@ -114,17 +119,6 @@
 
     int             num_video_devices;
 
-    int             audio_channels;
-    int             audio_bits_per_sample;
-    int             audio_float;
-    int             audio_be;
-    int             audio_signed_integer;
-    int             audio_packed;
-    int             audio_non_interleaved;
-
-    int32_t         *audio_buffer;
-    int             audio_buffer_size;
-
     enum AVPixelFormat pixel_format;
 
     AVCaptureSession         *capture_session;
@@ -301,14 +295,6 @@ static void destroy_context(AVFContext* ctx)
     ctx->audio_output    = NULL;
     ctx->avf_delegate    = NULL;
     ctx->avf_audio_delegate = NULL;
-
-    av_freep(&ctx->audio_buffer);
-
-    pthread_mutex_destroy(&ctx->frame_lock);
-
-    if (ctx->current_frame) {
-        CFRelease(ctx->current_frame);
-    }
 }
 
 static void parse_device_name(AVFormatContext *s)
@@ -674,88 +660,62 @@ static int get_video_config(AVFormatContext *s)
 static int get_audio_config(AVFormatContext *s)
 {
     AVFContext *ctx = (AVFContext*)s->priv_data;
-    CMFormatDescriptionRef format_desc;
-    AVStream* stream = avformat_new_stream(s, NULL);
+    AVStream* stream;
+    int bits_per_sample, is_float;
 
-    if (!stream) {
-        return 1;
-    }
+    enum AVCodecID codec_id = av_get_pcm_codec(ctx->sample_format, ctx->big_endian);
 
-    // Take stream info from the first frame.
-    while (ctx->audio_frames_captured < 1) {
-        CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
+    if (codec_id == AV_CODEC_ID_NONE) {
+       av_log(ctx, AV_LOG_ERROR, "Error: invalid sample format!\n");
+       return AVERROR(EINVAL);
     }
 
-    lock_frames(ctx);
-
-    ctx->audio_stream_index = stream->index;
-
-    avpriv_set_pts_info(stream, 64, 1, avf_time_base);
-
-    format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-    const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+    switch (ctx->sample_format) {
+        case AV_SAMPLE_FMT_S16:
+            bits_per_sample = 16;
+            is_float = 0;
+            break;
+        case AV_SAMPLE_FMT_S32:
+            bits_per_sample = 32;
+            is_float = 0;
+            break;
+        case AV_SAMPLE_FMT_FLT:
+            bits_per_sample = 32;
+            is_float = 1;
+            break;
+        default:
+            av_log(ctx, AV_LOG_ERROR, "Error: invalid sample format!\n");
+            unlock_frames(ctx);
+            return AVERROR(EINVAL);
+    }
 
-    if (!basic_desc) {
+    [ctx->audio_output setAudioSettings:@{
+        AVFormatIDKey:               @(kAudioFormatLinearPCM),
+        AVLinearPCMBitDepthKey:      @(bits_per_sample),
+        AVLinearPCMIsFloatKey:       @(is_float),
+        AVLinearPCMIsBigEndianKey:   @(ctx->big_endian),
+        AVNumberOfChannelsKey:       @(ctx->channels),
+        AVLinearPCMIsNonInterleaved: @NO,
+        AVSampleRateKey:             @(ctx->sample_rate)
+    }];
+
+    stream = avformat_new_stream(s, NULL);
+    if (!stream) {
         unlock_frames(ctx);
-        av_log(s, AV_LOG_ERROR, "audio format not available\n");
-        return 1;
+        return -1;
     }
 
+    avpriv_set_pts_info(stream, 64, 1, avf_time_base);
+
     stream->codecpar->codec_type     = AVMEDIA_TYPE_AUDIO;
-    stream->codecpar->sample_rate    = basic_desc->mSampleRate;
-    stream->codecpar->channels       = basic_desc->mChannelsPerFrame;
+    stream->codecpar->sample_rate    = ctx->sample_rate;
+    stream->codecpar->channels       = ctx->channels;
     stream->codecpar->channel_layout = av_get_default_channel_layout(stream->codecpar->channels);
+    stream->codecpar->codec_id       = codec_id;
 
-    ctx->audio_channels        = basic_desc->mChannelsPerFrame;
-    ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-    ctx->audio_float           = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat;
-    ctx->audio_be              = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian;
-    ctx->audio_signed_integer  = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger;
-    ctx->audio_packed          = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked;
-    ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved;
-
-    if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-        ctx->audio_float &&
-        ctx->audio_bits_per_sample == 32 &&
-        ctx->audio_packed) {
-        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;
-    } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-        ctx->audio_signed_integer &&
-        ctx->audio_bits_per_sample == 16 &&
-        ctx->audio_packed) {
-        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;
-    } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-        ctx->audio_signed_integer &&
-        ctx->audio_bits_per_sample == 24 &&
-        ctx->audio_packed) {
-        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE;
-    } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-        ctx->audio_signed_integer &&
-        ctx->audio_bits_per_sample == 32 &&
-        ctx->audio_packed) {
-        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE;
-    } else {
-        unlock_frames(ctx);
-        av_log(s, AV_LOG_ERROR, "audio format is not supported\n");
-        return 1;
-    }
-
-    if (ctx->audio_non_interleaved) {
-        CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
-        ctx->audio_buffer_size        = CMBlockBufferGetDataLength(block_buffer);
-        ctx->audio_buffer             = av_malloc(ctx->audio_buffer_size);
-        if (!ctx->audio_buffer) {
-            unlock_frames(ctx);
-            av_log(s, AV_LOG_ERROR, "error allocating audio buffer\n");
-            return 1;
-        }
-    }
-
-    CFRelease(ctx->current_audio_frame);
-    ctx->current_audio_frame = nil;
+    ctx->audio_stream_index = stream->index;
 
     unlock_frames(ctx);
-
     return 0;
 }
 
@@ -1056,6 +1016,7 @@ static int avf_read_header(AVFormatContext *s)
         goto fail;
     }
     if (audio_device && add_audio_device(s, audio_device)) {
+        goto fail;
     }
 
     [ctx->capture_session startRunning];
@@ -1129,6 +1090,7 @@ static int copy_cvpixelbuffer(AVFormatContext *s,
 
 static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
 {
+    OSStatus ret;
     AVFContext* ctx = (AVFContext*)s->priv_data;
 
     do {
@@ -1172,7 +1134,7 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
                 status = copy_cvpixelbuffer(s, image_buffer, pkt);
             } else {
                 status = 0;
-                OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
+                ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
                 if (ret != kCMBlockBufferNoErr) {
                     status = AVERROR(EIO);
                 }
@@ -1186,19 +1148,17 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
             }
         } else if (ctx->current_audio_frame != nil) {
             CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
-            int block_buffer_size         = CMBlockBufferGetDataLength(block_buffer);
 
-            if (!block_buffer || !block_buffer_size) {
-                unlock_frames(ctx);
-                return AVERROR(EIO);
-            }
+            size_t buffer_size = CMBlockBufferGetDataLength(block_buffer);
 
-            if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {
+            int status = av_new_packet(pkt, buffer_size);
+            if (status < 0) {
                 unlock_frames(ctx);
-                return AVERROR_BUFFER_TOO_SMALL;
+                return status;
             }
 
-            if (av_new_packet(pkt, block_buffer_size) < 0) {
+            ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
+            if (ret != kCMBlockBufferNoErr) {
                 unlock_frames(ctx);
                 return AVERROR(EIO);
             }
@@ -1214,54 +1174,10 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
             pkt->stream_index  = ctx->audio_stream_index;
             pkt->flags        |= AV_PKT_FLAG_KEY;
 
-            if (ctx->audio_non_interleaved) {
-                int sample, c, shift, num_samples;
-
-                OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);
-                if (ret != kCMBlockBufferNoErr) {
-                    unlock_frames(ctx);
-                    return AVERROR(EIO);
-                }
-
-                num_samples = pkt->size / (ctx->audio_channels * (ctx->audio_bits_per_sample >> 3));
-
-                // transform decoded frame into output format
-                #define INTERLEAVE_OUTPUT(bps)                                         \
-                {                                                                      \
-                    int##bps##_t **src;                                                \
-                    int##bps##_t *dest;                                                \
-                    src = av_malloc(ctx->audio_channels * sizeof(int##bps##_t*));      \
-                    if (!src) {                                                        \
-                        unlock_frames(ctx);                                            \
-                        return AVERROR(EIO);                                           \
-                    }                                                                  \
-                                                                                       \
-                    for (c = 0; c < ctx->audio_channels; c++) {                        \
-                        src[c] = ((int##bps##_t*)ctx->audio_buffer) + c * num_samples; \
-                    }                                                                  \
-                    dest  = (int##bps##_t*)pkt->data;                                  \
-                    shift = bps - ctx->audio_bits_per_sample;                          \
-                    for (sample = 0; sample < num_samples; sample++)                   \
-                        for (c = 0; c < ctx->audio_channels; c++)                      \
-                            *dest++ = src[c][sample] << shift;                         \
-                    av_freep(&src);                                                    \
-                }
-
-                if (ctx->audio_bits_per_sample <= 16) {
-                    INTERLEAVE_OUTPUT(16)
-                } else {
-                    INTERLEAVE_OUTPUT(32)
-                }
-            } else {
-                OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
-                if (ret != kCMBlockBufferNoErr) {
-                    unlock_frames(ctx);
-                    return AVERROR(EIO);
-                }
-            }
-
             CFRelease(ctx->current_audio_frame);
             ctx->current_audio_frame = nil;
+
+            unlock_frames(ctx);
         } else {
             pkt->data = NULL;
             unlock_frames(ctx);
@@ -1286,6 +1202,10 @@ static int avf_close(AVFormatContext *s)
 }
 
 static const AVOption options[] = {
+    { "channels", "number of audio channels", offsetof(AVFContext, channels), AV_OPT_TYPE_INT, {.i64=2}, 0, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM },
+    { "sample_rate", "audio sample rate", offsetof(AVFContext, sample_rate), AV_OPT_TYPE_INT, {.i64=44100}, 0, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM },
+    { "big_endian", "return big endian samples for audio data", offsetof(AVFContext, big_endian), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_ENCODING_PARAM },
+    { "sample_format", "audio sample format", offsetof(AVFContext, sample_format), AV_OPT_TYPE_INT, {.i64=AV_SAMPLE_FMT_S16}, 0, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM },
     { "list_devices", "list available devices", offsetof(AVFContext, list_devices), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
     { "video_device_index", "select video device by index for devices with same name (starts at 0)", offsetof(AVFContext, video_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
     { "audio_device_index", "select audio device by index for devices with same name (starts at 0)", offsetof(AVFContext, audio_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
-- 
2.32.0 (Apple Git-132)

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 4/4] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length

[toots]

From: Romain Beauxis <toots@rastageeks.org>

These changes rewrite the concurrency model in the avfoundation input to avoid concurrent situations where the consuming thread might get late by a frame or more, leading to input data being dropped.

This issue was particularly noticeable when working with audio input.

Signed-off-by: Romain Beauxis <toots@rastageeks.org>
---
 libavdevice/avfoundation.m | 227 +++++++++++++++++--------------------
 1 file changed, 101 insertions(+), 126 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index db9501445d..a473888574 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,8 +26,8 @@
  */
 
 #import <AVFoundation/AVFoundation.h>
-#include <pthread.h>
 #include <Availability.h>
+#import <CoreMedia/CoreMedia.h>
 
 #include "libavutil/channel_layout.h"
 #include "libavutil/pixdesc.h"
@@ -42,6 +42,13 @@
 
 #define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrencesOfString:@":" withString:@"."] UTF8String]
 
+static void av_log_avfoundation(void *s, int lvl, const char *str, OSStatus err) {
+    NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
+    av_log(s, lvl, "AVFoundation: %s, %s\n", str,
+        [[[NSError errorWithDomain:NSOSStatusErrorDomain code:err userInfo:nil] localizedDescription] UTF8String]);
+    [pool release];
+}
+
 static const int avf_time_base = 1000000;
 
 static const AVRational avf_time_base_q = {
@@ -87,9 +94,6 @@
 {
     AVClass*        class;
 
-    int             frames_captured;
-    int             audio_frames_captured;
-    pthread_mutex_t frame_lock;
     id              avf_delegate;
     id              avf_audio_delegate;
 
@@ -124,8 +128,9 @@
     AVCaptureSession         *capture_session;
     AVCaptureVideoDataOutput *video_output;
     AVCaptureAudioDataOutput *audio_output;
-    CMSampleBufferRef         current_frame;
-    CMSampleBufferRef         current_audio_frame;
+
+    CMSimpleQueueRef          frames_queue;
+    int                       max_frames;
 
     AVCaptureDevice          *observed_device;
 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -134,16 +139,6 @@
     int                      observed_quit;
 } AVFContext;
 
-static void lock_frames(AVFContext* ctx)
-{
-    pthread_mutex_lock(&ctx->frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-    pthread_mutex_unlock(&ctx->frame_lock);
-}
-
 /** FrameReciever class - delegate for AVCaptureSession
  */
 @interface AVFFrameReceiver : NSObject
@@ -221,17 +216,13 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
   didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
          fromConnection:(AVCaptureConnection *)connection
 {
-    lock_frames(_context);
+    OSStatus ret = CMSimpleQueueEnqueue(_context->frames_queue, videoFrame);
 
-    if (_context->current_frame != nil) {
-        CFRelease(_context->current_frame);
+    if (ret != noErr) {
+      av_log_avfoundation(_context, AV_LOG_DEBUG, "Error while queueing video frame", ret);
     }
 
-    _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
-
-    unlock_frames(_context);
-
-    ++_context->frames_captured;
+    CFRetain(videoFrame);
 }
 
 @end
@@ -265,17 +256,13 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
   didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
          fromConnection:(AVCaptureConnection *)connection
 {
-    lock_frames(_context);
+    OSStatus ret = CMSimpleQueueEnqueue(_context->frames_queue, audioFrame);
 
-    if (_context->current_audio_frame != nil) {
-        CFRelease(_context->current_audio_frame);
+    if (ret != noErr) {
+      av_log_avfoundation(_context, AV_LOG_DEBUG, "Error while queueing audio frame", ret);
     }
 
-    _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
-
-    unlock_frames(_context);
-
-    ++_context->audio_frames_captured;
+    CFRetain(audioFrame);
 }
 
 @end
@@ -290,6 +277,19 @@ static void destroy_context(AVFContext* ctx)
     [ctx->avf_delegate    release];
     [ctx->avf_audio_delegate release];
 
+    CMSampleBufferRef frame;
+
+    if (ctx->frames_queue) {
+        frame = (CMSampleBufferRef)CMSimpleQueueDequeue(ctx->frames_queue);
+        while (frame) {
+          CFRelease(frame);
+          frame = (CMSampleBufferRef)CMSimpleQueueDequeue(ctx->frames_queue);
+        }
+
+        CFRelease(ctx->frames_queue);
+        ctx->frames_queue = NULL;
+    }
+
     ctx->capture_session = NULL;
     ctx->video_output    = NULL;
     ctx->audio_output    = NULL;
@@ -330,15 +330,14 @@ static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_dev
     NSObject *format = nil;
     NSObject *selected_range = nil;
     NSObject *selected_format = nil;
+    CMFormatDescriptionRef formatDescription;
+    CMVideoDimensions dimensions;
 
     // try to configure format by formats list
     // might raise an exception if no format list is given
     // (then fallback to default, no configuration)
     @try {
         for (format in [video_device valueForKey:@"formats"]) {
-            CMFormatDescriptionRef formatDescription;
-            CMVideoDimensions dimensions;
-
             formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
             dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
 
@@ -365,6 +364,9 @@ static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_dev
             goto unsupported_format;
         }
 
+        ctx->width  = dimensions.width;
+        ctx->height = dimensions.height;
+
         if (!selected_range) {
             av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device.\n",
                 framerate);
@@ -612,47 +614,21 @@ static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
 static int get_video_config(AVFormatContext *s)
 {
     AVFContext *ctx = (AVFContext*)s->priv_data;
-    CVImageBufferRef image_buffer;
-    CMBlockBufferRef block_buffer;
-    CGSize image_buffer_size;
     AVStream* stream = avformat_new_stream(s, NULL);
 
     if (!stream) {
         return 1;
     }
 
-    // Take stream info from the first frame.
-    while (ctx->frames_captured < 1) {
-        CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
-    }
-
-    lock_frames(ctx);
-
     ctx->video_stream_index = stream->index;
 
     avpriv_set_pts_info(stream, 64, 1, avf_time_base);
 
-    image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
-    block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
-
-    if (image_buffer) {
-        image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
-
-        stream->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
-        stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
-        stream->codecpar->width      = (int)image_buffer_size.width;
-        stream->codecpar->height     = (int)image_buffer_size.height;
-        stream->codecpar->format     = ctx->pixel_format;
-    } else {
-        stream->codecpar->codec_id   = AV_CODEC_ID_DVVIDEO;
-        stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
-        stream->codecpar->format     = ctx->pixel_format;
-    }
-
-    CFRelease(ctx->current_frame);
-    ctx->current_frame = nil;
-
-    unlock_frames(ctx);
+    stream->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
+    stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
+    stream->codecpar->width      = ctx->width;
+    stream->codecpar->height     = ctx->height;
+    stream->codecpar->format     = ctx->pixel_format;
 
     return 0;
 }
@@ -685,7 +661,6 @@ static int get_audio_config(AVFormatContext *s)
             break;
         default:
             av_log(ctx, AV_LOG_ERROR, "Error: invalid sample format!\n");
-            unlock_frames(ctx);
             return AVERROR(EINVAL);
     }
 
@@ -700,10 +675,8 @@ static int get_audio_config(AVFormatContext *s)
     }];
 
     stream = avformat_new_stream(s, NULL);
-    if (!stream) {
-        unlock_frames(ctx);
+    if (!stream)
         return -1;
-    }
 
     avpriv_set_pts_info(stream, 64, 1, avf_time_base);
 
@@ -715,7 +688,6 @@ static int get_audio_config(AVFormatContext *s)
 
     ctx->audio_stream_index = stream->index;
 
-    unlock_frames(ctx);
     return 0;
 }
 
@@ -758,8 +730,6 @@ static int avf_read_header(AVFormatContext *s)
 
     ctx->num_video_devices = [devices count] + [devices_muxed count];
 
-    pthread_mutex_init(&ctx->frame_lock, NULL);
-
 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
     CGGetActiveDisplayList(0, NULL, &num_screens);
 #endif
@@ -1012,6 +982,14 @@ static int avf_read_header(AVFormatContext *s)
     // Initialize capture session
     ctx->capture_session = [[AVCaptureSession alloc] init];
 
+    OSStatus ret;
+    ret = CMSimpleQueueCreate(kCFAllocatorDefault, ctx->max_frames, &ctx->frames_queue);
+
+    if (ret != noErr) {
+        av_log_avfoundation(s, AV_LOG_ERROR, "error while creating frame queue", ret);
+        goto fail;
+    }
+
     if (video_device && add_video_device(s, video_device)) {
         goto fail;
     }
@@ -1042,7 +1020,8 @@ static int avf_read_header(AVFormatContext *s)
 fail:
     [pool release];
     destroy_context(ctx);
-    return AVERROR(EIO);
+    av_log(s, AV_LOG_ERROR, "Error while opening AVfoundation capture session\n");
+    return AVERROR_EXTERNAL;
 }
 
 static int copy_cvpixelbuffer(AVFormatContext *s,
@@ -1091,39 +1070,46 @@ static int copy_cvpixelbuffer(AVFormatContext *s,
 static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
 {
     OSStatus ret;
+    int status, length;
+    CMBlockBufferRef block_buffer;
+    CMSampleBufferRef frame;
+    CMFormatDescriptionRef format;
     AVFContext* ctx = (AVFContext*)s->priv_data;
+    CMItemCount count;
+    CMSampleTimingInfo timing_info;
+    CVImageBufferRef image_buffer;
+    size_t buffer_size;
+    AVRational timebase_q;
 
-    do {
-        CVImageBufferRef image_buffer;
-        CMBlockBufferRef block_buffer;
-        lock_frames(ctx);
+    if (CMSimpleQueueGetCount(ctx->frames_queue) < 1)
+        return AVERROR(EAGAIN);
 
-        if (ctx->current_frame != nil) {
-            int status;
-            int length = 0;
+    frame = (CMSampleBufferRef)CMSimpleQueueDequeue(ctx->frames_queue);
+    format = CMSampleBufferGetFormatDescription(frame);
 
-            image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
-            block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
+    switch (CMFormatDescriptionGetMediaType(format)) {
+        case kCMMediaType_Video:
+            length = 0;
+            image_buffer = CMSampleBufferGetImageBuffer(frame);
+            block_buffer = CMSampleBufferGetDataBuffer(frame);
 
             if (image_buffer != nil) {
                 length = (int)CVPixelBufferGetDataSize(image_buffer);
             } else if (block_buffer != nil) {
                 length = (int)CMBlockBufferGetDataLength(block_buffer);
             } else  {
-                unlock_frames(ctx);
+                CFRelease(frame);
                 return AVERROR(EINVAL);
             }
 
-            if (av_new_packet(pkt, length) < 0) {
-                unlock_frames(ctx);
-                return AVERROR(EIO);
+            status = av_new_packet(pkt, length);
+            if (status < 0) {
+                CFRelease(frame);
+                return status;
             }
 
-            CMItemCount count;
-            CMSampleTimingInfo timing_info;
-
-            if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
-                AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
+            if (CMSampleBufferGetOutputSampleTimingInfoArray(frame, 1, &timing_info, &count) == noErr) {
+                timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
                 pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
             }
 
@@ -1136,62 +1122,50 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
                 status = 0;
                 ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
                 if (ret != kCMBlockBufferNoErr) {
-                    status = AVERROR(EIO);
+                    av_log_avfoundation(s, AV_LOG_ERROR, "error while copying buffer data", ret);
+                    status = AVERROR_EXTERNAL;
                 }
-             }
-            CFRelease(ctx->current_frame);
-            ctx->current_frame = nil;
-
-            if (status < 0) {
-                unlock_frames(ctx);
-                return status;
             }
-        } else if (ctx->current_audio_frame != nil) {
-            CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
+            CFRelease(frame);
 
-            size_t buffer_size = CMBlockBufferGetDataLength(block_buffer);
+            return status;
 
-            int status = av_new_packet(pkt, buffer_size);
+        case kCMMediaType_Audio:
+            block_buffer = CMSampleBufferGetDataBuffer(frame);
+            buffer_size = CMBlockBufferGetDataLength(block_buffer);
+
+            status = av_new_packet(pkt, buffer_size);
             if (status < 0) {
-                unlock_frames(ctx);
+                CFRelease(frame);
                 return status;
             }
 
             ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
             if (ret != kCMBlockBufferNoErr) {
-                unlock_frames(ctx);
-                return AVERROR(EIO);
+                CFRelease(frame);
+                av_log_avfoundation(s, AV_LOG_ERROR, "error while copying audio data", ret);
+                return AVERROR_EXTERNAL;
             }
 
-            CMItemCount count;
-            CMSampleTimingInfo timing_info;
-
-            if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
-                AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
+            if (CMSampleBufferGetOutputSampleTimingInfoArray(frame, 1, &timing_info, &count) == noErr) {
+                timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
                 pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
             }
 
             pkt->stream_index  = ctx->audio_stream_index;
             pkt->flags        |= AV_PKT_FLAG_KEY;
 
-            CFRelease(ctx->current_audio_frame);
-            ctx->current_audio_frame = nil;
-
-            unlock_frames(ctx);
-        } else {
+            CFRelease(frame);
+            return 0;
+        default:
             pkt->data = NULL;
-            unlock_frames(ctx);
-            if (ctx->observed_quit) {
+            if (ctx->observed_quit)
                 return AVERROR_EOF;
-            } else {
-                return AVERROR(EAGAIN);
-            }
-        }
 
-        unlock_frames(ctx);
-    } while (!pkt->data);
+            return AVERROR(EAGAIN);
+    }
 
-    return 0;
+    return AVERROR_BUG;
 }
 
 static int avf_close(AVFormatContext *s)
@@ -1216,6 +1190,7 @@ static int avf_close(AVFormatContext *s)
     { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
     { "capture_raw_data", "capture the raw data from device connection", offsetof(AVFContext, capture_raw_data), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
     { "drop_late_frames", "drop frames that are available later than expected", offsetof(AVFContext, drop_late_frames), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
+    { "max_frames", "Maximun length of the queue of pending frames", offsetof(AVFContext, max_frames), AV_OPT_TYPE_INT, {.i64=10}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
 
     { NULL },
 };
-- 
2.32.0 (Apple Git-132)

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".