Re: [FFmpeg-devel] [PATCH] avdevice/xcbgrab: enable window resizing

[Aline Gondim Santos Gondim Santos <aline.gondimsantos@savoirfairelinux.com>]

Hello Nicolas,
Bellow you can find the bechmarks using `ffmpeg -benchmark` option.

1 - master

./ffmpeg -benchmark -t 10 -framerate 25 -f x11grab -i ":1+0,0 1920x1080"  output1master.mp4
ffmpeg version N-109782-g458ae405ef Copyright (c) 2000-2023 the FFmpeg developers
  built with gcc 11 (Ubuntu 11.3.0-1ubuntu1~22.04)
  configuration:
  libavutil      57. 44.100 / 57. 44.100
  libavcodec     59. 63.100 / 59. 63.100
  libavformat    59. 38.100 / 59. 38.100
  libavdevice    59.  8.101 / 59.  8.101
  libavfilter     8. 56.100 /  8. 56.100
  libswscale      6.  8.112 /  6.  8.112
  libswresample   4.  9.100 /  4.  9.100
[x11grab @ 0x564d03e165c0] Stream #0: not enough frames to estimate rate; consider increasing probesize
Input #0, x11grab, from ':1+0,0 1920x1080':
  Duration: N/A, start: 1675963927.428661, bitrate: 3379200 kb/s
  Stream #0:0: Video: rawvideo (BGR[0] / 0x524742), bgr0, 3520x1200, 3379200 kb/s, 25 fps, 1000k tbr, 1000k tbn
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> mpeg4 (native))
Press [q] to stop, [?] for help
Output #0, mp4, to 'output1master.mp4':
  Metadata:
    encoder         : Lavf59.38.100
  Stream #0:0: Video: mpeg4 (mp4v / 0x7634706D), yuv420p(tv, progressive), 3520x1200, q=2-31, 200 kb/s, 25 fps, 12800 tbn
    Metadata:
      encoder         : Lavc59.63.100 mpeg4
    Side data:
      cpb: bitrate max/min/avg: 0/0/200000 buffer size: 0 vbv_delay: N/A
frame=  251 fps= 25 q=31.0 Lsize=    5720kB time=00:00:10.00 bitrate=4686.0kbits/s speed=0.996x    
video:5718kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.034719%
bench: utime=13.142s stime=0.307s rtime=10.039s
bench: maxrss=207576kB

2 - master

./ffmpeg -benchmark -t 10 -framerate 25 -f x11grab -window_id 0x5600008 -i ":1+0,0 1920x1080"  output2master.mp4
ffmpeg version N-109782-g458ae405ef Copyright (c) 2000-2023 the FFmpeg developers
  built with gcc 11 (Ubuntu 11.3.0-1ubuntu1~22.04)
  configuration:
  libavutil      57. 44.100 / 57. 44.100
  libavcodec     59. 63.100 / 59. 63.100
  libavformat    59. 38.100 / 59. 38.100
  libavdevice    59.  8.101 / 59.  8.101
  libavfilter     8. 56.100 /  8. 56.100
  libswscale      6.  8.112 /  6.  8.112
  libswresample   4.  9.100 /  4.  9.100
Input #0, x11grab, from ':1+0,0 1920x1080':
  Duration: N/A, start: 1675963986.581500, bitrate: 472305 kb/s
  Stream #0:0: Video: rawvideo (BGR[0] / 0x524742), bgr0, 841x702, 472305 kb/s, 25 fps, 25 tbr, 1000k tbn
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> mpeg4 (native))
Press [q] to stop, [?] for help
Output #0, mp4, to 'output2master.mp4':
  Metadata:
    encoder         : Lavf59.38.100
  Stream #0:0: Video: mpeg4 (mp4v / 0x7634706D), yuv420p(tv, progressive), 841x702, q=2-31, 200 kb/s, 25 fps, 12800 tbn
    Metadata:
      encoder         : Lavc59.63.100 mpeg4
    Side data:
      cpb: bitrate max/min/avg: 0/0/200000 buffer size: 0 vbv_delay: N/A
frame=  250 fps= 25 q=31.0 Lsize=    1274kB time=00:00:09.96 bitrate=1047.9kbits/s speed=   1x    
video:1272kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.151768%
bench: utime=0.628s stime=1.465s rtime=9.920s
bench: maxrss=52268kB  


3 - patch applied

./ffmpeg -benchmark -t 10 -framerate 25 -f x11grab -i ":1+0,0 1920x1080"  output1.mp4
ffmpeg version N-109783-g2352934f8b Copyright (c) 2000-2023 the FFmpeg developers
  built with gcc 11 (Ubuntu 11.3.0-1ubuntu1~22.04)
  configuration:
  libavutil      57. 44.100 / 57. 44.100
  libavcodec     59. 63.100 / 59. 63.100
  libavformat    59. 38.100 / 59. 38.100
  libavdevice    59.  8.101 / 59.  8.101
  libavfilter     8. 56.100 /  8. 56.100
  libswscale      6.  8.112 /  6.  8.112
  libswresample   4.  9.100 /  4.  9.100
[x11grab @ 0x55e86905b5c0] Stream #0: not enough frames to estimate rate; consider increasing probesize
Input #0, x11grab, from ':1+0,0 1920x1080':
  Duration: N/A, start: 1675964519.431271, bitrate: 3379200 kb/s
  Stream #0:0: Video: rawvideo (BGR[0] / 0x524742), bgr0, 3520x1200, 3379200 kb/s, 25 fps, 1000k tbr, 1000k tbn
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> mpeg4 (native))
Press [q] to stop, [?] for help
Output #0, mp4, to 'output1.mp4':
  Metadata:
    encoder         : Lavf59.38.100
  Stream #0:0: Video: mpeg4 (mp4v / 0x7634706D), yuv420p(tv, progressive), 3520x1200, q=2-31, 200 kb/s, 25 fps, 12800 tbn
    Metadata:
      encoder         : Lavc59.63.100 mpeg4
    Side data:
      cpb: bitrate max/min/avg: 0/0/200000 buffer size: 0 vbv_delay: N/A
frame=  250 fps= 25 q=31.0 Lsize=    5723kB time=00:00:09.96 bitrate=4706.8kbits/s speed=0.996x    
video:5721kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.034227%
bench: utime=14.005s stime=0.168s rtime=9.998s
bench: maxrss=207828kB


4 - patch applied


./ffmpeg -benchmark -t 10 -framerate 25 -f x11grab -window_id 0x5600008 -i ":1+0,0 1920x1080"  output2.mp4
ffmpeg version N-109783-g2352934f8b Copyright (c) 2000-2023 the FFmpeg developers
  built with gcc 11 (Ubuntu 11.3.0-1ubuntu1~22.04)
  configuration:
  libavutil      57. 44.100 / 57. 44.100
  libavcodec     59. 63.100 / 59. 63.100
  libavformat    59. 38.100 / 59. 38.100
  libavdevice    59.  8.101 / 59.  8.101
  libavfilter     8. 56.100 /  8. 56.100
  libswscale      6.  8.112 /  6.  8.112
  libswresample   4.  9.100 /  4.  9.100
Input #0, x11grab, from ':1+0,0 1920x1080':
  Duration: N/A, start: 1675964455.191272, bitrate: 472305 kb/s
  Stream #0:0: Video: rawvideo (BGR[0] / 0x524742), bgr0, 841x702, 472305 kb/s, 25 fps, 24.83 tbr, 1000k tbn
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> mpeg4 (native))
Press [q] to stop, [?] for help
Output #0, mp4, to 'output2.mp4':
  Metadata:
    encoder         : Lavf59.38.100
  Stream #0:0: Video: mpeg4 (mp4v / 0x7634706D), yuv420p(tv, progressive), 841x702, q=2-31, 200 kb/s, 24.83 fps, 19072 tbn
    Metadata:
      encoder         : Lavc59.63.100 mpeg4
    Side data:
      cpb: bitrate max/min/avg: 0/0/200000 buffer size: 0 vbv_delay: N/A
frame=  250 fps= 25 q=31.0 Lsize=     961kB time=00:00:10.02 bitrate= 785.0kbits/s speed=1.01x    
video:959kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.199722%
bench: utime=1.624s stime=0.049s rtime=9.920s
bench: maxrss=51996kB

Aline Gondim Santos, 
Software Developer - Montreal, QC 

Savoir-faire Linux inc. 
Contactez moi sur [ https://jami.net/ | Jami ] : agsantos

----- Original Message -----
From: "Aline Gondim Santos Gondim Santos" <aline.gondimsantos@savoirfairelinux.com>
To: "ffmpeg-devel" <ffmpeg-devel@ffmpeg.org>
Cc: "Aline Gondim Santos Gondim Santos" <aline.gondimsantos@savoirfairelinux.com>
Sent: Thursday, 9 February, 2023 15:19:53
Subject: [FFmpeg-devel] [PATCH] avdevice/xcbgrab: enable window resizing

Signed-off-by: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com>
---
 libavdevice/xcbgrab.c | 180 +++++++++---------------------------------
 1 file changed, 39 insertions(+), 141 deletions(-)

diff --git a/libavdevice/xcbgrab.c b/libavdevice/xcbgrab.c
index 64a68ba497..05282911a9 100644
--- a/libavdevice/xcbgrab.c
+++ b/libavdevice/xcbgrab.c
@@ -29,11 +29,6 @@
 #include <xcb/xfixes.h>
 #endif
 
-#if CONFIG_LIBXCB_SHM
-#include <sys/shm.h>
-#include <xcb/shm.h>
-#endif
-
 #if CONFIG_LIBXCB_SHAPE
 #include <xcb/shape.h>
 #endif
@@ -53,9 +48,6 @@ typedef struct XCBGrabContext {
     xcb_connection_t *conn;
     xcb_screen_t *screen;
     xcb_window_t window;
-#if CONFIG_LIBXCB_SHM
-    AVBufferPool *shm_pool;
-#endif
     int64_t time_frame;
     AVRational time_base;
     int64_t frame_duration;
@@ -72,10 +64,9 @@ typedef struct XCBGrabContext {
     int region_border;
     int centered;
     int select_region;
+    int is_area;
 
     const char *framerate;
-
-    int has_shm;
 } XCBGrabContext;
 
 #define FOLLOW_CENTER -1
@@ -97,6 +88,7 @@ static const AVOption options[] = {
     { "show_region", "Show the grabbing region.", OFFSET(show_region), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, D },
     { "region_border", "Set the region border thickness.", OFFSET(region_border), AV_OPT_TYPE_INT, { .i64 = 3 }, 1, 128, D },
     { "select_region", "Select the grabbing region graphically using the pointer.", OFFSET(select_region), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, D },
+    { "is_area", "Define if we are grabing a region of the display/window.", OFFSET(is_area), AV_OPT_TYPE_INT, { .i64 = 1 }, 0, 1, D },
     { NULL },
 };
 
@@ -216,99 +208,6 @@ static int64_t wait_frame(AVFormatContext *s, AVPacket *pkt)
     return curtime;
 }
 
-#if CONFIG_LIBXCB_SHM
-static int check_shm(xcb_connection_t *conn)
-{
-    xcb_shm_query_version_cookie_t cookie = xcb_shm_query_version(conn);
-    xcb_shm_query_version_reply_t *reply;
-
-    reply = xcb_shm_query_version_reply(conn, cookie, NULL);
-    if (reply) {
-        free(reply);
-        return 1;
-    }
-
-    return 0;
-}
-
-static void free_shm_buffer(void *opaque, uint8_t *data)
-{
-    shmdt(data);
-}
-
-static AVBufferRef *allocate_shm_buffer(void *opaque, size_t size)
-{
-    xcb_connection_t *conn = opaque;
-    xcb_shm_seg_t segment;
-    AVBufferRef *ref;
-    uint8_t *data;
-    int id;
-
-    id = shmget(IPC_PRIVATE, size, IPC_CREAT | 0777);
-    if (id == -1)
-        return NULL;
-
-    segment = xcb_generate_id(conn);
-    xcb_shm_attach(conn, segment, id, 0);
-    data = shmat(id, NULL, 0);
-    shmctl(id, IPC_RMID, 0);
-    if ((intptr_t)data == -1 || !data)
-        return NULL;
-
-    ref = av_buffer_create(data, size, free_shm_buffer, (void *)(ptrdiff_t)segment, 0);
-    if (!ref)
-        shmdt(data);
-
-    return ref;
-}
-
-static int xcbgrab_frame_shm(AVFormatContext *s, AVPacket *pkt)
-{
-    XCBGrabContext *c = s->priv_data;
-    xcb_shm_get_image_cookie_t iq;
-    xcb_shm_get_image_reply_t *img;
-    xcb_drawable_t drawable = c->window_id;
-    xcb_generic_error_t *e = NULL;
-    AVBufferRef *buf;
-    xcb_shm_seg_t segment;
-
-    buf = av_buffer_pool_get(c->shm_pool);
-    if (!buf) {
-        av_log(s, AV_LOG_ERROR, "Could not get shared memory buffer.\n");
-        return AVERROR(ENOMEM);
-    }
-    segment = (xcb_shm_seg_t)(uintptr_t)av_buffer_pool_buffer_get_opaque(buf);
-
-    iq = xcb_shm_get_image(c->conn, drawable,
-                           c->x, c->y, c->width, c->height, ~0,
-                           XCB_IMAGE_FORMAT_Z_PIXMAP, segment, 0);
-    img = xcb_shm_get_image_reply(c->conn, iq, &e);
-
-    xcb_flush(c->conn);
-
-    if (e) {
-        av_log(s, AV_LOG_ERROR,
-               "Cannot get the image data "
-               "event_error: response_type:%u error_code:%u "
-               "sequence:%u resource_id:%u minor_code:%u major_code:%u.\n",
-               e->response_type, e->error_code,
-               e->sequence, e->resource_id, e->minor_code, e->major_code);
-
-        free(e);
-        av_buffer_unref(&buf);
-        return AVERROR(EACCES);
-    }
-
-    free(img);
-
-    pkt->buf = buf;
-    pkt->data = buf->data;
-    pkt->size = c->frame_size;
-
-    return 0;
-}
-#endif /* CONFIG_LIBXCB_SHM */
-
 #if CONFIG_LIBXCB_XFIXES
 static int check_xfixes(xcb_connection_t *conn)
 {
@@ -462,14 +361,7 @@ static int xcbgrab_read_packet(AVFormatContext *s, AVPacket *pkt)
     if (c->show_region)
         xcbgrab_update_region(s, win_x, win_y);
 
-#if CONFIG_LIBXCB_SHM
-    if (c->has_shm && xcbgrab_frame_shm(s, pkt) < 0) {
-        av_log(s, AV_LOG_WARNING, "Continuing without shared memory.\n");
-        c->has_shm = 0;
-    }
-#endif
-    if (!c->has_shm)
-        ret = xcbgrab_frame(s, pkt);
+    ret = xcbgrab_frame(s, pkt);
     pkt->dts = pkt->pts = pts;
     pkt->duration = c->frame_duration;
 
@@ -488,11 +380,8 @@ static av_cold int xcbgrab_read_close(AVFormatContext *s)
 {
     XCBGrabContext *ctx = s->priv_data;
 
-#if CONFIG_LIBXCB_SHM
-    av_buffer_pool_uninit(&ctx->shm_pool);
-#endif
-
     xcb_disconnect(ctx->conn);
+    ctx->conn = NULL;
 
     return 0;
 }
@@ -572,7 +461,15 @@ static int pixfmt_from_pixmap_format(AVFormatContext *s, int depth,
 static int create_stream(AVFormatContext *s)
 {
     XCBGrabContext *c = s->priv_data;
-    AVStream *st      = avformat_new_stream(s, NULL);
+
+    // If we try to open another stream to x11grab, there is no reason
+    // to keep more than one stream in the context.
+    AVStream *st;
+    if (!s->nb_streams) {
+        st = avformat_new_stream(s, NULL);
+    } else {
+        st = s->streams[0];
+    }
     xcb_get_geometry_cookie_t gc;
     xcb_get_geometry_reply_t *geo;
     int64_t frame_size_bits;
@@ -594,7 +491,16 @@ static int create_stream(AVFormatContext *s)
         return AVERROR_EXTERNAL;
     }
 
+    // Width and Height are not 0 only when we set a window area to share
+    // This if may be valid only in  the first call to create_stream
     if (!c->width || !c->height) {
+        c->is_area = 0;
+        c->width = geo->width;
+        c->height = geo->height;
+    }
+    // If not a predefined area, then we should follow geometry changes
+    // This can be valid only on the second call onwards
+    if (!c->is_area && (c->width != geo->width || c->height != geo->height)) {
         c->width = geo->width;
         c->height = geo->height;
     }
@@ -628,13 +534,6 @@ static int create_stream(AVFormatContext *s)
     }
     c->frame_size = frame_size_bits / 8;
 
-#if CONFIG_LIBXCB_SHM
-    c->shm_pool = av_buffer_pool_init2(c->frame_size + AV_INPUT_BUFFER_PADDING_SIZE,
-                                           c->conn, allocate_shm_buffer, NULL);
-    if (!c->shm_pool)
-        return AVERROR(ENOMEM);
-#endif
-
     st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
     st->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
     st->codecpar->width      = c->width;
@@ -829,23 +728,26 @@ static av_cold int xcbgrab_read_header(AVFormatContext *s)
         sscanf(s->url, "+%d,%d", &c->x, &c->y);
     }
 
-    c->conn = xcb_connect(display_name[0] ? display_name : NULL, &screen_num);
-    av_freep(&display_name);
+    if (!c->conn || !c->screen) {
+        xcbgrab_read_close(s);
+        c->conn = xcb_connect(display_name[0] ? display_name : NULL, &screen_num);
+        av_freep(&display_name);
 
-    if ((ret = xcb_connection_has_error(c->conn))) {
-        av_log(s, AV_LOG_ERROR, "Cannot open display %s, error %d.\n",
-               s->url[0] ? s->url : "default", ret);
-        return AVERROR(EIO);
-    }
+        if ((ret = xcb_connection_has_error(c->conn))) {
+            av_log(s, AV_LOG_ERROR, "Cannot open display %s, error %d.\n",
+                s->url[0] ? s->url : "default", ret);
+            return AVERROR(EIO);
+        }
 
-    setup = xcb_get_setup(c->conn);
+       setup = xcb_get_setup(c->conn);
 
-    c->screen = get_screen(setup, screen_num);
-    if (!c->screen) {
-        av_log(s, AV_LOG_ERROR, "The screen %d does not exist.\n",
-               screen_num);
-        xcbgrab_read_close(s);
-        return AVERROR(EIO);
+        c->screen = get_screen(setup, screen_num);
+        if (!c->screen) {
+            av_log(s, AV_LOG_ERROR, "The screen %d does not exist.\n",
+                screen_num);
+            xcbgrab_read_close(s);
+            return AVERROR(EIO);
+        }
     }
 
     if (c->window_id == XCB_NONE)
@@ -876,10 +778,6 @@ static av_cold int xcbgrab_read_header(AVFormatContext *s)
         return ret;
     }
 
-#if CONFIG_LIBXCB_SHM
-    c->has_shm = check_shm(c->conn);
-#endif
-
 #if CONFIG_LIBXCB_XFIXES
     if (c->draw_mouse) {
         if (!(c->draw_mouse = check_xfixes(c->conn))) {
-- 
2.34.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".