[FFmpeg-devel] [PATCH v3 0/5] Vision Pro Spatial Data

[FFmpeg-devel] [PATCH v3 0/5] Vision Pro Spatial Data

[Derek Buitenhuis]

Changes since v2:
* horizontal display adjustment is now a rational

Derek Buitenhuis (5):
  avutil/spherical: Add more spherical types
  avutil/stereo3d: Fill out stereo info provided by Vision Pro files
  fftools/ffprobe: Print more Stereo 3D info from side data
  avformat/mov: Add support for exporting Video Extension Usage info
  avformat/mov: Add support for reading and exporting horizontal field
    of view

 fftools/ffprobe.c                            |   8 +
 libavformat/mov.c                            | 312 +++++++++++++++++++
 libavutil/spherical.c                        |   5 +
 libavutil/spherical.h                        |  16 +
 libavutil/stereo3d.c                         |  52 ++++
 libavutil/stereo3d.h                         |  78 +++++
 libavutil/version.h                          |   2 +-
 tests/ref/fate/matroska-spherical-mono       |   2 +
 tests/ref/fate/matroska-spherical-mono-remux |   4 +
 tests/ref/fate/matroska-stereo_mode          |   8 +
 tests/ref/fate/matroska-vp8-alpha-remux      |   2 +
 tests/ref/fate/mov-spherical-mono            |   2 +
 12 files changed, 490 insertions(+), 1 deletion(-)

-- 
2.43.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH v3 1/5] avutil/spherical: Add more spherical types

[Derek Buitenhuis]

These originate from the Apple Vision Pro, and are documented here:

    https://developer.apple.com/documentation/coremedia/cmprojectiontype

Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>
---
 libavutil/spherical.c |  5 +++++
 libavutil/spherical.h | 16 ++++++++++++++++
 libavutil/version.h   |  2 +-
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/libavutil/spherical.c b/libavutil/spherical.c
index 800d3459a5..64ade1d0ec 100644
--- a/libavutil/spherical.c
+++ b/libavutil/spherical.c
@@ -29,6 +29,8 @@ AVSphericalMapping *av_spherical_alloc(size_t *size)
     if (!spherical)
         return NULL;
 
+    spherical->projection = AV_SPHERICAL_RECTILINEAR;
+
     if (size)
         *size = sizeof(*spherical);
 
@@ -57,6 +59,9 @@ static const char *const spherical_projection_names[] = {
     [AV_SPHERICAL_EQUIRECTANGULAR]      = "equirectangular",
     [AV_SPHERICAL_CUBEMAP]              = "cubemap",
     [AV_SPHERICAL_EQUIRECTANGULAR_TILE] = "tiled equirectangular",
+    [AV_SPHERICAL_HALF_EQUIRECTANGULAR] = "half equirectangular",
+    [AV_SPHERICAL_RECTILINEAR]          = "rectilinear",
+    [AV_SPHERICAL_FISHEYE]              = "fisheye",
 };
 
 const char *av_spherical_projection_name(enum AVSphericalProjection projection)
diff --git a/libavutil/spherical.h b/libavutil/spherical.h
index 828ac836da..2e90f7752d 100644
--- a/libavutil/spherical.h
+++ b/libavutil/spherical.h
@@ -66,6 +66,22 @@ enum AVSphericalProjection {
      * the position of the current video in a larger surface.
      */
     AV_SPHERICAL_EQUIRECTANGULAR_TILE,
+
+    /**
+     * Video frame displays as a 180 degree equirectangular projection.
+     */
+    AV_SPHERICAL_HALF_EQUIRECTANGULAR,
+
+    /**
+     * Video frame displays on a flat, rectangular 2D surface.
+     */
+    AV_SPHERICAL_RECTILINEAR,
+
+    /**
+     * Fisheye projection (Apple).
+     * See: https://developer.apple.com/documentation/coremedia/cmprojectiontype/fisheye
+     */
+    AV_SPHERICAL_FISHEYE,
 };
 
 /**
diff --git a/libavutil/version.h b/libavutil/version.h
index 2756f2aa03..7df546ee22 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -79,7 +79,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  59
-#define LIBAVUTIL_VERSION_MINOR  22
+#define LIBAVUTIL_VERSION_MINOR  23
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
-- 
2.43.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH v3 2/5] avutil/stereo3d: Fill out stereo info provided by Vision Pro files

[Derek Buitenhuis]

Based on what is in the files themselves, and what the API provides
to users.

URLs:
  * https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_heroeye
  * https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_stereocamerabaseline
  * https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_horizontaldisparityadjustment
  * https://developer.apple.com/documentation/coremedia/kcmformatdescriptionextension_horizontalfieldofview

Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>
---
 libavutil/stereo3d.c | 52 +++++++++++++++++++++++++++++
 libavutil/stereo3d.h | 78 ++++++++++++++++++++++++++++++++++++++++++++
 libavutil/version.h  |  2 +-
 3 files changed, 131 insertions(+), 1 deletion(-)

diff --git a/libavutil/stereo3d.c b/libavutil/stereo3d.c
index 9c29ab01b5..a40a9439bb 100644
--- a/libavutil/stereo3d.c
+++ b/libavutil/stereo3d.c
@@ -55,6 +55,18 @@ static const char * const stereo3d_type_names[] = {
     [AV_STEREO3D_COLUMNS]             = "interleaved columns",
 };
 
+static const char * const stereo3d_view_names[] = {
+    [AV_STEREO3D_VIEW_PACKED] = "packed",
+    [AV_STEREO3D_VIEW_LEFT]   = "left",
+    [AV_STEREO3D_VIEW_RIGHT]  = "right",
+};
+
+static const char * const stereo3d_primary_eye_names[] = {
+    [AV_PRIMARY_EYE_NONE]  = "none",
+    [AV_PRIMARY_EYE_LEFT]  = "left",
+    [AV_PRIMARY_EYE_RIGHT] = "right",
+};
+
 const char *av_stereo3d_type_name(unsigned int type)
 {
     if (type >= FF_ARRAY_ELEMS(stereo3d_type_names))
@@ -74,3 +86,43 @@ int av_stereo3d_from_name(const char *name)
 
     return -1;
 }
+
+const char *av_stereo3d_view_name(unsigned int view)
+{
+    if (view >= FF_ARRAY_ELEMS(stereo3d_view_names))
+        return "unknown";
+
+    return stereo3d_view_names[view];
+}
+
+int av_stereo3d_view_from_name(const char *name)
+{
+    int i;
+
+    for (i = 0; i < FF_ARRAY_ELEMS(stereo3d_view_names); i++) {
+        if (av_strstart(name, stereo3d_view_names[i], NULL))
+            return i;
+    }
+
+    return -1;
+}
+
+const char *av_stereo3d_primary_eye_name(unsigned int eye)
+{
+    if (eye >= FF_ARRAY_ELEMS(stereo3d_primary_eye_names))
+        return "unknown";
+
+    return stereo3d_primary_eye_names[eye];
+}
+
+int av_stereo3d_primary_eye_from_name(const char *name)
+{
+    int i;
+
+    for (i = 0; i < FF_ARRAY_ELEMS(stereo3d_primary_eye_names); i++) {
+        if (av_strstart(name, stereo3d_primary_eye_names[i], NULL))
+            return i;
+    }
+
+    return -1;
+}
diff --git a/libavutil/stereo3d.h b/libavutil/stereo3d.h
index 3aab959b79..00a5c3900e 100644
--- a/libavutil/stereo3d.h
+++ b/libavutil/stereo3d.h
@@ -158,6 +158,26 @@ enum AVStereo3DView {
     AV_STEREO3D_VIEW_RIGHT,
 };
 
+/**
+ * List of possible primary eyes.
+ */
+enum AVStereo3DPrimaryEye {
+    /**
+     * Neither eye.
+     */
+    AV_PRIMARY_EYE_NONE,
+
+    /**
+     * Left eye.
+     */
+    AV_PRIMARY_EYE_LEFT,
+
+    /**
+     * Right eye
+     */
+    AV_PRIMARY_EYE_RIGHT,
+};
+
 /**
  * Inverted views, Right/Bottom represents the left view.
  */
@@ -185,6 +205,28 @@ typedef struct AVStereo3D {
      * Determines which views are packed.
      */
     enum AVStereo3DView view;
+
+    /**
+     * Which eye is the primary eye when rendering in 2D.
+     */
+    enum AVStereo3DPrimaryEye primary_eye;
+
+    /**
+     * The distance between the centres of the lenses of the camera system,
+     * in micrometers. Zero if unset.
+     */
+    uint32_t baseline;
+
+    /**
+     * Relative shift of the left and right images, which changes the zero parallax plane.
+     * Range is -1.0 to 1.0. Zero if unset.
+     */
+    AVRational horizontal_disparity_adjustment;
+
+    /**
+     * Horizontal field of view in thousanths of a degree. Zero if unset.
+     */
+    uint32_t horizontal_field_of_view;
 } AVStereo3D;
 
 /**
@@ -222,6 +264,42 @@ const char *av_stereo3d_type_name(unsigned int type);
  */
 int av_stereo3d_from_name(const char *name);
 
+/**
+ * Provide a human-readable name of a given stereo3d view.
+ *
+ * @param type The input stereo3d view value.
+ *
+ * @return The name of the stereo3d view value, or "unknown".
+ */
+const char *av_stereo3d_view_name(unsigned int view);
+
+/**
+ * Get the AVStereo3DView form a human-readable name.
+ *
+ * @param name The input string.
+ *
+ * @return The AVStereo3DView value, or -1 if not found.
+ */
+int av_stereo3d_view_from_name(const char *name);
+
+/**
+ * Provide a human-readable name of a given stereo3d primary eye.
+ *
+ * @param type The input stereo3d primary eye value.
+ *
+ * @return The name of the stereo3d primary eye value, or "unknown".
+ */
+const char *av_stereo3d_primary_eye_name(unsigned int eye);
+
+/**
+ * Get the AVStereo3DPrimaryEye form a human-readable name.
+ *
+ * @param name The input string.
+ *
+ * @return The AVStereo3DPrimaryEye value, or -1 if not found.
+ */
+int av_stereo3d_primary_eye_from_name(const char *name);
+
 /**
  * @}
  */
diff --git a/libavutil/version.h b/libavutil/version.h
index 7df546ee22..8044fd3935 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -79,7 +79,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  59
-#define LIBAVUTIL_VERSION_MINOR  23
+#define LIBAVUTIL_VERSION_MINOR  24
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
-- 
2.43.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v3 2/5] avutil/stereo3d: Fill out stereo info provided by Vision Pro files

[James Almer]

On 6/17/2024 4:20 PM, Derek Buitenhuis wrote:
> Based on what is in the files themselves, and what the API provides
> to users.
> 
> URLs:
>    * https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_heroeye
>    * https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_stereocamerabaseline
>    * https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_horizontaldisparityadjustment
>    * https://developer.apple.com/documentation/coremedia/kcmformatdescriptionextension_horizontalfieldofview
> 
> Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>
> ---
>   libavutil/stereo3d.c | 52 +++++++++++++++++++++++++++++
>   libavutil/stereo3d.h | 78 ++++++++++++++++++++++++++++++++++++++++++++
>   libavutil/version.h  |  2 +-
>   3 files changed, 131 insertions(+), 1 deletion(-)

Missing APIChanges entry (no need to resend, you can just amend before 
pushing).
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v3 2/5] avutil/stereo3d: Fill out stereo info provided by Vision Pro files

[Derek Buitenhuis]

On 6/17/2024 9:03 PM, James Almer wrote:
> Missing APIChanges entry (no need to resend, you can just amend before 
> pushing).

Done locally for this and previous commit.

- Derek
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH v3 3/5] fftools/ffprobe: Print more Stereo 3D info from side data

[Derek Buitenhuis]

Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>
---
 fftools/ffprobe.c                            | 8 ++++++++
 tests/ref/fate/matroska-spherical-mono       | 2 ++
 tests/ref/fate/matroska-spherical-mono-remux | 4 ++++
 tests/ref/fate/matroska-stereo_mode          | 8 ++++++++
 tests/ref/fate/matroska-vp8-alpha-remux      | 2 ++
 tests/ref/fate/mov-spherical-mono            | 2 ++
 6 files changed, 26 insertions(+)

diff --git a/fftools/ffprobe.c b/fftools/ffprobe.c
index 2d38e5dfdc..a814cb5ade 100644
--- a/fftools/ffprobe.c
+++ b/fftools/ffprobe.c
@@ -2544,6 +2544,14 @@ static void print_pkt_side_data(WriterContext *w,
             const AVStereo3D *stereo = (AVStereo3D *)sd->data;
             print_str("type", av_stereo3d_type_name(stereo->type));
             print_int("inverted", !!(stereo->flags & AV_STEREO3D_FLAG_INVERT));
+            print_str("view", av_stereo3d_view_name(stereo->view));
+            print_str("primary_eye", av_stereo3d_primary_eye_name(stereo->primary_eye));
+            if (stereo->baseline)
+                print_int("baseline", stereo->baseline);
+            if (stereo->horizontal_disparity_adjustment.num && stereo->horizontal_disparity_adjustment.den)
+                print_q("horizontal_disparity_adjustment", stereo->horizontal_disparity_adjustment, '/');
+            if (stereo->horizontal_field_of_view)
+                print_int("horizontal_field_of_view", stereo->horizontal_field_of_view);
         } else if (sd->type == AV_PKT_DATA_SPHERICAL) {
             const AVSphericalMapping *spherical = (AVSphericalMapping *)sd->data;
             print_str("projection", av_spherical_projection_name(spherical->projection));
diff --git a/tests/ref/fate/matroska-spherical-mono b/tests/ref/fate/matroska-spherical-mono
index bd57d94514..08b94e455b 100644
--- a/tests/ref/fate/matroska-spherical-mono
+++ b/tests/ref/fate/matroska-spherical-mono
@@ -3,6 +3,8 @@
 side_data_type=Stereo 3D
 type=2D
 inverted=0
+view=packed
+primary_eye=none
 [/SIDE_DATA]
 [SIDE_DATA]
 side_data_type=Spherical Mapping
diff --git a/tests/ref/fate/matroska-spherical-mono-remux b/tests/ref/fate/matroska-spherical-mono-remux
index 6fcda14822..0ca77c8074 100644
--- a/tests/ref/fate/matroska-spherical-mono-remux
+++ b/tests/ref/fate/matroska-spherical-mono-remux
@@ -27,6 +27,8 @@ DISPOSITION:forced=1
 side_data_type=Stereo 3D
 type=2D
 inverted=0
+view=packed
+primary_eye=none
 [/SIDE_DATA]
 [SIDE_DATA]
 side_data_type=Spherical Mapping
@@ -51,6 +53,8 @@ DISPOSITION:forced=0
 side_data_type=Stereo 3D
 type=2D
 inverted=0
+view=packed
+primary_eye=none
 [/SIDE_DATA]
 [SIDE_DATA]
 side_data_type=Spherical Mapping
diff --git a/tests/ref/fate/matroska-stereo_mode b/tests/ref/fate/matroska-stereo_mode
index 739b789fea..13bce13cb8 100644
--- a/tests/ref/fate/matroska-stereo_mode
+++ b/tests/ref/fate/matroska-stereo_mode
@@ -132,6 +132,8 @@ TAG:DURATION=00:00:10.000000000
 side_data_type=Stereo 3D
 type=side by side
 inverted=0
+view=packed
+primary_eye=none
 [/SIDE_DATA]
 [/STREAM]
 [STREAM]
@@ -147,6 +149,8 @@ TAG:DURATION=00:00:10.000000000
 side_data_type=Stereo 3D
 type=top and bottom
 inverted=1
+view=packed
+primary_eye=none
 [/SIDE_DATA]
 [/STREAM]
 [STREAM]
@@ -160,6 +164,8 @@ TAG:DURATION=00:00:10.000000000
 side_data_type=Stereo 3D
 type=interleaved lines
 inverted=1
+view=packed
+primary_eye=none
 [/SIDE_DATA]
 [/STREAM]
 [STREAM]
@@ -174,6 +180,8 @@ TAG:DURATION=00:00:10.000000000
 side_data_type=Stereo 3D
 type=interleaved columns
 inverted=1
+view=packed
+primary_eye=none
 [/SIDE_DATA]
 [/STREAM]
 [STREAM]
diff --git a/tests/ref/fate/matroska-vp8-alpha-remux b/tests/ref/fate/matroska-vp8-alpha-remux
index f6c24dead6..e54304cafd 100644
--- a/tests/ref/fate/matroska-vp8-alpha-remux
+++ b/tests/ref/fate/matroska-vp8-alpha-remux
@@ -35,5 +35,7 @@ DISPOSITION:still_image=0
 side_data_type=Stereo 3D
 type=2D
 inverted=0
+view=packed
+primary_eye=none
 [/SIDE_DATA]
 [/STREAM]
diff --git a/tests/ref/fate/mov-spherical-mono b/tests/ref/fate/mov-spherical-mono
index bd57d94514..08b94e455b 100644
--- a/tests/ref/fate/mov-spherical-mono
+++ b/tests/ref/fate/mov-spherical-mono
@@ -3,6 +3,8 @@
 side_data_type=Stereo 3D
 type=2D
 inverted=0
+view=packed
+primary_eye=none
 [/SIDE_DATA]
 [SIDE_DATA]
 side_data_type=Spherical Mapping
-- 
2.43.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH v3 4/5] avformat/mov: Add support for exporting Video Extension Usage info

[Derek Buitenhuis]

This box is provided by files created by the Apple Vision Pro, as well
as the iPhone 15+ when capture for Vision Pro is enabled.

The boxes are a mix of things documented by Apple in some PDFs, their
API docs, and reverse engineering. Ideally we will have a real spec
one day.

Links:
  * https://developer.apple.com/av-foundation/Stereo-Video-ISOBMFF-Extensions.pdf
  * https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_horizontaldisparityadjustment
  * https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_stereocamerabaseline
  * https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_heroeye

Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>
---
 libavformat/mov.c | 283 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 283 insertions(+)

diff --git a/libavformat/mov.c b/libavformat/mov.c
index 9016cd5ad0..5724b4ef93 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -6477,6 +6477,288 @@ static int mov_read_sv3d(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     return 0;
 }
 
+static int mov_read_vexu_proj(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+    AVStream *st;
+    MOVStreamContext *sc;
+    int size;
+    uint32_t tag;
+    enum AVSphericalProjection projection;
+
+    if (c->fc->nb_streams < 1)
+        return 0;
+
+    st = c->fc->streams[c->fc->nb_streams - 1];
+    sc = st->priv_data;
+
+    if (atom.size != 16) {
+        av_log(c->fc, AV_LOG_ERROR, "Invalid size for proj box: %"PRIu64"\n", atom.size);
+        return AVERROR_INVALIDDATA;
+    }
+
+    size = avio_rb32(pb);
+    if (size != 16) {
+        av_log(c->fc, AV_LOG_ERROR, "Invalid size for prji box: %d\n", size);
+        return AVERROR_INVALIDDATA;
+    }
+
+    tag = avio_rl32(pb);
+    if (tag != MKTAG('p','r','j','i')) {
+        av_log(c->fc, AV_LOG_ERROR, "Invalid child box of proj box: 0x%08X\n", tag);
+        return AVERROR_INVALIDDATA;
+    }
+
+    avio_skip(pb, 1); // version
+    avio_skip(pb, 3); // flags
+
+    tag = avio_rl32(pb);
+    switch (tag) {
+    case MKTAG('r','e','c','t'):
+        projection = AV_SPHERICAL_RECTILINEAR;
+        break;
+    case MKTAG('e','q','u','i'):
+        projection = AV_SPHERICAL_EQUIRECTANGULAR;
+        break;
+    case MKTAG('h','e','q','u'):
+        projection = AV_SPHERICAL_HALF_EQUIRECTANGULAR;
+        break;
+    case MKTAG('f','i','s','h'):
+        projection = AV_SPHERICAL_FISHEYE;
+        break;
+    default:
+        av_log(c->fc, AV_LOG_ERROR, "Invalid projection type in prji box: 0x%08X\n", tag);
+        return AVERROR_INVALIDDATA;
+    }
+
+    sc->spherical = av_spherical_alloc(&sc->spherical_size);
+    if (!sc->spherical)
+        return AVERROR(ENOMEM);
+
+    sc->spherical->projection = projection;
+
+    return 0;
+}
+
+static int mov_read_eyes(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+    AVStream *st;
+    MOVStreamContext *sc;
+    int size, flags = 0;
+    int64_t remaining;
+    uint32_t tag, baseline = 0;
+    enum AVStereo3DView view = AV_STEREO3D_VIEW_PACKED;
+    enum AVStereo3DPrimaryEye primary_eye = AV_PRIMARY_EYE_NONE;
+    AVRational horizontal_disparity_adjustment = { 0, 0 };
+
+    if (c->fc->nb_streams < 1)
+        return 0;
+
+    st = c->fc->streams[c->fc->nb_streams - 1];
+    sc = st->priv_data;
+
+    remaining = atom.size;
+    while (remaining > 0) {
+        size = avio_rb32(pb);
+        if (size < 8 || size > remaining ) {
+            av_log(c->fc, AV_LOG_ERROR, "Invalid child size in eyes box\n");
+            return AVERROR_INVALIDDATA;
+        }
+
+        tag = avio_rl32(pb);
+        switch (tag) {
+        case MKTAG('s','t','r','i'): {
+            int has_right, has_left;
+            uint8_t tmp;
+            if (size != 13) {
+                av_log(c->fc, AV_LOG_ERROR, "Invalid size of stri box: %d\n", size);
+                return AVERROR_INVALIDDATA;
+            }
+            avio_skip(pb, 1); // version
+            avio_skip(pb, 3); // flags
+
+            tmp = avio_r8(pb);
+
+            // eye_views_reversed
+            if (tmp & 8) {
+                flags |= AV_STEREO3D_FLAG_INVERT;
+            }
+            // has_additional_views
+            if (tmp & 4) {
+                // skip...
+            }
+
+            has_right = tmp & 2; // has_right_eye_view
+            has_left = tmp & 1; // has_left_eye_view
+
+            if (has_left && has_right)
+                view = AV_STEREO3D_VIEW_PACKED;
+            else if (has_left)
+                view = AV_STEREO3D_VIEW_LEFT;
+            else if (has_right)
+                view = AV_STEREO3D_VIEW_RIGHT;
+            break;
+        }
+        case MKTAG('h','e','r','o'): {
+            int tmp;
+            if (size != 13) {
+                av_log(c->fc, AV_LOG_ERROR, "Invalid size of hero box: %d\n", size);
+                return AVERROR_INVALIDDATA;
+            }
+            avio_skip(pb, 1); // version
+            avio_skip(pb, 3); // flags
+
+            tmp = avio_r8(pb);
+            if (tmp == 0)
+                primary_eye = AV_PRIMARY_EYE_NONE;
+            else if (tmp == 1)
+                primary_eye = AV_PRIMARY_EYE_LEFT;
+            else if (tmp == 2)
+                primary_eye = AV_PRIMARY_EYE_RIGHT;
+            else
+                av_log(c->fc, AV_LOG_WARNING, "Unknown hero eye type: %d\n", tmp);
+
+            break;
+        }
+        case MKTAG('c','a','m','s'): {
+            uint32_t subtag;
+            int subsize;
+            if (size != 24) {
+                av_log(c->fc, AV_LOG_ERROR, "Invalid size of cams box: %d\n", size);
+                return AVERROR_INVALIDDATA;
+            }
+
+            subsize = avio_rb32(pb);
+            if (subsize != 16) {
+                av_log(c->fc, AV_LOG_ERROR, "Invalid size of blin box: %d\n", size);
+                return AVERROR_INVALIDDATA;
+            }
+
+            subtag = avio_rl32(pb);
+            if (subtag != MKTAG('b','l','i','n')) {
+                av_log(c->fc, AV_LOG_ERROR, "Expected blin box, got 0x%08X\n", subtag);
+                return AVERROR_INVALIDDATA;
+            }
+
+            avio_skip(pb, 1); // version
+            avio_skip(pb, 3); // flags
+
+            baseline = avio_rb32(pb);
+
+            break;
+        }
+        case MKTAG('c','m','f','y'): {
+            uint32_t subtag;
+            int subsize;
+            int32_t adjustment;
+            if (size != 24) {
+                av_log(c->fc, AV_LOG_ERROR, "Invalid size of cmfy box: %d\n", size);
+                return AVERROR_INVALIDDATA;
+            }
+
+            subsize = avio_rb32(pb);
+            if (subsize != 16) {
+                av_log(c->fc, AV_LOG_ERROR, "Invalid size of dadj box: %d\n", size);
+                return AVERROR_INVALIDDATA;
+            }
+
+            subtag = avio_rl32(pb);
+            if (subtag != MKTAG('d','a','d','j')) {
+                av_log(c->fc, AV_LOG_ERROR, "Expected dadj box, got 0x%08X\n", subtag);
+                return AVERROR_INVALIDDATA;
+            }
+
+            avio_skip(pb, 1); // version
+            avio_skip(pb, 3); // flags
+
+            adjustment = (int32_t) avio_rb32(pb);
+
+            horizontal_disparity_adjustment.num = (int) adjustment;
+            horizontal_disparity_adjustment.den = 10000;
+
+            break;
+        }
+        default:
+            av_log(c->fc, AV_LOG_WARNING, "Unknown tag in eyes: 0x%08X\n", tag);
+            avio_skip(pb, size - 8);
+            break;
+        }
+        remaining -= size;
+    }
+
+    if (remaining != 0) {
+        av_log(c->fc, AV_LOG_ERROR, "Broken eyes box\n");
+        return AVERROR_INVALIDDATA;
+    }
+
+    if (!sc->stereo3d) {
+        sc->stereo3d = av_stereo3d_alloc();
+        if (!sc->stereo3d)
+            return AVERROR(ENOMEM);
+    }
+
+    sc->stereo3d->flags                           = flags;
+    sc->stereo3d->view                            = view;
+    sc->stereo3d->primary_eye                     = primary_eye;
+    sc->stereo3d->baseline                        = baseline;
+    sc->stereo3d->horizontal_disparity_adjustment = horizontal_disparity_adjustment;
+
+    return 0;
+}
+
+static int mov_read_vexu(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+    int size;
+    int64_t remaining;
+    uint32_t tag;
+
+    if (c->fc->nb_streams < 1)
+        return 0;
+
+    if (atom.size < 8) {
+        av_log(c->fc, AV_LOG_ERROR, "Empty video extension usage box\n");
+        return AVERROR_INVALIDDATA;
+    }
+
+    remaining = atom.size;
+    while (remaining > 0) {
+        size = avio_rb32(pb);
+        if (size < 8 || size > remaining ) {
+            av_log(c->fc, AV_LOG_ERROR, "Invalid child size in vexu box\n");
+            return AVERROR_INVALIDDATA;
+        }
+
+        tag = avio_rl32(pb);
+        switch (tag) {
+        case MKTAG('p','r','o','j'): {
+            MOVAtom proj = { tag, size - 8 };
+            int ret = mov_read_vexu_proj(c, pb, proj);
+            if (ret < 0)
+                return ret;
+            break;
+        }
+        case MKTAG('e','y','e','s'): {
+            MOVAtom eyes = { tag, size - 8 };
+            int ret = mov_read_eyes(c, pb, eyes);
+            if (ret < 0)
+                return ret;
+            break;
+        }
+        default:
+            av_log(c->fc, AV_LOG_WARNING, "Unknown tag in vexu: 0x%08X\n", tag);
+            avio_skip(pb, size - 8);
+            break;
+        }
+        remaining -= size;
+    }
+
+    if (remaining != 0) {
+        av_log(c->fc, AV_LOG_ERROR, "Broken vexu box\n");
+        return AVERROR_INVALIDDATA;
+    }
+
+    return 0;
+}
+
 static int mov_parse_uuid_spherical(MOVStreamContext *sc, AVIOContext *pb, size_t len)
 {
     int ret = 0;
@@ -8595,6 +8877,7 @@ static const MOVParseTableEntry mov_default_parse_table[] = {
 { MKTAG('d','f','L','a'), mov_read_dfla },
 { MKTAG('s','t','3','d'), mov_read_st3d }, /* stereoscopic 3D video box */
 { MKTAG('s','v','3','d'), mov_read_sv3d }, /* spherical video box */
+{ MKTAG('v','e','x','u'), mov_read_vexu }, /* video extension usage */
 { MKTAG('d','O','p','s'), mov_read_dops },
 { MKTAG('d','m','l','p'), mov_read_dmlp },
 { MKTAG('S','m','D','m'), mov_read_smdm },
-- 
2.43.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v3 4/5] avformat/mov: Add support for exporting Video Extension Usage info

[James Almer]

On 6/17/2024 4:20 PM, Derek Buitenhuis wrote:
> This box is provided by files created by the Apple Vision Pro, as well
> as the iPhone 15+ when capture for Vision Pro is enabled.
> 
> The boxes are a mix of things documented by Apple in some PDFs, their
> API docs, and reverse engineering. Ideally we will have a real spec
> one day.
> 
> Links:
>    *https://developer.apple.com/av-foundation/Stereo-Video-ISOBMFF-Extensions.pdf
>    *https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_horizontaldisparityadjustment
>    *https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_stereocamerabaseline
>    *https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_heroeye
> 
> Signed-off-by: Derek Buitenhuis<derek.buitenhuis@gmail.com>
> ---
>   libavformat/mov.c | 283 ++++++++++++++++++++++++++++++++++++++++++++++
>   1 file changed, 283 insertions(+)
> 
> diff --git a/libavformat/mov.c b/libavformat/mov.c
> index 9016cd5ad0..5724b4ef93 100644
> --- a/libavformat/mov.c
> +++ b/libavformat/mov.c
> @@ -6477,6 +6477,288 @@ static int mov_read_sv3d(MOVContext *c, AVIOContext *pb, MOVAtom atom)
>       return 0;
>   }
>   
> +static int mov_read_vexu_proj(MOVContext *c, AVIOContext *pb, MOVAtom atom)
> +{
> +    AVStream *st;
> +    MOVStreamContext *sc;
> +    int size;
> +    uint32_t tag;
> +    enum AVSphericalProjection projection;
> +
> +    if (c->fc->nb_streams < 1)
> +        return 0;
> +
> +    st = c->fc->streams[c->fc->nb_streams - 1];
> +    sc = st->priv_data;
> +
> +    if (atom.size != 16) {
> +        av_log(c->fc, AV_LOG_ERROR, "Invalid size for proj box: %"PRIu64"\n", atom.size);
> +        return AVERROR_INVALIDDATA;
> +    }
> +
> +    size = avio_rb32(pb);
> +    if (size != 16) {
> +        av_log(c->fc, AV_LOG_ERROR, "Invalid size for prji box: %d\n", size);
> +        return AVERROR_INVALIDDATA;
> +    }
> +
> +    tag = avio_rl32(pb);
> +    if (tag != MKTAG('p','r','j','i')) {
> +        av_log(c->fc, AV_LOG_ERROR, "Invalid child box of proj box: 0x%08X\n", tag);
> +        return AVERROR_INVALIDDATA;
> +    }
> +
> +    avio_skip(pb, 1); // version
> +    avio_skip(pb, 3); // flags
> +
> +    tag = avio_rl32(pb);
> +    switch (tag) {
> +    case MKTAG('r','e','c','t'):
> +        projection = AV_SPHERICAL_RECTILINEAR;
> +        break;
> +    case MKTAG('e','q','u','i'):
> +        projection = AV_SPHERICAL_EQUIRECTANGULAR;
> +        break;
> +    case MKTAG('h','e','q','u'):
> +        projection = AV_SPHERICAL_HALF_EQUIRECTANGULAR;
> +        break;
> +    case MKTAG('f','i','s','h'):
> +        projection = AV_SPHERICAL_FISHEYE;
> +        break;
> +    default:
> +        av_log(c->fc, AV_LOG_ERROR, "Invalid projection type in prji box: 0x%08X\n", tag);
> +        return AVERROR_INVALIDDATA;
> +    }
> +
> +    sc->spherical = av_spherical_alloc(&sc->spherical_size);
> +    if (!sc->spherical)
> +        return AVERROR(ENOMEM);
> +
> +    sc->spherical->projection = projection;
> +
> +    return 0;
> +}
> +
> +static int mov_read_eyes(MOVContext *c, AVIOContext *pb, MOVAtom atom)
> +{
> +    AVStream *st;
> +    MOVStreamContext *sc;
> +    int size, flags = 0;
> +    int64_t remaining;
> +    uint32_t tag, baseline = 0;
> +    enum AVStereo3DView view = AV_STEREO3D_VIEW_PACKED;
> +    enum AVStereo3DPrimaryEye primary_eye = AV_PRIMARY_EYE_NONE;
> +    AVRational horizontal_disparity_adjustment = { 0, 0 };

nit: { 0, 1 }.

Should be ok either way.
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v3 4/5] avformat/mov: Add support for exporting Video Extension Usage info

[Derek Buitenhuis]

On 6/17/2024 9:07 PM, James Almer wrote:
> nit: { 0, 1 }.

If we set it to 0, 1 here, should it be set to that in the alloc function too?
It'll be 0, 0 at alloc time due to use of av_mallocz.

- Derek
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v3 4/5] avformat/mov: Add support for exporting Video Extension Usage info

[James Almer]

On 6/17/2024 5:27 PM, Derek Buitenhuis wrote:
> On 6/17/2024 9:07 PM, James Almer wrote:
>> nit: { 0, 1 }.
> 
> If we set it to 0, 1 here, should it be set to that in the alloc function too?
> It'll be 0, 0 at alloc time due to use of av_mallocz.

Personally I'd say yes, but other similar lavu APIs (Mastering, Ambient 
Viewing) don't seem to bother with it, so maybe just leave it as 0 
unless someone else has a strong opinion about it.
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v3 4/5] avformat/mov: Add support for exporting Video Extension Usage info

[Derek Buitenhuis]

On 6/17/2024 9:33 PM, James Almer wrote:
> Personally I'd say yes, but other similar lavu APIs (Mastering, Ambient 
> Viewing) don't seem to bother with it, so maybe just leave it as 0 
> unless someone else has a strong opinion about it.

I'll leave it for consistency then.

- Derek
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH v3 5/5] avformat/mov: Add support for reading and exporting horizontal field of view

[Derek Buitenhuis]

These boxes are created by the Apple Vision Pro and the iPhone 15+ when
capture for the Vision Pro is enabled.

Based off of the swift API:
  * https://developer.apple.com/documentation/coremedia/kcmformatdescriptionextension_horizontalfieldofview

Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>
---
 libavformat/mov.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/libavformat/mov.c b/libavformat/mov.c
index 5724b4ef93..367af8478b 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -6759,6 +6759,34 @@ static int mov_read_vexu(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     return 0;
 }
 
+static int mov_read_hfov(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+    AVStream *st;
+    MOVStreamContext *sc;
+
+    if (c->fc->nb_streams < 1)
+        return 0;
+
+    st = c->fc->streams[c->fc->nb_streams - 1];
+    sc = st->priv_data;
+
+    if (atom.size != 4) {
+         av_log(c->fc, AV_LOG_ERROR, "Invalid size of hfov box: %"PRIu64"\n", atom.size);
+         return AVERROR_INVALIDDATA;
+    }
+
+
+    if (!sc->stereo3d) {
+        sc->stereo3d = av_stereo3d_alloc();
+        if (!sc->stereo3d)
+            return AVERROR(ENOMEM);
+    }
+
+    sc->stereo3d->horizontal_field_of_view = avio_rb32(pb);
+
+    return 0;
+}
+
 static int mov_parse_uuid_spherical(MOVStreamContext *sc, AVIOContext *pb, size_t len)
 {
     int ret = 0;
@@ -8878,6 +8906,7 @@ static const MOVParseTableEntry mov_default_parse_table[] = {
 { MKTAG('s','t','3','d'), mov_read_st3d }, /* stereoscopic 3D video box */
 { MKTAG('s','v','3','d'), mov_read_sv3d }, /* spherical video box */
 { MKTAG('v','e','x','u'), mov_read_vexu }, /* video extension usage */
+{ MKTAG('h','f','o','v'), mov_read_hfov },
 { MKTAG('d','O','p','s'), mov_read_dops },
 { MKTAG('d','m','l','p'), mov_read_dmlp },
 { MKTAG('S','m','D','m'), mov_read_smdm },
-- 
2.43.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v3 0/5] Vision Pro Spatial Data

[Derek Buitenhuis]

On 6/17/2024 8:20 PM, Derek Buitenhuis wrote:
>  12 files changed, 490 insertions(+), 1 deletion(-)

Will push later today if there are no objections.

- Derek
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".