Video stream bframe support

crates/store/re_data_loader/src/lerobot/datasetv3.rs

@@ -603,7 +603,7 @@ impl LeRobotDatasetV3 {
         let (samples_meta, samples): (Vec<_>, Vec<_>) = samples.into_iter().unzip();
 
         let samples_column = VideoStream::update_fields()
-            .with_many_sample(samples)
+            .with_sample(samples)
             .columns_of_unit_batches()
             .with_context(|| "Failed to create VideoStream")?;
 

crates/store/re_mcap/src/parsers/ros2msg/sensor_msgs/compressed_image.rs

@@ -82,7 +82,7 @@ impl MessageParser for CompressedImageMessageParser {
                     .collect()
             }
             ParsedPayloadKind::H264 => VideoStream::update_fields()
-                .with_many_sample(blobs)
+                .with_sample(blobs)
                 .columns_of_unit_batches()?
                 .collect(),
 

crates/store/re_sdk_types/definitions/rerun/archetypes/video_stream.fbs

@@ -28,11 +28,17 @@ table VideoStream (
 
     /// Video sample data (also known as "video chunk").
     ///
-    /// The current timestamp is used as presentation timestamp (PTS) for all data in this sample.
-    /// There is currently no way to log differing decoding timestamps, meaning
-    /// that there is no support for B-frames.
-    /// See <https://github.com/rerun-io/rerun/issues/10090> for more details.
-    // TODO(#10090): See above.
+    /// Each element in this array must contain the encoded data for exactly one video frame.
+    /// When logging a single frame per row, this is simply a single-element array (or a scalar).
+    ///
+    /// For B-frame support (H.264/H.265), multiple samples can be batched into a single row.
+    /// Samples within a row must be in **decode order**.
+    /// The current timestamp on the timeline is used as the decode timestamp (DTS) for the first sample;
+    /// subsequent samples in the same row are assigned sequentially increasing DTS values.
+    /// See `presentation_time_offset` for specifying presentation timestamps that differ from decode timestamps.
+    ///
+    /// When no `presentation_time_offset` is provided, the presentation timestamp (PTS)
+    /// equals the decode timestamp for each sample (i.e. no B-frames).
     ///
     /// Rerun chunks containing frames (i.e. bundles of sample data) may arrive out of order,
     /// but may cause the video playback in the Viewer to reset.
@@ -42,18 +48,30 @@ table VideoStream (
     /// codec parameters & resolution.
     ///
     /// The samples are expected to be encoded using the `codec` field.
-    /// Each video sample must contain enough data for exactly one video frame
-    /// (this restriction may be relaxed in the future for some codecs).
     ///
     /// Unless your stream consists entirely of key-frames (in which case you should consider [archetypes.EncodedImage])
     /// never log this component as static data as this means that you loose all information of
     /// previous samples which may be required to decode an image.
     ///
     /// See [components.VideoCodec] for codec specific requirements.
-    sample: rerun.components.VideoSample ("attr.rerun.component_recommended", nullable, order: 2000);
+    sample: [rerun.components.VideoSample] ("attr.rerun.component_recommended", nullable, order: 2000);
 
     // --- Optional ---
 
+    /// Per-sample offset from decode timestamp (DTS) to presentation timestamp (PTS).
+    ///
+    /// When present, the presentation timestamp for sample `i` is computed as
+    /// `DTS_i + presentation_time_offset[i]`, where `DTS_i` is the decode timestamp of that sample.
+    ///
+    /// This is needed for H.264/H.265 streams that use B-frames (bidirectionally predicted frames),
+    /// as B-frames cause decode order and presentation order to differ.
+    ///
+    /// When absent, the presentation timestamp equals the decode timestamp for all samples
+    /// in this row (i.e. no B-frames, which is the common case for low-latency streaming).
+    ///
+    /// The number of offsets should match the number of samples in this row.
+    presentation_time_offset: [rerun.components.VideoPresentationTimestampOffset] ("attr.rerun.component_optional", nullable, order: 2500);
+
     // TODO(#3982): add orientation.
 
     /// Opacity of the video stream, useful for layering several media.

crates/store/re_sdk_types/definitions/rerun/components/video_presentation_timestamp_offset.fbs

@@ -0,0 +1,16 @@
+namespace rerun.components;
+
+/// An offset from decode timestamp (DTS) to presentation timestamp (PTS) for a video sample.
+///
+/// Used with [archetypes.VideoStream] to support B-frames in H.264/H.265 streams.
+/// When present, the presentation timestamp is computed as `DTS + offset`.
+/// When absent, the presentation timestamp equals the decode timestamp (no B-frames).
+struct VideoPresentationTimestampOffset (
+  "attr.arrow.transparent",
+  "attr.python.array_aliases": "npt.NDArray[np.int64]",
+  "attr.rerun.state": "unstable",
+  "attr.rust.derive": "Copy, PartialEq, Eq, Default",
+  "attr.rust.repr": "transparent"
+) {
+  offset: rerun.datatypes.VideoPresentationTimestampOffset (order: 100);
+}

crates/store/re_sdk_types/definitions/rerun/datatypes/video_presentation_timestamp_offset.fbs

@@ -0,0 +1,20 @@
+namespace rerun.datatypes;
+
+/// An offset from decode timestamp (DTS) to presentation timestamp (PTS) for a video sample.
+///
+/// The offset is specified in timeline units (the same units as the timeline on which
+/// the video stream samples are logged).
+///
+/// The presentation timestamp of a sample is `DTS + offset`.
+/// When no offset is specified, the presentation timestamp equals the decode timestamp
+/// (i.e. zero offset, no B-frames).
+struct VideoPresentationTimestampOffset (
+  "attr.arrow.transparent",
+  "attr.python.aliases": "int",
+  "attr.python.array_aliases": "npt.NDArray[np.int64]",
+  "attr.rust.derive": "Default, Copy, PartialEq, Eq, PartialOrd, Ord",
+  "attr.rust.tuple_struct"
+) {
+    /// Offset value in timeline units.
+    offset: long (order: 100);
+}