qtmux: handle 'late' streams

When muxing streams, some can start later than others. qtmux
now handle this by adding an empty edts entry with the
duration of the 'lateness' to the stream's trak.
It tolerates a stream to be up to 0.1s late.

Fixes #586848

4 files changed, 89 insertions, 0 deletions

diff --git a/gst/qtmux/atoms.c b/gst/qtmux/atoms.c
index 2165e7e0e..a0844c2e9 100644
--- a/gst/qtmux/atoms.c
+++ b/gst/qtmux/atoms.c
@@ -2889,6 +2889,34 @@ atom_tkhd_set_video (AtomTKHD * tkhd, AtomsContext * context, guint32 width,
   tkhd->height = height;
 }
 
+static void
+atom_edts_add_entry (AtomEDTS * edts, EditListEntry * entry)
+{
+  edts->elst.entries = g_slist_append (edts->elst.entries, entry);
+}
+
+/* 
+ * Adds a new entry to this trak edits list
+ * duration is in the moov's timescale
+ * media_time is the offset in the media time to start from (media's timescale)
+ * rate is a 32 bits fixed-point
+ */
+void
+atom_trak_add_elst_entry (AtomTRAK * trak, guint32 duration, guint32 media_time,
+    guint32 rate)
+{
+  EditListEntry *entry = g_new (EditListEntry, 1);
+
+  entry->duration = duration;
+  entry->media_time = media_time;
+  entry->media_rate = rate;
+
+  if (trak->edts == NULL) {
+    trak->edts = atom_edts_new ();
+  }
+  atom_edts_add_entry (trak->edts, entry);
+}
+
 /* re-negotiation is prevented at top-level, so only 1 entry expected.
  * Quite some more care here and elsewhere may be needed to
  * support several entries */
diff --git a/gst/qtmux/atoms.h b/gst/qtmux/atoms.h
index 6c4b2790c..07efef5dc 100644
--- a/gst/qtmux/atoms.h
+++ b/gst/qtmux/atoms.h
@@ -611,6 +611,8 @@ AtomTRAK*  atom_trak_new               (AtomsContext *context);
 void       atom_trak_add_samples       (AtomTRAK * trak, guint32 nsamples, guint32 delta,
                                         guint32 size, guint64 chunk_offset, gboolean sync,
                                         gboolean do_pts, gint64 pts_offset);
+void       atom_trak_add_elst_entry    (AtomTRAK * trak, guint32 duration,
+                                        guint32 media_time, guint32 rate);
 guint32    atom_trak_get_timescale     (AtomTRAK *trak);
 
 AtomMOOV*  atom_moov_new               (AtomsContext *context);
diff --git a/gst/qtmux/gstqtmux.c b/gst/qtmux/gstqtmux.c
index 41ea97f7a..2e873c9d8 100644
--- a/gst/qtmux/gstqtmux.c
+++ b/gst/qtmux/gstqtmux.c
@@ -113,6 +113,7 @@ enum
 
 /* some spare for header size as well */
 #define MDAT_LARGE_FILE_LIMIT           ((guint64) 1024 * 1024 * 1024 * 2)
+#define MAX_TOLERATED_LATENESS          (GST_SECOND / 10)
 
 #define DEFAULT_LARGE_FILE              FALSE
 #define DEFAULT_MOVIE_TIMESCALE         1000
@@ -248,6 +249,7 @@ gst_qt_mux_pad_reset (GstQTPad * qtpad)
   qtpad->sample_size = 0;
   qtpad->sync = FALSE;
   qtpad->last_dts = 0;
+  qtpad->first_ts = GST_CLOCK_TIME_NONE;
 
   if (qtpad->last_buf)
     gst_buffer_replace (&qtpad->last_buf, NULL);
@@ -1113,6 +1115,7 @@ gst_qt_mux_stop_file (GstQTMux * qtmux)
   GSList *walk;
   gboolean large_file;
   guint32 timescale;
+  GstClockTime first_ts = GST_CLOCK_TIME_NONE;
 
   GST_DEBUG_OBJECT (qtmux, "Updating remaining values and sending last data");
 
@@ -1146,6 +1149,44 @@ gst_qt_mux_stop_file (GstQTMux * qtmux)
   atom_moov_set_64bits (qtmux->moov, large_file);
   atom_moov_update_duration (qtmux->moov);
 
+  /* check for late streams */
+  for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) {
+    GstCollectData *cdata = (GstCollectData *) walk->data;
+    GstQTPad *qtpad = (GstQTPad *) cdata;
+
+    if (!GST_CLOCK_TIME_IS_VALID (first_ts) ||
+        (GST_CLOCK_TIME_IS_VALID (qtpad->first_ts) &&
+            qtpad->first_ts < first_ts)) {
+      first_ts = qtpad->first_ts;
+    }
+  }
+  GST_DEBUG_OBJECT (qtmux, "Media first ts selected: %" GST_TIME_FORMAT,
+      GST_TIME_ARGS (first_ts));
+  /* add EDTSs for late streams */
+  for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) {
+    GstCollectData *cdata = (GstCollectData *) walk->data;
+    GstQTPad *qtpad = (GstQTPad *) cdata;
+    guint32 lateness;
+    guint32 duration;
+
+    if (GST_CLOCK_TIME_IS_VALID (qtpad->first_ts) &&
+        qtpad->first_ts > first_ts + MAX_TOLERATED_LATENESS) {
+      GST_DEBUG_OBJECT (qtmux, "Pad %s is a late stream by %" GST_TIME_FORMAT,
+          GST_PAD_NAME (qtpad->collect.pad),
+          GST_TIME_ARGS (qtpad->first_ts - first_ts));
+      lateness = gst_util_uint64_scale_round (qtpad->first_ts - first_ts,
+          timescale, GST_SECOND);
+      duration = qtpad->trak->tkhd.duration;
+      atom_trak_add_elst_entry (qtpad->trak, lateness, (guint32) - 1,
+          (guint32) (1 * 65536.0));
+      atom_trak_add_elst_entry (qtpad->trak, duration, 0,
+          (guint32) (1 * 65536.0));
+
+      /* need to add the empty time to the trak duration */
+      qtpad->trak->tkhd.duration += lateness;
+    }
+  }
+
   /* tags into file metadata */
   gst_qt_mux_setup_metadata (qtmux);
 
@@ -1370,6 +1411,20 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
     qtmux->longest_chunk = duration;
   }
 
+  /* if this is the first buffer, store the timestamp */
+  if (G_UNLIKELY (pad->first_ts == GST_CLOCK_TIME_NONE) && last_buf) {
+    if (GST_CLOCK_TIME_IS_VALID (GST_BUFFER_TIMESTAMP (last_buf))) {
+      pad->first_ts = GST_BUFFER_TIMESTAMP (last_buf);
+    } else {
+      GST_DEBUG_OBJECT (qtmux, "First buffer for pad %s has no timestamp, "
+          "using 0 as first timestamp");
+      pad->first_ts = 0;
+    }
+    GST_DEBUG_OBJECT (qtmux, "Stored first timestamp for pad %s %"
+        GST_TIME_FORMAT, GST_PAD_NAME (pad->collect.pad),
+        GST_TIME_ARGS (pad->first_ts));
+  }
+
   /* now we go and register this buffer/sample all over */
   /* note that a new chunk is started each time (not fancy but works) */
   atom_trak_add_samples (pad->trak, nsamples, scaled_duration, sample_size,
diff --git a/gst/qtmux/gstqtmux.h b/gst/qtmux/gstqtmux.h
index d12c23cdb..9d92bc1d4 100644
--- a/gst/qtmux/gstqtmux.h
+++ b/gst/qtmux/gstqtmux.h
@@ -82,6 +82,10 @@ typedef struct _GstQTPad
   /* dts of last_buf */
   GstClockTime last_dts;
 
+  /* store the first timestamp for comparing with other streams and
+   * know if there are late streams */
+  GstClockTime first_ts;
+
   /* all the atom and chunk book-keeping is delegated here
    * unowned/uncounted reference, parent MOOV owns */
   AtomTRAK *trak;