summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>2011-01-03 17:24:23 +0100
committerMark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>2011-01-03 17:40:52 +0100
commit38c93429784d10ddd4da0dbe390616b7e9196729 (patch)
tree5901fba387e96687e633d26fad25280ad2d684ef
parent460785352c0739529b17aa4ebc6558c8eb086ca1 (diff)
qtmux: adjust nasty case timestamp tracking
That is, all sorts of problems arise with re-ordered input timestamps that tend to defy automagic handling for every case, so allow for a few variations that can be tried depending on circumstances. Also try to document accordingly. Also fixes #638288.
-rw-r--r--gst/qtmux/gstqtmux.c235
-rw-r--r--gst/qtmux/gstqtmux.h8
2 files changed, 217 insertions, 26 deletions
diff --git a/gst/qtmux/gstqtmux.c b/gst/qtmux/gstqtmux.c
index 29edc2a23..0ac1e9114 100644
--- a/gst/qtmux/gstqtmux.c
+++ b/gst/qtmux/gstqtmux.c
@@ -82,6 +82,18 @@
* <link linkend="GstQTMux--streamable">streamable</link> allows foregoing to add
* index metadata (at the end of file).
*
+ * <link linkend="GstQTMux--dts-method">dts-method</link> allows selecting a
+ * method for managing input timestamps (stay tuned for 0.11 to have this
+ * automagically settled). The default delta/duration method should handle nice
+ * (aka perfect streams) just fine, but may experience problems otherwise
+ * (e.g. input stream with re-ordered B-frames and/or with frame dropping).
+ * The re-ordering approach re-assigns incoming timestamps in ascending order
+ * to incoming buffers and offers an alternative in such cases. In cases where
+ * that might fail, the remaining method can be tried, which is exact and
+ * according to specs, but might experience playback on not so spec-wise players.
+ * Note that this latter approach also requires one to enable
+ * <link linkend="GstQTMux--presentation-timestamp">presentation-timestamp</link>.
+ *
* <refsect2>
* <title>Example pipelines</title>
* |[
@@ -128,6 +140,36 @@
GST_DEBUG_CATEGORY_STATIC (gst_qt_mux_debug);
#define GST_CAT_DEFAULT gst_qt_mux_debug
+enum
+{
+ DTS_METHOD_DD,
+ DTS_METHOD_REORDER,
+ DTS_METHOD_ASC
+};
+
+static GType
+gst_qt_mux_dts_method_get_type (void)
+{
+ static GType gst_qt_mux_dts_method = 0;
+
+ if (!gst_qt_mux_dts_method) {
+ static const GEnumValue dts_methods[] = {
+ {DTS_METHOD_DD, "delta/duration", "dd"},
+ {DTS_METHOD_REORDER, "reorder", "reorder"},
+ {DTS_METHOD_ASC, "ascending", "asc"},
+ {0, NULL, NULL},
+ };
+
+ gst_qt_mux_dts_method =
+ g_enum_register_static ("GstQTMuxDtsMethods", dts_methods);
+ }
+
+ return gst_qt_mux_dts_method;
+}
+
+#define GST_TYPE_QT_MUX_DTS_METHOD \
+ (gst_qt_mux_dts_method_get_type ())
+
/* QTMux signals and args */
enum
{
@@ -140,12 +182,13 @@ enum
PROP_0,
PROP_MOVIE_TIMESCALE,
PROP_TRAK_TIMESCALE,
- PROP_DO_CTTS,
PROP_FAST_START,
PROP_FAST_START_TEMP_FILE,
PROP_MOOV_RECOV_FILE,
PROP_FRAGMENT_DURATION,
- PROP_STREAMABLE
+ PROP_STREAMABLE,
+ PROP_DTS_METHOD,
+ PROP_DO_CTTS,
};
/* some spare for header size as well */
@@ -160,6 +203,7 @@ enum
#define DEFAULT_MOOV_RECOV_FILE NULL
#define DEFAULT_FRAGMENT_DURATION 0
#define DEFAULT_STREAMABLE FALSE
+#define DEFAULT_DTS_METHOD DTS_METHOD_DD
static void gst_qt_mux_finalize (GObject * object);
@@ -264,6 +308,11 @@ gst_qt_mux_class_init (GstQTMuxClass * klass)
"(in addition to decoding time) (use with caution)",
DEFAULT_DO_CTTS,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
+ g_object_class_install_property (gobject_class, PROP_DTS_METHOD,
+ g_param_spec_enum ("dts-method", "dts-method",
+ "Method to determine DTS time",
+ GST_TYPE_QT_MUX_DTS_METHOD, DEFAULT_DTS_METHOD,
+ G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_FAST_START,
g_param_spec_boolean ("faststart", "Format file to faststart",
"If the file should be formated for faststart (headers first). ",
@@ -304,6 +353,8 @@ gst_qt_mux_class_init (GstQTMuxClass * klass)
static void
gst_qt_mux_pad_reset (GstQTPad * qtpad)
{
+ gint i;
+
qtpad->fourcc = 0;
qtpad->is_out_of_order = FALSE;
qtpad->have_dts = FALSE;
@@ -314,6 +365,16 @@ gst_qt_mux_pad_reset (GstQTPad * qtpad)
qtpad->prepare_buf_func = NULL;
qtpad->avg_bitrate = 0;
qtpad->max_bitrate = 0;
+ qtpad->ts_n_entries = 0;
+
+ qtpad->buf_head = 0;
+ qtpad->buf_tail = 0;
+ for (i = 0; i < G_N_ELEMENTS (qtpad->buf_entries); i++) {
+ if (qtpad->buf_entries[i]) {
+ gst_buffer_unref (qtpad->buf_entries[i]);
+ qtpad->buf_entries[i] = NULL;
+ }
+ }
if (qtpad->last_buf)
gst_buffer_replace (&qtpad->last_buf, NULL);
@@ -1930,12 +1991,50 @@ init:
return ret;
}
-/* check whether @a differs from @b by order of @magn */
-static gboolean inline
-gst_qtmux_check_difference (GstQTMux * qtmux, GstClockTime a,
- GstClockTime b, GstClockTime magn)
+/* sigh, tiny list helpers to re-order stuff */
+static void
+gst_qt_mux_push_ts (GstQTMux * qtmux, GstQTPad * pad, GstClockTime ts)
+{
+ gint i;
+
+ for (i = 0; (i < QTMUX_NO_OF_TS) && (i < pad->ts_n_entries); i++) {
+ if (ts > pad->ts_entries[i])
+ break;
+ }
+ memmove (&pad->ts_entries[i + 1], &pad->ts_entries[i],
+ sizeof (GstClockTime) * (pad->ts_n_entries - i));
+ pad->ts_entries[i] = ts;
+ pad->ts_n_entries++;
+}
+
+/* takes ownership of @buf */
+static GstBuffer *
+gst_qt_mux_get_asc_buffer_ts (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
{
- return ((a >= b) ? (a - b >= (magn >> 1)) : (b - a >= (magn >> 1)));
+ const gint wrap = G_N_ELEMENTS (pad->buf_entries);
+ GstClockTime ts;
+
+ /* store buffer and ts, latter ordered */
+ if (buf) {
+ pad->buf_entries[pad->buf_tail++] = buf;
+ pad->buf_tail %= wrap;
+ gst_qt_mux_push_ts (qtmux, pad, GST_BUFFER_TIMESTAMP (buf));
+ }
+
+ if (pad->ts_n_entries && (!buf || pad->ts_n_entries >= QTMUX_NO_OF_TS)) {
+ ts = pad->ts_entries[--pad->ts_n_entries];
+ buf = pad->buf_entries[pad->buf_head];
+ pad->buf_entries[pad->buf_head++] = NULL;
+ pad->buf_head %= wrap;
+ buf = gst_buffer_make_metadata_writable (buf);
+ GST_BUFFER_TIMESTAMP (buf) = ts;
+ GST_DEBUG_OBJECT (qtmux, "next buffer uses reordered ts %" GST_TIME_FORMAT,
+ GST_TIME_ARGS (ts));
+ } else {
+ buf = NULL;
+ }
+
+ return buf;
}
/*
@@ -1947,10 +2046,12 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
GstBuffer *last_buf = NULL;
GstClockTime duration;
guint nsamples, sample_size;
- guint64 scaled_duration, chunk_offset;
- gint64 last_dts;
+ guint64 chunk_offset;
+ gint64 last_dts, scaled_duration;
gint64 pts_offset = 0;
gboolean sync = FALSE, do_pts = FALSE;
+ gboolean drain = (buf == NULL);
+ GstFlowReturn ret;
if (!pad->fourcc)
goto not_negotiated;
@@ -1960,6 +2061,15 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
buf = pad->prepare_buf_func (pad, buf, qtmux);
}
+again:
+ if (G_UNLIKELY (qtmux->dts_method == DTS_METHOD_REORDER)) {
+ buf = gst_qt_mux_get_asc_buffer_ts (qtmux, pad, buf);
+ if (!buf) {
+ GST_DEBUG_OBJECT (qtmux, "no reordered buffer yet");
+ return GST_FLOW_OK;
+ }
+ }
+
last_buf = pad->last_buf;
if (last_buf == NULL) {
#ifndef GST_DISABLE_GST_DEBUG
@@ -1979,16 +2089,29 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
gst_buffer_ref (last_buf);
/* nasty heuristic mess to guestimate dealing with DTS/PTS,
- * while also trying to stay close to input ts to preserve sync, so:
+ * while also trying to stay close to input ts to preserve sync,
+ * so in DTS_METHOD_DD:
* - prefer using input ts where possible
- * - if those detected out-of-order (*), and input duration available,
- * mark as out-of-order and fallback to duration
- * - if in out-of-order, need to preserve sync between streams, and adding
- * durations might drift, so try to resync when we expect
- * input ts == (sum of durations), which is at some keyframe input frame.
+ * - if those detected out-of-order (*), mark as out-of-order
+ * - if in out-of-order, then
+ * - if duration available, use that as delta
+ * Also mind to preserve sync between streams, and adding
+ * durations might drift, so try to resync when we expect
+ * input ts == (sum of durations), which is at some keyframe input frame.
+ * - if no duration available, we are actually in serious trouble and need
+ * to hack around that, so we fail.
+ * To remedy failure, alternatively, in DTS_METHOD_REORDER:
+ * - collect some buffers and re-order timestamp,
+ * then process the oldest buffer with smallest timestamps.
+ * This should typically compensate for some codec's handywork with ts.
+ * ... but in case this makes ts end up where not expected:
+ * - keep each ts with its buffer and still keep a list of most recent X ts,
+ * use the (ascending) minimum of those as DTS (and the difference as ts delta),
+ * and use this DTS as a basis to obtain a (positive) CTS offset.
+ * This should yield exact PTS == buffer ts, but it seems not all players
+ * out there are aware of ctts pts ...
*
- * (*) if input ts out-of-order, or if ts differs from (sum of durations)
- * by an (approx) order-of-duration magnitude
+ * 0.11 Phew, can we (pretty) please please sort out DTS/PTS on buffers ...
*/
if (G_LIKELY (buf) && !pad->is_out_of_order) {
if (G_LIKELY (GST_BUFFER_TIMESTAMP_IS_VALID (last_buf) &&
@@ -2005,9 +2128,16 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
}
}
+ /* would have to be some unusual input, but not impossible */
+ if (G_UNLIKELY (qtmux->dts_method == DTS_METHOD_REORDER &&
+ pad->is_out_of_order)) {
+ goto no_order;
+ }
+
/* fall back to duration if last buffer or
* out-of-order (determined previously), otherwise use input ts */
- if (buf == NULL || pad->is_out_of_order) {
+ if (buf == NULL ||
+ (pad->is_out_of_order && qtmux->dts_method == DTS_METHOD_DD)) {
if (!GST_BUFFER_DURATION_IS_VALID (last_buf)) {
/* be forgiving for some possibly last upstream flushed buffer */
if (buf)
@@ -2020,9 +2150,7 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
/* avoid drift in sum timestamps,
* so use input timestamp for suitable keyframe */
if (buf && !GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT) &&
- GST_BUFFER_TIMESTAMP (buf) >= pad->last_dts &&
- !gst_qtmux_check_difference (qtmux, pad->last_dts + duration,
- GST_BUFFER_TIMESTAMP (buf), duration)) {
+ GST_BUFFER_TIMESTAMP (buf) >= pad->last_dts) {
GST_DEBUG_OBJECT (qtmux, "resyncing out-of-order input to ts; "
"replacing %" GST_TIME_FORMAT " by %" GST_TIME_FORMAT,
GST_TIME_ARGS (pad->last_dts + duration),
@@ -2030,8 +2158,37 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
duration = GST_BUFFER_TIMESTAMP (buf) - pad->last_dts;
}
}
- } else {
+ } else if (qtmux->dts_method != DTS_METHOD_ASC) {
duration = GST_BUFFER_TIMESTAMP (buf) - GST_BUFFER_TIMESTAMP (last_buf);
+ } else {
+ GstClockTime ts;
+
+ g_assert (qtmux->dts_method == DTS_METHOD_ASC);
+ if (!qtmux->guess_pts)
+ goto need_pts;
+
+ /* add timestamp to queue; keeps in descending order */
+ gst_qt_mux_push_ts (qtmux, pad, GST_BUFFER_TIMESTAMP (last_buf));
+ /* chuck out smallest/last one if we have enough */
+ if (G_LIKELY (pad->ts_n_entries > QTMUX_NO_OF_TS))
+ pad->ts_n_entries--;
+ /* peek the now smallest timestamp */
+ ts = pad->ts_entries[pad->ts_n_entries - 1];
+ /* these tails are expected to be (strictly) ascending with
+ * large enough history */
+ GST_DEBUG_OBJECT (qtmux, "ASC method; base timestamp %" GST_TIME_FORMAT,
+ GST_TIME_ARGS (ts));
+ if (ts >= pad->last_dts) {
+ duration = ts - pad->last_dts;
+ } else {
+ /* fallback to previous value, negative ct offset might handle */
+ GST_WARNING_OBJECT (qtmux, "unexpected decrease in timestamp");
+ duration = 0;
+ }
+ /* arrange for small non-zero duration/delta << expected frame time */
+ ts = gst_util_uint64_scale (10, GST_SECOND,
+ atom_trak_get_timescale (pad->trak));
+ duration = MAX (duration, ts);
}
gst_buffer_replace (&pad->last_buf, buf);
@@ -2166,15 +2323,23 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
if (qtmux->fragment_sequence) {
/* ensure that always sync samples are marked as such */
- return gst_qt_mux_pad_fragment_add_buffer (qtmux, pad, last_buf,
- buf == NULL, nsamples, last_dts, scaled_duration, sample_size,
+ ret = gst_qt_mux_pad_fragment_add_buffer (qtmux, pad, last_buf,
+ buf == NULL, nsamples, last_dts, (gint32) scaled_duration, sample_size,
!pad->sync || sync, pts_offset);
} else {
- atom_trak_add_samples (pad->trak, nsamples, scaled_duration,
+ atom_trak_add_samples (pad->trak, nsamples, (gint32) scaled_duration,
sample_size, chunk_offset, sync, pts_offset);
- return gst_qt_mux_send_buffer (qtmux, last_buf, &qtmux->mdat_size, TRUE);
+ ret = gst_qt_mux_send_buffer (qtmux, last_buf, &qtmux->mdat_size, TRUE);
+ }
+
+ if (G_UNLIKELY (drain && qtmux->dts_method == DTS_METHOD_REORDER &&
+ ret == GST_FLOW_OK)) {
+ buf = NULL;
+ goto again;
}
+ return ret;
+
/* ERRORS */
bail:
{
@@ -2189,6 +2354,18 @@ no_time:
("Received buffer without timestamp/duration."));
goto bail;
}
+no_order:
+ {
+ GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL),
+ ("DTS method failed to re-order timestamps."));
+ goto bail;
+ }
+need_pts:
+ {
+ GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL),
+ ("Selected DTS method also needs PTS enabled."));
+ goto bail;
+ }
fragmented_sample:
{
GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL),
@@ -3087,6 +3264,9 @@ gst_qt_mux_get_property (GObject * object,
case PROP_DO_CTTS:
g_value_set_boolean (value, qtmux->guess_pts);
break;
+ case PROP_DTS_METHOD:
+ g_value_set_enum (value, qtmux->dts_method);
+ break;
case PROP_FAST_START:
g_value_set_boolean (value, qtmux->fast_start);
break;
@@ -3139,6 +3319,9 @@ gst_qt_mux_set_property (GObject * object,
case PROP_DO_CTTS:
qtmux->guess_pts = g_value_get_boolean (value);
break;
+ case PROP_DTS_METHOD:
+ qtmux->dts_method = g_value_get_enum (value);
+ break;
case PROP_FAST_START:
qtmux->fast_start = g_value_get_boolean (value);
break;
diff --git a/gst/qtmux/gstqtmux.h b/gst/qtmux/gstqtmux.h
index 4df0c3e2b..3a2cb492f 100644
--- a/gst/qtmux/gstqtmux.h
+++ b/gst/qtmux/gstqtmux.h
@@ -78,6 +78,8 @@ typedef struct _GstQTPad GstQTPad;
typedef GstBuffer * (*GstQTPadPrepareBufferFunc) (GstQTPad * pad,
GstBuffer * buf, GstQTMux * qtmux);
+#define QTMUX_NO_OF_TS 10
+
struct _GstQTPad
{
GstCollectData collect; /* we extend the CollectData */
@@ -102,6 +104,11 @@ struct _GstQTPad
/* store the first timestamp for comparing with other streams and
* know if there are late streams */
GstClockTime first_ts;
+ GstClockTime ts_entries[QTMUX_NO_OF_TS + 2];
+ guint ts_n_entries;
+ GstBuffer *buf_entries[QTMUX_NO_OF_TS + 2];
+ guint buf_head;
+ guint buf_tail;
/* all the atom and chunk book-keeping is delegated here
* unowned/uncounted reference, parent MOOV owns */
@@ -174,6 +181,7 @@ struct _GstQTMux
AtomsTreeFlavor flavor;
gboolean fast_start;
gboolean guess_pts;
+ gint dts_method;
gchar *fast_start_file_path;
gchar *moov_recov_file_path;
guint32 fragment_duration;