rtph264depay: Make output in AVC stream format work even without complete sprop-parameter-set

This allows outputting streams in AVC format even if the SPS/PPS are sent inside
the RTP stream.

https://bugzilla.gnome.org/show_bug.cgi?id=654850

2 files changed, 303 insertions, 95 deletions

diff --git a/gst/rtp/gstrtph264depay.c b/gst/rtp/gstrtph264depay.c
index 295ea31bd..1a92dd205 100644
--- a/gst/rtp/gstrtph264depay.c
+++ b/gst/rtp/gstrtph264depay.c
@@ -24,6 +24,7 @@
 #include <stdio.h>
 #include <string.h>
 
+#include <gst/base/gstbitreader.h>
 #include <gst/rtp/gstrtpbuffer.h>
 #include "gstrtph264depay.h"
 
@@ -160,6 +161,10 @@ gst_rtp_h264_depay_init (GstRtpH264Depay * rtph264depay,
   rtph264depay->picture_adapter = gst_adapter_new ();
   rtph264depay->byte_stream = DEFAULT_BYTE_STREAM;
   rtph264depay->merge = DEFAULT_ACCESS_UNIT;
+  rtph264depay->sps = g_ptr_array_new_with_free_func (
+      (GDestroyNotify) gst_buffer_unref);
+  rtph264depay->pps = g_ptr_array_new_with_free_func (
+      (GDestroyNotify) gst_buffer_unref);
 }
 
 static void
@@ -172,6 +177,9 @@ gst_rtp_h264_depay_reset (GstRtpH264Depay * rtph264depay)
   rtph264depay->last_keyframe = FALSE;
   rtph264depay->last_ts = 0;
   rtph264depay->current_fu_type = 0;
+  rtph264depay->new_codec_data = FALSE;
+  g_ptr_array_set_size (rtph264depay->sps, 0);
+  g_ptr_array_set_size (rtph264depay->pps, 0);
 }
 
 static void
@@ -187,6 +195,9 @@ gst_rtp_h264_depay_finalize (GObject * object)
   g_object_unref (rtph264depay->adapter);
   g_object_unref (rtph264depay->picture_adapter);
 
+  g_ptr_array_free (rtph264depay->sps, TRUE);
+  g_ptr_array_free (rtph264depay->pps, TRUE);
+
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
 
@@ -291,17 +302,261 @@ gst_rtp_h264_depay_negotiate (GstRtpH264Depay * rtph264depay)
   }
 }
 
+/* Stolen from bad/gst/mpegtsdemux/payloader_parsers.c */
+/* variable length Exp-Golomb parsing according to H.264 spec 9.1*/
 static gboolean
-gst_rtp_h264_depay_setcaps (GstBaseRTPDepayload * depayload, GstCaps * caps)
+read_golomb (GstBitReader * br, guint32 * value)
+{
+  guint8 b, leading_zeros = -1;
+  *value = 1;
+
+  for (b = 0; !b; leading_zeros++) {
+    if (!gst_bit_reader_get_bits_uint8 (br, &b, 1))
+      return FALSE;
+    *value *= 2;
+  }
+
+  *value = (*value >> 1) - 1;
+  if (leading_zeros > 0) {
+    guint32 tmp = 0;
+    if (!gst_bit_reader_get_bits_uint32 (br, &tmp, leading_zeros))
+      return FALSE;
+    *value += tmp;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+parse_sps (GstBuffer * nal, guint32 * sps_id)
+{
+  GstBitReader br = GST_BIT_READER_INIT (GST_BUFFER_DATA (nal) + 4,
+      GST_BUFFER_SIZE (nal) - 4);
+
+  if (GST_BUFFER_SIZE (nal) < 5)
+    return FALSE;
+
+  if (!read_golomb (&br, sps_id))
+    return FALSE;
+
+  return TRUE;
+}
+
+static gboolean
+parse_pps (GstBuffer * nal, guint32 * sps_id, guint32 * pps_id)
+{
+  GstBitReader br = GST_BIT_READER_INIT (GST_BUFFER_DATA (nal) + 1,
+      GST_BUFFER_SIZE (nal) - 1);
+
+  if (GST_BUFFER_SIZE (nal) < 2)
+    return FALSE;
+
+  if (!read_golomb (&br, pps_id))
+    return FALSE;
+  if (!read_golomb (&br, sps_id))
+    return FALSE;
+
+  return TRUE;
+}
+
+static gboolean
+gst_rtp_h264_set_src_caps (GstRtpH264Depay * rtph264depay)
 {
+  gboolean res;
   GstCaps *srccaps;
+  guchar level = 0;
+  guchar profile_compat = G_MAXUINT8;
+
+  if (!rtph264depay->byte_stream &&
+      (!rtph264depay->new_codec_data ||
+          rtph264depay->sps->len == 0 || rtph264depay->pps->len == 0))
+    return TRUE;
+
+  srccaps = gst_caps_new_simple ("video/x-h264",
+      "stream-format", G_TYPE_STRING,
+      rtph264depay->byte_stream ? "byte-stream" : "avc",
+      "alignment", G_TYPE_STRING, rtph264depay->merge ? "au" : "nal", NULL);
+
+  if (!rtph264depay->byte_stream) {
+    GstBuffer *codec_data;
+    guchar *data;
+    guint len;
+    guint i;
+
+    /* start with 7 bytes header */
+    len = 7;
+    /* count sps & pps */
+    for (i = 0; i < rtph264depay->sps->len; i++)
+      len += 2 + GST_BUFFER_SIZE (g_ptr_array_index (rtph264depay->sps, i));
+    for (i = 0; i < rtph264depay->pps->len; i++)
+      len += 2 + GST_BUFFER_SIZE (g_ptr_array_index (rtph264depay->pps, i));
+
+    codec_data = gst_buffer_new_and_alloc (len);
+    g_debug ("alloc_len: %u", len);
+    data = GST_BUFFER_DATA (codec_data);
+
+    /* 8 bits version == 1 */
+    *data++ = 1;
+
+    /* According to: ISO/IEC 14496-15:2004(E) section 5.2.4.1
+     * The level is the max level of all SPSes
+     * A profile compat bit can only be set if all SPSes include that bit
+     */
+    for (i = 0; i < rtph264depay->sps->len; i++) {
+      guchar *nal = GST_BUFFER_DATA (g_ptr_array_index (rtph264depay->sps, i));
+
+      profile_compat &= nal[2];
+      level = MAX (level, nal[3]);
+    }
+
+    /* Assume all SPSes use the same profile, so extract from the first SPS */
+    *data++ = GST_BUFFER_DATA (g_ptr_array_index (rtph264depay->sps, 0))[1];
+    *data++ = profile_compat;
+    *data++ = level;
+
+    /* 6 bits reserved | 2 bits lengthSizeMinusOn */
+    *data++ = 0xff;
+    /* 3 bits reserved | 5 bits numOfSequenceParameterSets */
+    *data++ = 0xe0 | (rtph264depay->sps->len & 0x1f);
+
+    /* copy all SPS */
+    for (i = 0; i < rtph264depay->sps->len; i++) {
+      GstBuffer *sps = g_ptr_array_index (rtph264depay->sps, i);
+
+      GST_DEBUG_OBJECT (rtph264depay, "copy SPS %d of length %d", i,
+          GST_BUFFER_SIZE (sps));
+      GST_WRITE_UINT16_BE (data, GST_BUFFER_SIZE (sps));
+      data += 2;
+      memcpy (data, GST_BUFFER_DATA (sps), GST_BUFFER_SIZE (sps));
+      data += GST_BUFFER_SIZE (sps);
+    }
+
+    /* 8 bits numOfPictureParameterSets */
+    *data++ = rtph264depay->pps->len;
+    /* copy all PPS */
+    for (i = 0; i < rtph264depay->pps->len; i++) {
+      GstBuffer *pps = g_ptr_array_index (rtph264depay->pps, i);
+
+      GST_DEBUG_OBJECT (rtph264depay, "copy PPS %d of length %d", i,
+          GST_BUFFER_SIZE (pps));
+      GST_WRITE_UINT16_BE (data, GST_BUFFER_SIZE (pps));
+      data += 2;
+      memcpy (data, GST_BUFFER_DATA (pps), GST_BUFFER_SIZE (pps));
+      data += GST_BUFFER_SIZE (pps);
+    }
+
+    GST_BUFFER_SIZE (codec_data) = data - GST_BUFFER_DATA (codec_data);
+
+    gst_caps_set_simple (srccaps,
+        "codec_data", GST_TYPE_BUFFER, codec_data, NULL);
+    gst_buffer_unref (codec_data);
+  }
+
+  res = gst_pad_set_caps (GST_BASE_RTP_DEPAYLOAD_SRCPAD (rtph264depay),
+      srccaps);
+  gst_caps_unref (srccaps);
+
+  if (res)
+    rtph264depay->new_codec_data = FALSE;
+
+  return res;
+}
+
+static gboolean
+gst_rtp_h264_add_sps_pps (GstRtpH264Depay * rtph264depay, GstBuffer * nal)
+{
+  guchar type = GST_BUFFER_DATA (nal)[0] & 0x1f;
+  guint i;
+
+  if (type == 7) {
+    guint32 sps_id;
+
+    if (!parse_sps (nal, &sps_id)) {
+      GST_WARNING_OBJECT (rtph264depay, "Invalid SPS,"
+          " can't parse seq_parameter_set_id");
+      goto drop;
+    }
+
+    for (i = 0; i < rtph264depay->sps->len; i++) {
+      GstBuffer *sps = g_ptr_array_index (rtph264depay->sps, i);
+      guint32 tmp_sps_id;
+
+      parse_sps (sps, &tmp_sps_id);
+      if (sps_id == tmp_sps_id) {
+        if (GST_BUFFER_SIZE (nal) == GST_BUFFER_SIZE (sps) &&
+            memcmp (GST_BUFFER_DATA (nal), GST_BUFFER_DATA (sps),
+                GST_BUFFER_SIZE (sps)) == 0) {
+          GST_LOG_OBJECT (rtph264depay, "Unchanged SPS %u, not updating",
+              sps_id);
+          goto drop;
+        } else {
+          g_ptr_array_remove_index_fast (rtph264depay->sps, i);
+          g_ptr_array_add (rtph264depay->sps, nal);
+          GST_LOG_OBJECT (rtph264depay, "Modified SPS %u, replacing", sps_id);
+          goto done;
+        }
+      }
+    }
+    GST_LOG_OBJECT (rtph264depay, "Adding new SPS %u", sps_id);
+    g_ptr_array_add (rtph264depay->sps, nal);
+  } else if (type == 8) {
+    guint32 sps_id;
+    guint32 pps_id;
+
+    if (!parse_pps (nal, &sps_id, &pps_id)) {
+      GST_WARNING_OBJECT (rtph264depay, "Invalid PPS,"
+          " can't parse seq_parameter_set_id or pic_parameter_set_id");
+      goto drop;
+    }
+
+    for (i = 0; i < rtph264depay->pps->len; i++) {
+      GstBuffer *pps = g_ptr_array_index (rtph264depay->pps, i);
+      guint32 tmp_sps_id;
+      guint32 tmp_pps_id;
+
+      parse_pps (pps, &tmp_sps_id, &tmp_pps_id);
+      if (sps_id == tmp_sps_id && pps_id == tmp_pps_id) {
+        if (GST_BUFFER_SIZE (nal) == GST_BUFFER_SIZE (pps) &&
+            memcmp (GST_BUFFER_DATA (nal), GST_BUFFER_DATA (pps),
+                GST_BUFFER_SIZE (pps)) == 0) {
+          GST_LOG_OBJECT (rtph264depay, "Unchanged PPS %u:%u, not updating",
+              sps_id, pps_id);
+          goto drop;
+        } else {
+          g_ptr_array_remove_index_fast (rtph264depay->pps, i);
+          g_ptr_array_add (rtph264depay->pps, nal);
+          GST_LOG_OBJECT (rtph264depay, "Modified PPS %u:%u, replacing",
+              sps_id, pps_id);
+          goto done;
+        }
+      }
+    }
+    GST_LOG_OBJECT (rtph264depay, "Adding new PPS %u:%i", sps_id, pps_id);
+    g_ptr_array_add (rtph264depay->pps, nal);
+  } else {
+    goto drop;
+  }
+
+done:
+  rtph264depay->new_codec_data = TRUE;
+
+  return TRUE;
+
+drop:
+  gst_buffer_unref (nal);
+
+  return FALSE;
+}
+
+static gboolean
+gst_rtp_h264_depay_setcaps (GstBaseRTPDepayload * depayload, GstCaps * caps)
+{
   gint clock_rate;
   GstStructure *structure = gst_caps_get_structure (caps, 0);
   GstRtpH264Depay *rtph264depay;
-  const gchar *ps, *profile;
+  const gchar *ps;
   GstBuffer *codec_data;
   guint8 *b64;
-  gboolean res;
 
   rtph264depay = GST_RTP_H264_DEPAY (depayload);
 
@@ -309,12 +564,8 @@ gst_rtp_h264_depay_setcaps (GstBaseRTPDepayload * depayload, GstCaps * caps)
     clock_rate = 90000;
   depayload->clock_rate = clock_rate;
 
-  srccaps = gst_caps_new_simple ("video/x-h264", NULL);
-
   /* Base64 encoded, comma separated config NALs */
   ps = gst_structure_get_string (structure, "sprop-parameter-sets");
-  /* hex: AVCProfileIndication:8 | profile_compat:8 | AVCLevelIndication:8 */
-  profile = gst_structure_get_string (structure, "profile-level-id");
 
   /* negotiate with downstream w.r.t. output format and alignment */
   gst_rtp_h264_depay_negotiate (rtph264depay);
@@ -364,123 +615,51 @@ gst_rtp_h264_depay_setcaps (GstBaseRTPDepayload * depayload, GstCaps * caps)
     rtph264depay->codec_data = codec_data;
   } else if (!rtph264depay->byte_stream) {
     gchar **params;
-    guint8 **sps, **pps;
-    guint len, num_sps, num_pps;
     gint i;
-    guint8 *data;
 
     if (ps == NULL)
       goto incomplete_caps;
 
     params = g_strsplit (ps, ",", 0);
-    len = g_strv_length (params);
-
-    GST_DEBUG_OBJECT (depayload, "we have %d params", len);
 
-    sps = g_new0 (guint8 *, len + 1);
-    pps = g_new0 (guint8 *, len + 1);
-    num_sps = num_pps = 0;
+    GST_DEBUG_OBJECT (depayload, "we have %d params", g_strv_length (params));
 
     /* start with 7 bytes header */
-    len = 7;
     for (i = 0; params[i]; i++) {
+      GstBuffer *nal;
       gsize nal_len;
-      guint8 *nalp;
       guint save = 0;
       gint state = 0;
 
       nal_len = strlen (params[i]);
-      nalp = g_malloc (nal_len + 2);
+      nal = gst_buffer_new_and_alloc (nal_len);
 
       nal_len =
-          g_base64_decode_step (params[i], nal_len, nalp + 2, &state, &save);
-      nalp[0] = (nal_len >> 8) & 0xff;
-      nalp[1] = nal_len & 0xff;
-      len += nal_len + 2;
-
-      /* copy to the right list */
-      if ((nalp[2] & 0x1f) == 7) {
-        GST_DEBUG_OBJECT (depayload, "adding param %d as SPS %d", i, num_sps);
-        sps[num_sps++] = nalp;
-      } else {
-        GST_DEBUG_OBJECT (depayload, "adding param %d as PPS %d", i, num_pps);
-        pps[num_pps++] = nalp;
-      }
-    }
-    g_strfreev (params);
+          g_base64_decode_step (params[i], nal_len, GST_BUFFER_DATA (nal),
+          &state, &save);
 
-    if (num_sps == 0 || (GST_READ_UINT16_BE (sps[0]) < 3) || num_pps == 0) {
-      g_strfreev ((gchar **) pps);
-      g_strfreev ((gchar **) sps);
-      goto incomplete_caps;
-    }
+      GST_BUFFER_SIZE (nal) = nal_len;
 
-    codec_data = gst_buffer_new_and_alloc (len);
-    data = GST_BUFFER_DATA (codec_data);
+      GST_DEBUG_OBJECT (depayload, "adding param %d as %s", i,
+          ((GST_BUFFER_DATA (nal)[0] & 0x1f) == 7) ? "SPS" : "PPS");
 
-    /* 8 bits version == 1 */
-    *data++ = 1;
-    if (profile) {
-      guint32 profile_id;
-
-      /* hex: AVCProfileIndication:8 | profile_compat:8 | AVCLevelIndication:8 */
-      sscanf (profile, "%6x", &profile_id);
-      *data++ = (profile_id >> 16) & 0xff;
-      *data++ = (profile_id >> 8) & 0xff;
-      *data++ = profile_id & 0xff;
-    } else {
-      /* extract from SPS */
-      *data++ = sps[0][3];
-      *data++ = sps[0][4];
-      *data++ = sps[0][5];
+      gst_rtp_h264_add_sps_pps (rtph264depay, nal);
     }
-    /* 6 bits reserved | 2 bits lengthSizeMinusOn */
-    *data++ = 0xff;
-    /* 3 bits reserved | 5 bits numOfSequenceParameterSets */
-    *data++ = 0xe0 | (num_sps & 0x1f);
+    g_strfreev (params);
 
-    /* copy all SPS */
-    for (i = 0; sps[i]; i++) {
-      len = ((sps[i][0] << 8) | sps[i][1]) + 2;
-      GST_DEBUG_OBJECT (depayload, "copy SPS %d of length %d", i, len);
-      memcpy (data, sps[i], len);
-      g_free (sps[i]);
-      data += len;
-    }
-    g_free (sps);
-    /* 8 bits numOfPictureParameterSets */
-    *data++ = num_pps;
-    /* copy all PPS */
-    for (i = 0; pps[i]; i++) {
-      len = ((pps[i][0] << 8) | pps[i][1]) + 2;
-      GST_DEBUG_OBJECT (depayload, "copy PPS %d of length %d", i, len);
-      memcpy (data, pps[i], len);
-      g_free (pps[i]);
-      data += len;
-    }
-    g_free (pps);
-    GST_BUFFER_SIZE (codec_data) = data - GST_BUFFER_DATA (codec_data);
+    if (rtph264depay->sps->len == 0 || rtph264depay->pps->len == 0)
+      goto incomplete_caps;
 
-    gst_caps_set_simple (srccaps,
-        "codec_data", GST_TYPE_BUFFER, codec_data, NULL);
-    gst_buffer_unref (codec_data);
   }
 
-  gst_caps_set_simple (srccaps, "stream-format", G_TYPE_STRING,
-      rtph264depay->byte_stream ? "byte-stream" : "avc",
-      "alignment", G_TYPE_STRING, rtph264depay->merge ? "au" : "nal", NULL);
-
-  res = gst_pad_set_caps (depayload->srcpad, srccaps);
-  gst_caps_unref (srccaps);
-
-  return res;
+  return gst_rtp_h264_set_src_caps (rtph264depay);
 
   /* ERRORS */
 incomplete_caps:
   {
-    GST_DEBUG_OBJECT (depayload, "we have incomplete caps");
-    gst_caps_unref (srccaps);
-    return FALSE;
+    GST_DEBUG_OBJECT (depayload, "we have incomplete caps,"
+        " doing setcaps later");
+    return TRUE;
   }
 }
 
@@ -535,6 +714,30 @@ gst_rtp_h264_depay_handle_nal (GstRtpH264Depay * rtph264depay, GstBuffer * nal,
   out_keyframe = keyframe;
   out_timestamp = in_timestamp;
 
+  if (!rtph264depay->byte_stream) {
+    if (nal_type == 7 || nal_type == 8) {
+      gst_rtp_h264_add_sps_pps (rtph264depay, gst_buffer_create_sub (nal, 4,
+              GST_BUFFER_SIZE (nal) - 4));
+      gst_buffer_unref (nal);
+      return NULL;
+    } else if (rtph264depay->sps->len == 0 || rtph264depay->pps->len == 0) {
+      /* Down push down any buffer in non-bytestream mode if the SPS/PPS haven't
+       * go through yet
+       */
+      gst_pad_push_event (GST_BASE_RTP_DEPAYLOAD_SINKPAD (depayload),
+          gst_event_new_custom (GST_EVENT_CUSTOM_UPSTREAM,
+              gst_structure_new ("GstForceKeyUnit",
+                  "all-headers", G_TYPE_BOOLEAN, TRUE, NULL)));
+      gst_buffer_unref (nal);
+      return FALSE;
+    }
+
+    if (rtph264depay->new_codec_data &&
+        rtph264depay->sps->len > 0 && rtph264depay->pps->len > 0)
+      gst_rtp_h264_set_src_caps (rtph264depay);
+  }
+
+
   if (rtph264depay->merge) {
     gboolean start = FALSE, complete = FALSE;
 
diff --git a/gst/rtp/gstrtph264depay.h b/gst/rtp/gstrtph264depay.h
index f50ffe6a3..fd2603f76 100644
--- a/gst/rtp/gstrtph264depay.h
+++ b/gst/rtp/gstrtph264depay.h
@@ -61,6 +61,11 @@ struct _GstRtpH264Depay
   guint8 current_fu_type;
   GstClockTime fu_timestamp;
   gboolean fu_marker;
+
+  /* misc */
+  GPtrArray *sps;
+  GPtrArray *pps;
+  gboolean new_codec_data;
 };
 
 struct _GstRtpH264DepayClass