h264parse: Add support for stream-format=avc3

When outputting in AVC3 stream format, the codec_data should not
contain any SPS or PPS, because they are embedded inside the stream.

In case of avc->bytestream h264parse will push the SPS and PPS from
codec_data downstream at the start of the stream, at intervals
controlled by "config-interval" and when there is a codec_data change.

In the case of avc3->bytstream h264parse detects that there is
already SPS/PPS in the stream and sets h264parse->push_codec to FALSE.
Therefore avc3->bytstream was already supported, except for the stream
type.

In the case of bystream->avc h264parse will generate codec_data caps
from the parsed SPS/PPS in the stream. However it does not remove these
SPS/PPS from the stream. bytestream->avc3 is the same as bytestream->avc
except that the codec_data must not have any SPS/PPS in it.

|--------------+-------------+-------------------|
|stream-format | SPS in-band | SPS in codec_data |
|--------------+-------------+-------------------|
| avc          | maybe       | always            |
|--------------+-------------+-------------------|
| avc3         | always      | never             |
|--------------+-------------+-------------------|

Amendment 2 of ISO/IEC 14496-15 (AVC file format) is defining a new
structure for fragmented MP4 called "avc3". The principal difference
between AVC1 and AVC3 is the location of the codec initialisation
data (e.g. SPS, PPS). In AVC1 this data is placed in the initial MOOV box
(moov.trak.mdia.minf.stbl.stsd.avc1) but in AVC3 this data goes in the
first sample of every fragment.

https://bugzilla.gnome.org/show_bug.cgi?id=702004

2 files changed, 69 insertions, 14 deletions

diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c
index 48dcadd00..9f79a41d0 100644
--- a/gst/videoparsers/gsth264parse.c
+++ b/gst/videoparsers/gsth264parse.c
@@ -50,7 +50,8 @@ enum
 {
   GST_H264_PARSE_FORMAT_NONE,
   GST_H264_PARSE_FORMAT_AVC,
-  GST_H264_PARSE_FORMAT_BYTE
+  GST_H264_PARSE_FORMAT_BYTE,
+  GST_H264_PARSE_FORMAT_AVC3
 };
 
 enum
@@ -69,7 +70,7 @@ static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
     GST_PAD_ALWAYS,
     GST_STATIC_CAPS ("video/x-h264, parsed = (boolean) true, "
-        "stream-format=(string) { avc, byte-stream }, "
+        "stream-format=(string) { avc, avc3, byte-stream }, "
         "alignment=(string) { au, nal }"));
 
 #define parent_class gst_h264_parse_parent_class
@@ -261,6 +262,8 @@ gst_h264_parse_get_string (GstH264Parse * parse, gboolean format, gint code)
         return "avc";
       case GST_H264_PARSE_FORMAT_BYTE:
         return "byte-stream";
+      case GST_H264_PARSE_FORMAT_AVC3:
+        return "avc3";
       default:
         return "none";
     }
@@ -299,6 +302,8 @@ gst_h264_parse_format_from_caps (GstCaps * caps, guint * format, guint * align)
           *format = GST_H264_PARSE_FORMAT_AVC;
         else if (strcmp (str, "byte-stream") == 0)
           *format = GST_H264_PARSE_FORMAT_BYTE;
+        else if (strcmp (str, "avc3") == 0)
+          *format = GST_H264_PARSE_FORMAT_AVC3;
       }
     }
 
@@ -378,7 +383,8 @@ gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data,
   GST_DEBUG_OBJECT (h264parse, "nal length %d", size);
 
   buf = gst_buffer_new_allocate (NULL, 4 + size, NULL);
-  if (format == GST_H264_PARSE_FORMAT_AVC) {
+  if (format == GST_H264_PARSE_FORMAT_AVC
+      || format == GST_H264_PARSE_FORMAT_AVC3) {
     tmp = GUINT32_TO_BE (size << (32 - 8 * nl));
   } else {
     /* HACK: nl should always be 4 here, otherwise this won't work. 
@@ -994,7 +1000,9 @@ gst_h264_parse_make_codec_data (GstH264Parse * h264parse)
       }
     }
   }
-  for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
+  for (i = 0;
+      i < GST_H264_MAX_PPS_COUNT
+      && h264parse->format != GST_H264_PARSE_FORMAT_AVC3; i++) {
     if ((nal = h264parse->pps_nals[i])) {
       num_pps++;
       /* size bytes also count */
@@ -1002,10 +1010,15 @@ gst_h264_parse_make_codec_data (GstH264Parse * h264parse)
     }
   }
 
+  if (h264parse->format == GST_H264_PARSE_FORMAT_AVC3) {
+    num_sps = sps_size = 0;
+  }
+
   GST_DEBUG_OBJECT (h264parse,
       "constructing codec_data: num_sps=%d, num_pps=%d", num_sps, num_pps);
 
-  if (!found || !num_pps)
+  if (!found || (0 == num_pps
+          && GST_H264_PARSE_FORMAT_AVC3 != h264parse->format))
     return NULL;
 
   buf = gst_buffer_new_allocate (NULL, 5 + 1 + sps_size + 1 + pps_size, NULL);
@@ -1021,7 +1034,7 @@ gst_h264_parse_make_codec_data (GstH264Parse * h264parse)
   data[5] = 0xe0 | num_sps;     /* number of SPSs */
 
   data += 6;
-  for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
+  for (i = 0; i < num_sps; i++) {
     if ((nal = h264parse->sps_nals[i])) {
       gsize nal_size = gst_buffer_get_size (nal);
       GST_WRITE_UINT16_BE (data, nal_size);
@@ -1032,7 +1045,7 @@ gst_h264_parse_make_codec_data (GstH264Parse * h264parse)
 
   data[0] = num_pps;
   data++;
-  for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
+  for (i = 0; i < num_pps; i++) {
     if ((nal = h264parse->pps_nals[i])) {
       gsize nal_size = gst_buffer_get_size (nal);
       GST_WRITE_UINT16_BE (data, nal_size);
@@ -1170,8 +1183,9 @@ gst_h264_parse_update_src_caps (GstH264Parse * h264parse, GstCaps * caps)
   GST_DEBUG_OBJECT (h264parse, "sps: %p", sps);
 
   /* only codec-data for nice-and-clean au aligned packetized avc format */
-  if (h264parse->format == GST_H264_PARSE_FORMAT_AVC &&
-      h264parse->align == GST_H264_PARSE_ALIGN_AU) {
+  if ((h264parse->format == GST_H264_PARSE_FORMAT_AVC
+          || h264parse->format == GST_H264_PARSE_FORMAT_AVC3)
+      && h264parse->align == GST_H264_PARSE_ALIGN_AU) {
     buf = gst_h264_parse_make_codec_data (h264parse);
     if (buf && h264parse->codec_data) {
       GstMapInfo map;
@@ -1808,7 +1822,7 @@ gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps)
     size = map.size;
 
     /* parse the avcC data */
-    if (size < 8) {
+    if (size < 7) {             /* when numSPS==0 and numPPS==0, length is 7 bytes */
       gst_buffer_unmap (codec_data, &map);
       goto avcc_too_small;
     }
@@ -1907,7 +1921,8 @@ gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps)
       /* we did parse codec-data and might supplement src caps */
       gst_h264_parse_update_src_caps (h264parse, caps);
     }
-  } else if (format == GST_H264_PARSE_FORMAT_AVC) {
+  } else if (format == GST_H264_PARSE_FORMAT_AVC
+      || format == GST_H264_PARSE_FORMAT_AVC3) {
     /* if input != output, and input is avc, must split before anything else */
     /* arrange to insert codec-data in-stream if needed.
      * src caps are only arranged for later on */
diff --git a/tests/check/elements/h264parse.c b/tests/check/elements/h264parse.c
index 7539e63c2..3df62f030 100644
--- a/tests/check/elements/h264parse.c
+++ b/tests/check/elements/h264parse.c
@@ -43,6 +43,13 @@ GstStaticPadTemplate sinktemplate_avc_au = GST_STATIC_PAD_TEMPLATE ("sink",
         ", stream-format = (string) avc, alignment = (string) au")
     );
 
+GstStaticPadTemplate sinktemplate_avc3_au = GST_STATIC_PAD_TEMPLATE ("sink",
+    GST_PAD_SINK,
+    GST_PAD_ALWAYS,
+    GST_STATIC_CAPS (SINK_CAPS_TMPL
+        ", stream-format = (string) avc3, alignment = (string) au")
+    );
+
 GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
     GST_PAD_ALWAYS,
@@ -65,7 +72,7 @@ static guint8 h264_pps[] = {
 };
 
 /* combines to this codec-data */
-static guint8 h264_codec_data[] = {
+static guint8 h264_avc_codec_data[] = {
   0x01, 0x4d, 0x40, 0x15, 0xff, 0xe1, 0x00, 0x17,
   0x67, 0x4d, 0x40, 0x15, 0xec, 0xa4, 0xbf, 0x2e,
   0x02, 0x20, 0x00, 0x00, 0x03, 0x00, 0x2e, 0xe6,
@@ -73,6 +80,20 @@ static guint8 h264_codec_data[] = {
   0x00, 0x04, 0x68, 0xeb, 0xec, 0xb2
 };
 
+/* codec-data for avc3 where there are no SPS/PPS in the codec_data */
+static guint8 h264_avc3_codec_data[] = {
+  0x01,                         /* config version, always == 1 */
+  0x4d,                         /* profile */
+  0x40,                         /* profile compatibility */
+  0x15, 0xff,                   /* 6 reserved bits, lengthSizeMinusOne */
+  0xe0,                         /* 3 reserved bits, numSPS */
+  0x00                          /* numPPS */
+};
+
+static guint8 *h264_codec_data = NULL;
+static guint8 h264_codec_data_size = 0;
+
+
 /* keyframes all around */
 static guint8 h264_idrframe[] = {
   0x00, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x00,
@@ -214,7 +235,7 @@ GST_START_TEST (test_parse_detect_stream)
     fail_unless (val != NULL);
     buf = gst_value_get_buffer (val);
     fail_unless (buf != NULL);
-    fail_unless (gst_buffer_get_size (buf) == sizeof (h264_codec_data));
+    fail_unless (gst_buffer_get_size (buf) == h264_codec_data_size);
     fail_unless (gst_buffer_memcmp (buf, 0, h264_codec_data,
             gst_buffer_get_size (buf)) == 0);
   }
@@ -293,7 +314,7 @@ GST_START_TEST (test_parse_packetized)
   caps = gst_caps_from_string (SRC_CAPS_TMPL);
   cdata =
       gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, h264_codec_data,
-      sizeof (h264_codec_data), 0, sizeof (h264_codec_data), NULL, NULL);
+      h264_codec_data_size, 0, h264_codec_data_size, NULL, NULL);
   gst_caps_set_simple (caps, "codec_data", GST_TYPE_BUFFER, cdata, NULL);
   gst_buffer_unref (cdata);
   desc = gst_caps_to_string (caps);
@@ -359,6 +380,9 @@ main (int argc, char **argv)
   ctx_no_metadata = TRUE;
   ctx_codec_data = FALSE;
 
+  h264_codec_data = h264_avc_codec_data;
+  h264_codec_data_size = sizeof (h264_avc_codec_data);
+
   ctx_suite = "h264parse_to_bs_nal";
   s = h264parse_suite ();
   sr = srunner_create (s);
@@ -378,7 +402,23 @@ main (int argc, char **argv)
   nf += srunner_ntests_failed (sr);
   srunner_free (sr);
 
+  /* setup and tweak to handle avc3 au output */
+  h264_codec_data = h264_avc3_codec_data;
+  h264_codec_data_size = sizeof (h264_avc3_codec_data);
+  ctx_suite = "h264parse_to_avc3_au";
+  ctx_sink_template = &sinktemplate_avc3_au;
+  ctx_discard = 0;
+  ctx_codec_data = TRUE;
+
+  s = h264parse_suite ();
+  sr = srunner_create (s);
+  srunner_run_all (sr, CK_NORMAL);
+  nf += srunner_ntests_failed (sr);
+  srunner_free (sr);
+
   /* setup and tweak to handle avc packetized input */
+  h264_codec_data = h264_avc_codec_data;
+  h264_codec_data_size = sizeof (h264_avc_codec_data);
   ctx_suite = "h264parse_packetized";
   /* turn into separate byte stream NALs */
   ctx_sink_template = &sinktemplate_bs_nal;