isomp4: qtmux: Add Closed Caption support

Supports CEA 608 and CEA 708 CC streams (as byte pairs)

https://bugzilla.gnome.org/show_bug.cgi?id=606643

6 files changed, 190 insertions, 10 deletions

diff --git a/gst/isomp4/atoms.c b/gst/isomp4/atoms.c
index aaf34885a..fa40947c2 100644
--- a/gst/isomp4/atoms.c
+++ b/gst/isomp4/atoms.c
@@ -692,6 +692,7 @@ atom_stsd_remove_entries (AtomSTSD * stsd)
       case TIMECODE:
         sample_entry_tmcd_free ((SampleTableEntryTMCD *) se);
         break;
+      case CLOSEDCAPTION:
       default:
         /* best possible cleanup */
         atom_sample_entry_free (se);
@@ -2229,6 +2230,20 @@ sample_entry_tmcd_copy_data (SampleTableEntryTMCD * tmcd, guint8 ** buffer,
   return *offset - original_offset;
 }
 
+static guint64
+sample_entry_generic_copy_data (SampleTableEntry * entry, guint8 ** buffer,
+    guint64 * size, guint64 * offset)
+{
+  guint64 original_offset = *offset;
+
+  if (!atom_sample_entry_copy_data (entry, buffer, size, offset)) {
+    return 0;
+  }
+
+  atom_write_size (buffer, size, offset, original_offset);
+  return *offset - original_offset;
+}
+
 guint64
 atom_stsz_copy_data (AtomSTSZ * stsz, guint8 ** buffer, guint64 * size,
     guint64 * offset)
@@ -2462,6 +2477,11 @@ atom_stsd_copy_data (AtomSTSD * stsd, guint8 ** buffer, guint64 * size,
                   walker->data, buffer, size, offset)) {
             return 0;
           }
+        } else if (se->kind == CLOSEDCAPTION) {
+          if (!sample_entry_generic_copy_data ((SampleTableEntry *)
+                  walker->data, buffer, size, offset)) {
+            return 0;
+          }
         } else {
           if (!atom_hint_sample_entry_copy_data (
                   (AtomHintSampleEntry *) walker->data, buffer, size, offset)) {
@@ -4082,6 +4102,45 @@ atom_trak_set_timecode_type (AtomTRAK * trak, AtomsContext * context,
   return ste;
 }
 
+SampleTableEntry *
+atom_trak_set_caption_type (AtomTRAK * trak, AtomsContext * context,
+    guint32 caption_type)
+{
+  SampleTableEntry *ste;
+  AtomGMHD *gmhd = trak->mdia.minf.gmhd;
+  AtomSTSD *stsd = &trak->mdia.minf.stbl.stsd;
+
+  if (context->flavor != ATOMS_TREE_FLAVOR_MOV) {
+    return NULL;
+  }
+
+  /* FIXME : Set the timescale to the same timescale as the video trak ! */
+  trak->mdia.mdhd.time_info.timescale = 30000;
+  trak->mdia.hdlr.component_type = FOURCC_mhlr;
+  trak->mdia.hdlr.handler_type = FOURCC_clcp;
+  g_free (trak->mdia.hdlr.name);
+  trak->mdia.hdlr.name = g_strdup ("Closed Caption Media Handler");
+
+  ste = g_new0 (SampleTableEntry, 1);
+  atom_sample_entry_init (ste, caption_type);
+  ste->kind = CLOSEDCAPTION;
+  ste->data_reference_index = 1;
+  stsd->entries = g_list_prepend (stsd->entries, ste);
+  stsd->n_entries++;
+
+  gmhd = atom_gmhd_new ();
+  gmhd->gmin.graphics_mode = 0x0040;
+  gmhd->gmin.opcolor[0] = 0x8000;
+  gmhd->gmin.opcolor[1] = 0x8000;
+  gmhd->gmin.opcolor[2] = 0x8000;
+
+  trak->mdia.minf.gmhd = gmhd;
+  trak->is_video = FALSE;
+  trak->is_h264 = FALSE;
+
+  return ste;
+}
+
 static AtomInfo *
 build_pasp_extension (gint par_width, gint par_height)
 {
diff --git a/gst/isomp4/atoms.h b/gst/isomp4/atoms.h
index 963c4b99a..eb54a69d9 100644
--- a/gst/isomp4/atoms.h
+++ b/gst/isomp4/atoms.h
@@ -386,7 +386,8 @@ typedef enum _SampleEntryKind
   AUDIO,
   VIDEO,
   SUBTITLE,
-  TIMECODE
+  TIMECODE,
+  CLOSEDCAPTION
 } SampleEntryKind;
 
 typedef struct _SampleTableEntry
@@ -999,7 +1000,6 @@ guint64    atom_mfra_copy_data         (AtomMFRA *mfra, guint8 **buffer, guint64
 
 
 /* media sample description related helpers */
-
 typedef struct
 {
   guint16 version;
@@ -1056,6 +1056,8 @@ SampleTableEntryTX3G * atom_trak_set_subtitle_type (AtomTRAK * trak, AtomsContex
 SampleTableEntryTMCD *
 atom_trak_set_timecode_type (AtomTRAK * trak, AtomsContext * context, guint trak_timescale, GstVideoTimeCode * tc);
 
+SampleTableEntry * atom_trak_set_caption_type (AtomTRAK *trak, AtomsContext *context, guint32 caption_type);
+
 void atom_trak_update_bitrates (AtomTRAK * trak, guint32 avg_bitrate,
                                 guint32 max_bitrate);
 
diff --git a/gst/isomp4/gstqtmux.c b/gst/isomp4/gstqtmux.c
index 0dee96e88..8808203ac 100644
--- a/gst/isomp4/gstqtmux.c
+++ b/gst/isomp4/gstqtmux.c
@@ -449,7 +449,8 @@ gst_qt_mux_base_init (gpointer g_class)
   GstElementClass *element_class = GST_ELEMENT_CLASS (g_class);
   GstQTMuxClass *klass = (GstQTMuxClass *) g_class;
   GstQTMuxClassParams *params;
-  GstPadTemplate *videosinktempl, *audiosinktempl, *subtitlesinktempl;
+  GstPadTemplate *videosinktempl, *audiosinktempl, *subtitlesinktempl,
+      *captionsinktempl;
   GstPadTemplate *srctempl;
   gchar *longname, *description;
 
@@ -494,6 +495,13 @@ gst_qt_mux_base_init (gpointer g_class)
     gst_element_class_add_pad_template (element_class, subtitlesinktempl);
   }
 
+  if (params->caption_sink_caps) {
+    captionsinktempl = gst_pad_template_new_with_gtype ("caption_%u",
+        GST_PAD_SINK, GST_PAD_REQUEST, params->caption_sink_caps,
+        GST_TYPE_QT_MUX_PAD);
+    gst_element_class_add_pad_template (element_class, captionsinktempl);
+  }
+
   klass->format = params->prop->format;
 }
 
@@ -855,6 +863,40 @@ gst_qt_mux_prepare_jpc_buffer (GstQTPad * qtpad, GstBuffer * buf,
 }
 
 static GstBuffer *
+gst_qt_mux_prepare_cdat_buffer (GstQTPad * qtpad, GstBuffer * buf,
+    GstQTMux * qtmux)
+{
+  GstBuffer *newbuf;
+  GstMapInfo map, inmap;
+  gsize size;
+
+  GST_LOG_OBJECT (qtmux, "Preparing cdat buffer");
+
+  if (buf == NULL)
+    return NULL;
+
+  size = gst_buffer_get_size (buf);
+  GST_LOG_OBJECT (qtmux, "Preparing buffer of size %" G_GSIZE_FORMAT, size + 8);
+  newbuf = gst_buffer_new_and_alloc (size + 8);
+
+  /* Let's copy over all metadata and not the memory */
+  gst_buffer_copy_into (newbuf, buf, GST_BUFFER_COPY_METADATA, 0, size);
+
+  gst_buffer_map (newbuf, &map, GST_MAP_WRITE);
+  gst_buffer_map (buf, &inmap, GST_MAP_READ);
+
+  GST_WRITE_UINT32_BE (map.data, map.size);
+  GST_WRITE_UINT32_LE (map.data + 4, FOURCC_cdat);
+  memcpy (map.data + 8, inmap.data, inmap.size);
+
+  gst_buffer_unmap (newbuf, &map);
+  gst_buffer_unmap (buf, &inmap);
+  gst_buffer_unref (buf);
+
+  return newbuf;
+}
+
+static GstBuffer *
 gst_qt_mux_prepare_tx3g_buffer (GstQTPad * qtpad, GstBuffer * buf,
     GstQTMux * qtmux)
 {
@@ -4486,8 +4528,10 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
           gst_qt_mux_register_and_push_sample (qtmux, pad, empty_buf, FALSE, 1,
           last_dts + scaled_duration, empty_duration_scaled,
           empty_size, chunk_offset, sync, TRUE, 0);
-    } else {
-      /* our only case currently is tx3g subtitles, so there is no reason to fill this yet */
+    } else if (pad->fourcc != FOURCC_c608 && pad->fourcc != FOURCC_c708) {
+      /* This assert is kept here to make sure implementors of new
+       * sparse input format decide whether there needs to be special
+       * gap handling or not */
       g_assert_not_reached ();
       GST_WARNING_OBJECT (qtmux,
           "no empty buffer creation function found for pad %s",
@@ -5754,6 +5798,55 @@ refuse_caps:
 }
 
 static gboolean
+gst_qt_mux_caption_sink_set_caps (GstQTPad * qtpad, GstCaps * caps)
+{
+  GstPad *pad = qtpad->collect.pad;
+  GstQTMux *qtmux = GST_QT_MUX_CAST (gst_pad_get_parent (pad));
+  GstStructure *structure;
+  guint32 fourcc_entry;
+
+  if (qtpad->fourcc)
+    return gst_qt_mux_can_renegotiate (qtmux, pad, caps);
+
+  GST_DEBUG_OBJECT (qtmux, "%s:%s, caps=%" GST_PTR_FORMAT,
+      GST_DEBUG_PAD_NAME (pad), caps);
+
+  /* captions default */
+  qtpad->is_out_of_order = FALSE;
+  qtpad->sync = FALSE;
+  qtpad->sparse = TRUE;
+  /* Closed caption data is within a cdat atom */
+  qtpad->prepare_buf_func = gst_qt_mux_prepare_cdat_buffer;
+
+  structure = gst_caps_get_structure (caps, 0);
+
+
+  if (gst_structure_has_name (structure, "closedcaption/x-cea-608")) {
+    fourcc_entry = FOURCC_c608;
+  } else if (gst_structure_has_name (structure, "closedcaption/x-cea-708")) {
+    fourcc_entry = FOURCC_c708;
+  } else
+    goto refuse_caps;
+
+  qtpad->fourcc = fourcc_entry;
+  qtpad->trak_ste =
+      (SampleTableEntry *) atom_trak_set_caption_type (qtpad->trak,
+      qtmux->context, fourcc_entry);
+
+  gst_object_unref (qtmux);
+  return TRUE;
+
+  /* ERRORS */
+refuse_caps:
+  {
+    GST_WARNING_OBJECT (qtmux, "pad %s refused caps %" GST_PTR_FORMAT,
+        GST_PAD_NAME (pad), caps);
+    gst_object_unref (qtmux);
+    return FALSE;
+  }
+}
+
+static gboolean
 gst_qt_mux_sink_event (GstCollectPads * pads, GstCollectData * data,
     GstEvent * event, gpointer user_data)
 {
@@ -5924,6 +6017,14 @@ gst_qt_mux_request_new_pad (GstElement * element,
       name = g_strdup_printf ("subtitle_%u", qtmux->subtitle_pads++);
     }
     lock = FALSE;
+  } else if (templ == gst_element_class_get_pad_template (klass, "caption_%u")) {
+    setcaps_func = gst_qt_mux_caption_sink_set_caps;
+    if (req_name != NULL && sscanf (req_name, "caption_%u", &pad_id) == 1) {
+      name = g_strdup (req_name);
+    } else {
+      name = g_strdup_printf ("caption_%u", qtmux->caption_pads++);
+    }
+    lock = FALSE;
   } else
     goto wrong_template;
 
@@ -6218,7 +6319,7 @@ gst_qt_mux_register (GstPlugin * plugin)
 
   while (TRUE) {
     GstQTMuxFormatProp *prop;
-    GstCaps *subtitle_caps;
+    GstCaps *subtitle_caps, *caption_caps;
 
     prop = &gst_qt_mux_format_list[i];
     format = prop->format;
@@ -6237,6 +6338,12 @@ gst_qt_mux_register (GstPlugin * plugin)
     } else {
       gst_caps_unref (subtitle_caps);
     }
+    caption_caps = gst_static_caps_get (&prop->caption_sink_caps);
+    if (!gst_caps_is_equal (caption_caps, GST_CAPS_NONE)) {
+      params->caption_sink_caps = caption_caps;
+    } else {
+      gst_caps_unref (caption_caps);
+    }
 
     /* create the type now */
     type = g_type_register_static (GST_TYPE_ELEMENT, prop->type_name, &typeinfo,
diff --git a/gst/isomp4/gstqtmux.h b/gst/isomp4/gstqtmux.h
index c9ef21ddf..d8cb1dbfa 100644
--- a/gst/isomp4/gstqtmux.h
+++ b/gst/isomp4/gstqtmux.h
@@ -290,7 +290,7 @@ struct _GstQTMux
   gboolean reserved_prefill;
 
   /* for request pad naming */
-  guint video_pads, audio_pads, subtitle_pads;
+  guint video_pads, audio_pads, subtitle_pads, caption_pads;
 };
 
 struct _GstQTMuxClass
@@ -308,6 +308,7 @@ typedef struct _GstQTMuxClassParams
   GstCaps *video_sink_caps;
   GstCaps *audio_sink_caps;
   GstCaps *subtitle_sink_caps;
+  GstCaps *caption_sink_caps;
 } GstQTMuxClassParams;
 
 #define GST_QT_MUX_PARAMS_QDATA g_quark_from_static_string("qt-mux-params")
diff --git a/gst/isomp4/gstqtmuxmap.c b/gst/isomp4/gstqtmuxmap.c
index 4e34e137f..5a19ea9cf 100644
--- a/gst/isomp4/gstqtmuxmap.c
+++ b/gst/isomp4/gstqtmuxmap.c
@@ -166,6 +166,11 @@
   "text/x-raw, " \
   "format=(string)utf8"
 
+#define CEA608_CAPS \
+  "closedcaption/x-cea-608"
+#define CEA708_CAPS \
+  "closedcaption/x-cea-708"
+
 /* FIXME 0.11 - take a look at bugs #580005 and #340375 */
 GstQTMuxFormatProp gst_qt_mux_format_list[] = {
   /* original QuickTime format; see Apple site (e.g. qtff.pdf) */
@@ -209,7 +214,8 @@ GstQTMuxFormatProp gst_qt_mux_format_list[] = {
             "audio/x-alaw, " COMMON_AUDIO_CAPS (2, MAX) "; "
             "audio/x-mulaw, " COMMON_AUDIO_CAPS (2, MAX) "; "
             AMR_CAPS " ; " ALAC_CAPS " ; " OPUS_CAPS),
-      GST_STATIC_CAPS (TEXT_UTF8)}
+        GST_STATIC_CAPS (TEXT_UTF8),
+      GST_STATIC_CAPS (CEA608_CAPS "; " CEA708_CAPS)}
   ,
   /* ISO 14496-14: mp42 as ISO base media extension
    * (supersedes original ISO 144996-1 mp41) */
@@ -224,7 +230,8 @@ GstQTMuxFormatProp gst_qt_mux_format_list[] = {
             "video/x-mp4-part," COMMON_VIDEO_CAPS),
         GST_STATIC_CAPS (MP123_CAPS "; "
             AAC_CAPS " ; " AC3_CAPS " ; " ALAC_CAPS " ; " OPUS_CAPS),
-      GST_STATIC_CAPS (TEXT_UTF8)}
+        GST_STATIC_CAPS (TEXT_UTF8),
+      GST_STATIC_CAPS_NONE}
   ,
   /* Microsoft Smooth Streaming fmp4/isml */
   /* TODO add WMV/WMA support */
@@ -237,6 +244,7 @@ GstQTMuxFormatProp gst_qt_mux_format_list[] = {
         GST_STATIC_CAPS ("video/quicktime, variant = (string) iso-fragmented"),
         GST_STATIC_CAPS (MPEG4V_CAPS "; " H264_CAPS),
         GST_STATIC_CAPS (MP3_CAPS "; " AAC_CAPS),
+        GST_STATIC_CAPS_NONE,
       GST_STATIC_CAPS_NONE}
   ,
   /* 3GPP Technical Specification 26.244 V7.3.0
@@ -250,7 +258,8 @@ GstQTMuxFormatProp gst_qt_mux_format_list[] = {
         GST_STATIC_CAPS ("video/quicktime, variant = (string) 3gpp"),
         GST_STATIC_CAPS (H263_CAPS "; " MPEG4V_CAPS "; " H264_CAPS),
         GST_STATIC_CAPS (AMR_CAPS "; " MP3_CAPS "; " AAC_CAPS "; " AC3_CAPS),
-      GST_STATIC_CAPS (TEXT_UTF8)}
+        GST_STATIC_CAPS (TEXT_UTF8),
+      GST_STATIC_CAPS_NONE}
   ,
   /* ISO 15444-3: Motion-JPEG-2000 (also ISO base media extension) */
   {
@@ -263,6 +272,7 @@ GstQTMuxFormatProp gst_qt_mux_format_list[] = {
         GST_STATIC_CAPS ("image/x-j2c, " COMMON_VIDEO_CAPS "; "
             "image/x-jpc, " COMMON_VIDEO_CAPS),
         GST_STATIC_CAPS (PCM_CAPS),
+        GST_STATIC_CAPS_NONE,
       GST_STATIC_CAPS_NONE}
   ,
   {
diff --git a/gst/isomp4/gstqtmuxmap.h b/gst/isomp4/gstqtmuxmap.h
index e578a3674..f0bae16cb 100644
--- a/gst/isomp4/gstqtmuxmap.h
+++ b/gst/isomp4/gstqtmuxmap.h
@@ -70,6 +70,7 @@ typedef struct _GstQTMuxFormatProp
   GstStaticCaps video_sink_caps;
   GstStaticCaps audio_sink_caps;
   GstStaticCaps subtitle_sink_caps;
+  GstStaticCaps caption_sink_caps;
 } GstQTMuxFormatProp;
 
 extern GstQTMuxFormatProp gst_qt_mux_format_list[];