applemedia: port vtdec

7 files changed, 164 insertions, 87 deletions

diff --git a/sys/applemedia/Makefile.am b/sys/applemedia/Makefile.am
index 445712c11..f7b853dfe 100644
--- a/sys/applemedia/Makefile.am
+++ b/sys/applemedia/Makefile.am
@@ -78,6 +78,7 @@ else
 
 libgstapplemedia_la_SOURCES +=			\
 	qtkitvideosrc.m				\
+	vtdec.c 				\
 	mioapi.c
 
 libgstapplemedia_la_LDFLAGS +=			\
diff --git a/sys/applemedia/bufferfactory.m b/sys/applemedia/bufferfactory.m
index 27fce4de9..744f0c33a 100644
--- a/sys/applemedia/bufferfactory.m
+++ b/sys/applemedia/bufferfactory.m
@@ -50,7 +50,7 @@
 
 - (GstBuffer *)createGstBufferForCoreVideoBuffer:(CFTypeRef)cvbuf
 {
-  return gst_core_video_buffer_new (coreMediaCtx, (CVBufferRef) cvbuf);
+  return gst_core_video_buffer_new (coreMediaCtx, (CVBufferRef) cvbuf, NULL);
 }
 
 - (GstBuffer *)createGstBufferForSampleBuffer:(CFTypeRef)sbuf
diff --git a/sys/applemedia/corevideobuffer.c b/sys/applemedia/corevideobuffer.c
index ee1f29eea..eee0a8fc0 100644
--- a/sys/applemedia/corevideobuffer.c
+++ b/sys/applemedia/corevideobuffer.c
@@ -49,7 +49,8 @@ gst_core_video_meta_get_info (void)
 }
 
 GstBuffer *
-gst_core_video_buffer_new (GstCoreMediaCtx * ctx, CVBufferRef cvbuf)
+gst_core_video_buffer_new (GstCoreMediaCtx * ctx, CVBufferRef cvbuf,
+    GstVideoInfo * vinfo)
 {
   GstCVApi *cv = ctx->cv;
   void *data;
@@ -57,32 +58,65 @@ gst_core_video_buffer_new (GstCoreMediaCtx * ctx, CVBufferRef cvbuf)
   CVPixelBufferRef pixbuf = NULL;
   GstBuffer *buf;
   GstCoreVideoMeta *meta;
+  guint width, height, n_planes, i;
+  gsize offset[GST_VIDEO_MAX_PLANES];
+  gint stride[GST_VIDEO_MAX_PLANES];
 
-  if (CFGetTypeID (cvbuf) == cv->CVPixelBufferGetTypeID ()) {
-    pixbuf = (CVPixelBufferRef) cvbuf;
-
-    if (cv->CVPixelBufferLockBaseAddress (pixbuf,
-            kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) {
-      goto error;
-    }
-    data = cv->CVPixelBufferGetBaseAddress (pixbuf);
-    size = cv->CVPixelBufferGetBytesPerRow (pixbuf) *
-        cv->CVPixelBufferGetHeight (pixbuf);
-  } else {
+  if (CFGetTypeID (cvbuf) != cv->CVPixelBufferGetTypeID ())
     /* TODO: Do we need to handle other buffer types? */
     goto error;
+
+  pixbuf = (CVPixelBufferRef) cvbuf;
+
+  if (cv->CVPixelBufferLockBaseAddress (pixbuf,
+          kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) {
+    goto error;
   }
 
   buf = gst_buffer_new ();
+
+  /* add the corevideo meta to free the underlying corevideo buffer */
   meta = (GstCoreVideoMeta *) gst_buffer_add_meta (buf,
       gst_core_video_meta_get_info (), NULL);
   meta->ctx = g_object_ref (ctx);
   meta->cvbuf = cv->CVBufferRetain (cvbuf);
   meta->pixbuf = pixbuf;
+
+  /* set stride, offset and size */
+  memset (&offset, 0, sizeof (offset));
+  memset (&stride, 0, sizeof (stride));
+
+  data = cv->CVPixelBufferGetBaseAddress (pixbuf);
+  height = cv->CVPixelBufferGetHeight (pixbuf);
+  if (cv->CVPixelBufferIsPlanar (pixbuf)) {
+    GstVideoInfo tmp_vinfo;
+
+    n_planes = cv->CVPixelBufferGetPlaneCount (pixbuf);
+    for (i = 0; i < n_planes; ++i)
+      stride[i] = cv->CVPixelBufferGetBytesPerRowOfPlane (pixbuf, i);
+
+    /* FIXME: don't hardcode NV12 */
+    gst_video_info_set_format (&tmp_vinfo,
+        GST_VIDEO_FORMAT_NV12, stride[0], height);
+    offset[1] = tmp_vinfo.offset[1];
+    size = tmp_vinfo.size;
+  } else {
+    n_planes = 1;
+    size = cv->CVPixelBufferGetBytesPerRow (pixbuf) * height;
+  }
+
   gst_buffer_take_memory (buf, -1,
       gst_memory_new_wrapped (GST_MEMORY_FLAG_NO_SHARE, data,
           size, 0, size, NULL, NULL));
 
+  if (vinfo) {
+    GstVideoMeta *video_meta;
+
+    width = vinfo->width;
+    video_meta = gst_buffer_add_video_meta_full (buf, GST_VIDEO_FLAG_NONE,
+        GST_VIDEO_FORMAT_NV12, width, height, n_planes, offset, stride);
+  }
+
   return buf;
 
 error:
diff --git a/sys/applemedia/corevideobuffer.h b/sys/applemedia/corevideobuffer.h
index 7c6b12334..09040d28c 100644
--- a/sys/applemedia/corevideobuffer.h
+++ b/sys/applemedia/corevideobuffer.h
@@ -21,6 +21,8 @@
 #define __GST_CORE_VIDEO_BUFFER_H__
 
 #include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/video/gstvideometa.h>
 
 #include "coremediactx.h"
 
@@ -36,7 +38,8 @@ typedef struct _GstCoreVideoMeta
 } GstCoreVideoMeta;
 
 GstBuffer * gst_core_video_buffer_new      (GstCoreMediaCtx * ctx,
-                                            CVBufferRef cvbuf);
+                                            CVBufferRef cvbuf,
+                                            GstVideoInfo *info);
 
 G_END_DECLS
 
diff --git a/sys/applemedia/plugin.m b/sys/applemedia/plugin.m
index 199a5d70c..3d211aa74 100644
--- a/sys/applemedia/plugin.m
+++ b/sys/applemedia/plugin.m
@@ -73,8 +73,8 @@ plugin_init (GstPlugin * plugin)
       GST_TYPE_VT_H264_ENC_BIN);
 
   gst_vtenc_register_elements (plugin);
-  gst_vtdec_register_elements (plugin);
 #endif
+  gst_vtdec_register_elements (plugin);
 
   return res;
 }
diff --git a/sys/applemedia/vtdec.c b/sys/applemedia/vtdec.c
index 22db2e869..6ddf7fbb3 100644
--- a/sys/applemedia/vtdec.c
+++ b/sys/applemedia/vtdec.c
@@ -32,8 +32,9 @@ static GstElementClass *parent_class = NULL;
 
 static GstStateChangeReturn gst_vtdec_change_state (GstElement * element,
     GstStateChange transition);
-static gboolean gst_vtdec_sink_setcaps (GstPad * pad, GstCaps * caps);
-static GstFlowReturn gst_vtdec_chain (GstPad * pad, GstBuffer * buf);
+static gboolean gst_vtdec_sink_setcaps (GstVTDec * dec, GstCaps * caps);
+static GstFlowReturn gst_vtdec_chain (GstPad * pad, GstObject * parent,
+    GstBuffer * buf);
 
 static CMFormatDescriptionRef gst_vtdec_create_format_description
     (GstVTDec * self);
@@ -47,6 +48,8 @@ static void gst_vtdec_destroy_session (GstVTDec * self,
 static GstFlowReturn gst_vtdec_decode_buffer (GstVTDec * self, GstBuffer * buf);
 static void gst_vtdec_enqueue_frame (void *data, gsize unk1, VTStatus result,
     gsize unk2, CVBufferRef cvbuf);
+static gboolean gst_vtdec_sink_event (GstPad * pad, GstObject * parent,
+    GstEvent * event);
 
 static CMSampleBufferRef gst_vtdec_sample_buffer_from (GstVTDec * self,
     GstBuffer * buf);
@@ -63,19 +66,17 @@ gst_vtdec_base_init (GstVTDecClass * klass)
   const int min_fps_d = 1, max_fps_d = 1;
   GstPadTemplate *sink_template, *src_template;
   GstCaps *sink_caps;
-  GstElementDetails details;
+  gchar *longname, *description;
 
-  details.longname = g_strdup_printf ("%s decoder", codec_details->name);
-  details.klass = g_strdup_printf ("Codec/Decoder/Video");
-  details.description = g_strdup_printf ("%s decoder", codec_details->name);
+  longname = g_strdup_printf ("%s decoder", codec_details->name);
+  description = g_strdup_printf ("%s decoder", codec_details->name);
 
-  gst_element_class_set_details_simple (element_class,
-      details.longname, details.klass, details.description,
+  gst_element_class_set_metadata (element_class, longname,
+      "Codec/Decoder/Video", description,
       "Ole André Vadla Ravnås <oravnas@cisco.com>");
 
-  g_free (details.longname);
-  g_free (details.klass);
-  g_free (details.description);
+  g_free (longname);
+  g_free (description);
 
   sink_caps = gst_caps_new_simple (codec_details->mimetype,
       "width", GST_TYPE_INT_RANGE, min_width, max_width,
@@ -93,8 +94,8 @@ gst_vtdec_base_init (GstVTDecClass * klass)
   src_template = gst_pad_template_new ("src",
       GST_PAD_SRC,
       GST_PAD_ALWAYS,
-      gst_caps_new_simple ("video/x-raw-yuv",
-          "format", GST_TYPE_FOURCC, GST_MAKE_FOURCC ('N', 'V', '1', '2'),
+      gst_caps_new_simple ("video/x-raw",
+          "format", G_TYPE_STRING, "NV12",
           "width", GST_TYPE_INT_RANGE, min_width, max_width,
           "height", GST_TYPE_INT_RANGE, min_height, max_height,
           "framerate", GST_TYPE_FRACTION_RANGE,
@@ -126,7 +127,7 @@ gst_vtdec_init (GstVTDec * self)
   self->sinkpad = gst_pad_new_from_template
       (gst_element_class_get_pad_template (element_klass, "sink"), "sink");
   gst_element_add_pad (element, self->sinkpad);
-  gst_pad_set_setcaps_function (self->sinkpad, gst_vtdec_sink_setcaps);
+  gst_pad_set_event_function (self->sinkpad, gst_vtdec_sink_event);
   gst_pad_set_chain_function (self->sinkpad, gst_vtdec_chain);
 
   self->srcpad = gst_pad_new_from_template
@@ -158,10 +159,7 @@ gst_vtdec_change_state (GstElement * element, GstStateChange transition)
     self->ctx->cm->FigFormatDescriptionRelease (self->fmt_desc);
     self->fmt_desc = NULL;
 
-    self->negotiated_width = self->negotiated_height = 0;
-    self->negotiated_fps_n = self->negotiated_fps_d = 0;
-    self->caps_width = self->caps_height = 0;
-    self->caps_fps_n = self->caps_fps_d = 0;
+    gst_video_info_init (&self->vinfo);
 
     g_ptr_array_free (self->cur_outbufs, TRUE);
     self->cur_outbufs = NULL;
@@ -182,25 +180,69 @@ api_error:
 }
 
 static gboolean
-gst_vtdec_sink_setcaps (GstPad * pad, GstCaps * caps)
+gst_vtdec_sink_event (GstPad * pad, GstObject * parent, GstEvent * event)
+{
+  GstVTDec *self = GST_VTDEC_CAST (parent);
+  gboolean forward = TRUE;
+  gboolean res = TRUE;
+
+  switch (GST_EVENT_TYPE (event)) {
+    case GST_EVENT_CAPS:
+    {
+      GstCaps *caps;
+
+      gst_event_parse_caps (event, &caps);
+      res = gst_vtdec_sink_setcaps (self, caps);
+    }
+    default:
+      break;
+  }
+
+  if (forward)
+    res = gst_pad_event_default (pad, parent, event);
+  return res;
+}
+
+static gboolean
+gst_vtdec_sink_setcaps (GstVTDec * self, GstCaps * caps)
 {
-  GstVTDec *self = GST_VTDEC_CAST (GST_PAD_PARENT (pad));
   GstStructure *structure;
   CMFormatDescriptionRef fmt_desc = NULL;
+  GstVideoFormat format = GST_VIDEO_FORMAT_NV12;
+  gint width, height;
+  gint fps_n, fps_d;
+  gint par_n, par_d;
 
   structure = gst_caps_get_structure (caps, 0);
-  if (!gst_structure_get_int (structure, "width", &self->negotiated_width))
+  if (!gst_structure_get_int (structure, "width", &width))
     goto incomplete_caps;
-  if (!gst_structure_get_int (structure, "height", &self->negotiated_height))
+  if (!gst_structure_get_int (structure, "height", &height))
     goto incomplete_caps;
-  gst_structure_get_fraction (structure, "framerate",
-      &self->negotiated_fps_n, &self->negotiated_fps_d);
 
-  /* FIXME */
-  if (self->negotiated_fps_n == 0)
-    self->negotiated_fps_n = 30;
-  if (self->negotiated_fps_d == 0)
-    self->negotiated_fps_d = 1;
+  gst_video_info_set_format (&self->vinfo, format, width, height);
+
+  if (gst_structure_get_fraction (structure, "framerate", &fps_n, &fps_d)) {
+    if (fps_n == 0) {
+      /* variable framerate */
+      self->vinfo.flags |= GST_VIDEO_FLAG_VARIABLE_FPS;
+      /* see if we have a max-framerate */
+      gst_structure_get_fraction (structure, "max-framerate", &fps_n, &fps_d);
+    }
+    self->vinfo.fps_n = fps_n;
+    self->vinfo.fps_d = fps_d;
+  } else {
+    /* unspecified is variable framerate */
+    self->vinfo.fps_n = 0;
+    self->vinfo.fps_d = 1;
+  }
+  if (gst_structure_get_fraction (structure, "pixel-aspect-ratio",
+          &par_n, &par_d)) {
+    self->vinfo.par_n = par_n;
+    self->vinfo.par_d = par_d;
+  } else {
+    self->vinfo.par_n = 1;
+    self->vinfo.par_d = 1;
+  }
 
   if (self->details->format_id == kVTFormatH264) {
     const GValue *codec_data_value;
@@ -226,12 +268,14 @@ gst_vtdec_sink_setcaps (GstPad * pad, GstCaps * caps)
       goto session_create_error;
   }
 
+  /* renegotiate when upstream caps change */
+  gst_pad_mark_reconfigure (self->srcpad);
+
   return TRUE;
 
   /* ERRORS */
 incomplete_caps:
   {
-    self->negotiated_width = self->negotiated_height = -1;
     return TRUE;
   }
 session_create_error:
@@ -245,7 +289,7 @@ session_create_error:
 static gboolean
 gst_vtdec_is_negotiated (GstVTDec * self)
 {
-  return self->negotiated_width != 0;
+  return self->vinfo.width != 0;
 }
 
 static gboolean
@@ -253,42 +297,26 @@ gst_vtdec_negotiate_downstream (GstVTDec * self)
 {
   gboolean result;
   GstCaps *caps;
-  GstStructure *s;
 
-  if (self->caps_width == self->negotiated_width &&
-      self->caps_height == self->negotiated_height &&
-      self->caps_fps_n == self->negotiated_fps_n &&
-      self->caps_fps_d == self->negotiated_fps_d) {
+  if (!gst_pad_check_reconfigure (self->srcpad))
     return TRUE;
-  }
 
-  caps = gst_caps_copy (gst_pad_get_pad_template_caps (self->srcpad));
-  s = gst_caps_get_structure (caps, 0);
-  gst_structure_set (s,
-      "width", G_TYPE_INT, self->negotiated_width,
-      "height", G_TYPE_INT, self->negotiated_height,
-      "framerate", GST_TYPE_FRACTION,
-      self->negotiated_fps_n, self->negotiated_fps_d, NULL);
+  caps = gst_video_info_to_caps (&self->vinfo);
   result = gst_pad_set_caps (self->srcpad, caps);
   gst_caps_unref (caps);
 
-  self->caps_width = self->negotiated_width;
-  self->caps_height = self->negotiated_height;
-  self->caps_fps_n = self->negotiated_fps_n;
-  self->caps_fps_d = self->negotiated_fps_d;
-
   return result;
 }
 
 static GstFlowReturn
-gst_vtdec_chain (GstPad * pad, GstBuffer * buf)
+gst_vtdec_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
 {
-  GstVTDec *self = GST_VTDEC_CAST (GST_PAD_PARENT (pad));
+  GstVTDec *self = GST_VTDEC_CAST (parent);
 
   if (!gst_vtdec_is_negotiated (self))
     goto not_negotiated;
 
-  if (self->session == NULL || self->negotiated_width < 0)
+  if (self->session == NULL)
     goto pending_caps;
 
   return gst_vtdec_decode_buffer (self, buf);
@@ -311,7 +339,7 @@ gst_vtdec_create_format_description (GstVTDec * self)
   OSStatus status;
 
   status = self->ctx->cm->CMVideoFormatDescriptionCreate (NULL,
-      self->details->format_id, self->negotiated_width, self->negotiated_height,
+      self->details->format_id, self->vinfo.width, self->vinfo.height,
       NULL, &fmt_desc);
   if (status == noErr)
     return fmt_desc;
@@ -325,13 +353,18 @@ gst_vtdec_create_format_description_from_codec_data (GstVTDec * self,
 {
   CMFormatDescriptionRef fmt_desc;
   OSStatus status;
+  GstMapInfo map;
+
+  gst_buffer_map (codec_data, &map, GST_MAP_READ);
 
   status =
       self->ctx->cm->
       FigVideoFormatDescriptionCreateWithSampleDescriptionExtensionAtom (NULL,
-      self->details->format_id, self->negotiated_width, self->negotiated_height,
-      'avcC', GST_BUFFER_DATA (codec_data), GST_BUFFER_SIZE (codec_data),
-      &fmt_desc);
+      self->details->format_id, self->vinfo.width, self->vinfo.height,
+      'avcC', map.data, map.size, NULL, &fmt_desc);
+
+  gst_buffer_unmap (codec_data, &map);
+
   if (status == noErr)
     return fmt_desc;
   else
@@ -352,11 +385,11 @@ gst_vtdec_create_session (GstVTDec * self, CMFormatDescriptionRef fmt_desc)
   gst_vtutil_dict_set_i32 (pb_attrs, *(cv->kCVPixelBufferPixelFormatTypeKey),
       kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
   gst_vtutil_dict_set_i32 (pb_attrs, *(cv->kCVPixelBufferWidthKey),
-      self->negotiated_width);
+      self->vinfo.width);
   gst_vtutil_dict_set_i32 (pb_attrs, *(cv->kCVPixelBufferHeightKey),
-      self->negotiated_height);
+      self->vinfo.height);
   gst_vtutil_dict_set_i32 (pb_attrs,
-      *(cv->kCVPixelBufferBytesPerRowAlignmentKey), 2 * self->negotiated_width);
+      *(cv->kCVPixelBufferBytesPerRowAlignmentKey), 2 * self->vinfo.width);
 
   callback.func = gst_vtdec_enqueue_frame;
   callback.data = self;
@@ -364,7 +397,7 @@ gst_vtdec_create_session (GstVTDec * self, CMFormatDescriptionRef fmt_desc)
   status = self->ctx->vt->VTDecompressionSessionCreate (NULL, fmt_desc,
       NULL, pb_attrs, &callback, &session);
   GST_INFO_OBJECT (self, "VTDecompressionSessionCreate for %d x %d => %d",
-      self->negotiated_width, self->negotiated_height, status);
+      self->vinfo.width, self->vinfo.height, status);
 
   CFRelease (pb_attrs);
 
@@ -416,7 +449,6 @@ gst_vtdec_decode_buffer (GstVTDec * self, GstBuffer * buf)
     GstBuffer *buf = g_ptr_array_index (self->cur_outbufs, i);
 
     if (ret == GST_FLOW_OK) {
-      gst_buffer_set_caps (buf, GST_PAD_CAPS (self->srcpad));
       ret = gst_pad_push (self->srcpad, buf);
     } else {
       gst_buffer_unref (buf);
@@ -437,9 +469,8 @@ gst_vtdec_enqueue_frame (void *data, gsize unk1, VTStatus result, gsize unk2,
   if (result != kVTSuccess)
     goto beach;
 
-  buf = gst_core_video_buffer_new (self->ctx, cvbuf);
-  gst_buffer_copy_metadata (buf, self->cur_inbuf,
-      GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS);
+  buf = gst_core_video_buffer_new (self->ctx, cvbuf, &self->vinfo);
+  gst_buffer_copy_into (buf, self->cur_inbuf, GST_BUFFER_COPY_METADATA, 0, -1);
 
   g_ptr_array_add (self->cur_outbufs, buf);
 
@@ -454,23 +485,33 @@ gst_vtdec_sample_buffer_from (GstVTDec * self, GstBuffer * buf)
   OSStatus status;
   CMBlockBufferRef bbuf = NULL;
   CMSampleBufferRef sbuf = NULL;
+  GstMapInfo map;
 
   g_assert (self->fmt_desc != NULL);
 
+  gst_buffer_map (buf, &map, GST_MAP_READ);
+
   status = cm->CMBlockBufferCreateWithMemoryBlock (NULL,
-      GST_BUFFER_DATA (buf), GST_BUFFER_SIZE (buf), kCFAllocatorNull, NULL,
-      0, GST_BUFFER_SIZE (buf), FALSE, &bbuf);
+      map.data, (gint64) map.size, kCFAllocatorNull, NULL, 0, (gint64) map.size,
+      FALSE, &bbuf);
+
+  gst_buffer_unmap (buf, &map);
+
   if (status != noErr)
-    goto beach;
+    goto error;
 
   status = cm->CMSampleBufferCreate (NULL, bbuf, TRUE, 0, 0, self->fmt_desc,
       1, 0, NULL, 0, NULL, &sbuf);
   if (status != noErr)
-    goto beach;
+    goto error;
 
 beach:
   cm->FigBlockBufferRelease (bbuf);
   return sbuf;
+
+error:
+  GST_ERROR_OBJECT (self, "err %d", status);
+  goto beach;
 }
 
 static void
diff --git a/sys/applemedia/vtdec.h b/sys/applemedia/vtdec.h
index 3ebb26f9a..e1f3bbe21 100644
--- a/sys/applemedia/vtdec.h
+++ b/sys/applemedia/vtdec.h
@@ -21,6 +21,7 @@
 #define __GST_VTDEC_H__
 
 #include <gst/gst.h>
+#include <gst/video/video.h>
 
 #include "coremediactx.h"
 
@@ -62,10 +63,7 @@ struct _GstVTDec
 
   GstCoreMediaCtx * ctx;
 
-  gint negotiated_width, negotiated_height;
-  gint negotiated_fps_n, negotiated_fps_d;
-  gint caps_width, caps_height;
-  gint caps_fps_n, caps_fps_d;
+  GstVideoInfo vinfo;
   CMFormatDescriptionRef fmt_desc;
   VTDecompressionSessionRef session;