video-texture: Respect the pixel aspect ratio of the incoming frames

Video frames come with a pixel-aspect-ratio cap that tells the sink what
should be the final size of the video.

This information is now given to ClutterGstVideoTexture that derives the
preferred width and height based on the base size of the frames and that
pixel-aspect-ratio.

A new debug category "aspect-ratio" has been created to trace what

10 files changed, 272 insertions, 17 deletions

diff --git a/clutter-gst/clutter-gst-debug.c b/clutter-gst/clutter-gst-debug.c
index bf7c2dd..fac4798 100644
--- a/clutter-gst/clutter-gst-debug.c
+++ b/clutter-gst/clutter-gst-debug.c
@@ -36,8 +36,9 @@ guint clutter_gst_debug_flags = 0;  /* global clutter-gst debug flag */
 static GTimer *clutter_gst_timer;
 
 static const GDebugKey clutter_gst_debug_keys[] = {
-  { "misc",  CLUTTER_GST_DEBUG_MISC },
-  { "media", CLUTTER_GST_DEBUG_MEDIA },
+  { "misc",         CLUTTER_GST_DEBUG_MISC },
+  { "media",        CLUTTER_GST_DEBUG_MEDIA },
+  { "aspect-ratio", CLUTTER_GST_DEBUG_ASPECT_RATIO }
 };
 
 /**
diff --git a/clutter-gst/clutter-gst-debug.h b/clutter-gst/clutter-gst-debug.h
index c5706be..02b218a 100644
--- a/clutter-gst/clutter-gst-debug.h
+++ b/clutter-gst/clutter-gst-debug.h
@@ -36,6 +36,7 @@ G_BEGIN_DECLS
 typedef enum {
   CLUTTER_GST_DEBUG_MISC            = 1 << 0,
   CLUTTER_GST_DEBUG_MEDIA           = 1 << 1,
+  CLUTTER_GST_DEBUG_ASPECT_RATIO    = 1 << 2
 } ClutterDebugFlag;
 
 #ifdef __GNUC__
diff --git a/clutter-gst/clutter-gst-private.h b/clutter-gst/clutter-gst-private.h
index 7d63899..81c6b10 100644
--- a/clutter-gst/clutter-gst-private.h
+++ b/clutter-gst/clutter-gst-private.h
@@ -28,6 +28,8 @@
 
 #include <glib.h>
 
+#include "clutter-gst-video-texture.h"
+
 G_BEGIN_DECLS
 
 /* GLib has some define for that, but defining it ourselves allows to require a
@@ -44,6 +46,11 @@ G_BEGIN_DECLS
 #define CLUTTER_GST_PARAM_READWRITE     \
   (G_PARAM_READABLE | G_PARAM_WRITABLE | CLUTTER_GST_PARAM_STATIC)
 
+void
+_clutter_gst_video_texture_set_par (ClutterGstVideoTexture *texture,
+                                    guint                   par_n,
+                                    guint                   par_d);
+
 G_END_DECLS
 
 #endif /* __CLUTTER_GST_PRIVATE_H__ */
diff --git a/clutter-gst/clutter-gst-video-sink.c b/clutter-gst/clutter-gst-video-sink.c
index 20742dc..125402f 100644
--- a/clutter-gst/clutter-gst-video-sink.c
+++ b/clutter-gst/clutter-gst-video-sink.c
@@ -42,6 +42,7 @@
 #endif
 
 #include "clutter-gst-video-sink.h"
+#include "clutter-gst-video-texture.h"
 #include "clutter-gst-private.h"
 #include "clutter-gst-shaders.h"
 
@@ -969,6 +970,17 @@ clutter_gst_video_sink_set_caps (GstBaseSink *bsink,
   else 
     priv->par_n = priv->par_d = 1;
 
+  /* If we happen to use a ClutterGstVideoTexture, now is to good time to
+   * instruct it about the pixel aspect ratio so we can have a correct
+   * natural width/height */
+  if (CLUTTER_GST_IS_VIDEO_TEXTURE (priv->texture))
+    {
+      ClutterGstVideoTexture *texture =
+        (ClutterGstVideoTexture *) priv->texture;
+
+      _clutter_gst_video_texture_set_par (texture, priv->par_n, priv->par_d);
+    }
+
   ret = gst_structure_get_fourcc (structure, "format", &fourcc);
   if (ret && (fourcc == GST_MAKE_FOURCC ('Y', 'V', '1', '2')))
     {
diff --git a/clutter-gst/clutter-gst-video-texture.c b/clutter-gst/clutter-gst-video-texture.c
index 6e18643..cd4e044 100644
--- a/clutter-gst/clutter-gst-video-texture.c
+++ b/clutter-gst/clutter-gst-video-texture.c
@@ -44,6 +44,7 @@
 #include <glib.h>
 #include <gio/gio.h>
 #include <gst/gst.h>
+#include <gst/video/video.h>
 
 #include "clutter-gst-debug.h"
 #include "clutter-gst-private.h"
@@ -64,6 +65,18 @@ struct _ClutterGstVideoTexturePrivate
 
   guint tick_timeout_id;
 
+  /* width / height (in pixels) of the frame data before applying the pixel
+   * aspect ratio */
+  gint buffer_width;
+  gint buffer_height;
+
+  /* Pixel aspect ration is par_n / par_d. this is set by the sink */
+  guint par_n, par_d;
+
+  /* natural width / height (in pixels) of the texture (after par applied) */
+  guint texture_width;
+  guint texture_height;
+
   gdouble buffer_fill;
   gdouble duration;
   gchar *font_name;
@@ -597,9 +610,205 @@ clutter_media_init (ClutterMediaIface *iface)
 }
 
 /*
+ * ClutterTexture implementation
+ */
+
+static void
+clutter_gst_video_texture_size_change (ClutterTexture *texture,
+                                       gint            width,
+                                       gint            height)
+{
+  ClutterGstVideoTexture *video_texture = CLUTTER_GST_VIDEO_TEXTURE (texture);
+  ClutterGstVideoTexturePrivate *priv = video_texture->priv;
+  gboolean changed;
+
+  /* we are being told the actual (as in number of pixels in the buffers)
+   * frame size. Store the values to be used in preferred_width/height() */
+  changed = (priv->buffer_width != width) || (priv->buffer_height != height);
+  priv->buffer_width = width;
+  priv->buffer_height = height;
+
+  if (changed)
+    {
+      /* reset the computed texture dimensions if the underlying frames have
+       * changed size */
+      CLUTTER_GST_NOTE (ASPECT_RATIO, "frame size has been updated to %dx%d",
+                        width, height);
+
+      priv->texture_width = priv->texture_height = 0;
+
+      /* queue a relayout to ask containers/layout manager to ask for
+       * the preferred size again */
+      clutter_actor_queue_relayout (CLUTTER_ACTOR (texture));
+    }
+}
+
+/*
  * Clutter actor implementation
  */
 
+static void
+clutter_gst_video_texture_get_natural_size (ClutterGstVideoTexture *texture,
+                                            gfloat                 *width,
+                                            gfloat                 *height)
+{
+  ClutterGstVideoTexturePrivate *priv = texture->priv;
+  guint dar_n, dar_d;
+  gboolean ret;
+
+  /* we cache texture_width and texture_height */
+
+  if (G_UNLIKELY (priv->buffer_width == 0 || priv->buffer_height == 0))
+    {
+      /* we don't know the size of the frames yet default to 0,0 */
+      priv->texture_width = 0;
+      priv->texture_height = 0;
+    }
+  else if (G_UNLIKELY (priv->texture_width == 0 || priv->texture_height == 0))
+    {
+      CLUTTER_GST_NOTE (ASPECT_RATIO, "frame is %dx%d with par %d/%d",
+                        priv->buffer_width, priv->buffer_height,
+                        priv->par_n, priv->par_d);
+
+      ret = gst_video_calculate_display_ratio (&dar_n, &dar_d,
+                                               priv->buffer_width,
+                                               priv->buffer_height,
+                                               priv->par_n, priv->par_d,
+                                               1, 1);
+      if (ret == FALSE)
+        dar_n = dar_d = 1;
+
+      if (priv->buffer_height % dar_d == 0)
+        {
+          priv->texture_width = gst_util_uint64_scale (priv->buffer_height,
+                                                       dar_n, dar_d);
+          priv->texture_height = priv->buffer_height;
+        }
+      else if (priv->buffer_width % dar_n == 0)
+        {
+          priv->texture_width = priv->buffer_width;
+          priv->texture_height = gst_util_uint64_scale (priv->buffer_width,
+                                                        dar_d, dar_n);
+
+        }
+      else
+        {
+          priv->texture_width = gst_util_uint64_scale (priv->buffer_height,
+                                                       dar_n, dar_d);
+          priv->texture_height = priv->buffer_height;
+        }
+
+      CLUTTER_GST_NOTE (ASPECT_RATIO,
+                        "final size is %dx%d (calculated par is %d/%d)",
+                        priv->texture_width, priv->texture_height,
+                        dar_n, dar_d);
+    }
+
+  if (width)
+    *width = (gfloat)priv->texture_width;
+
+  if (height)
+    *height = (gfloat)priv->texture_height;
+}
+
+static void
+clutter_gst_video_texture_get_preferred_width (ClutterActor *self,
+                                               gfloat        for_height,
+                                               gfloat       *min_width_p,
+                                               gfloat       *natural_width_p)
+{
+  ClutterGstVideoTexture *texture = CLUTTER_GST_VIDEO_TEXTURE (self);
+  ClutterGstVideoTexturePrivate *priv = texture->priv;
+  gboolean sync_size, keep_aspect_ratio;
+  gfloat natural_width, natural_height;
+
+  /* Min request is always 0 since we can scale down or clip */
+  if (min_width_p)
+    *min_width_p = 0;
+
+  sync_size = clutter_texture_get_sync_size (CLUTTER_TEXTURE (self));
+  keep_aspect_ratio =
+    clutter_texture_get_keep_aspect_ratio (CLUTTER_TEXTURE (self));
+
+  clutter_gst_video_texture_get_natural_size (texture,
+                                              &natural_width,
+                                              &natural_height);
+
+  if (sync_size)
+    {
+      if (natural_width_p)
+        {
+          if (!keep_aspect_ratio ||
+              for_height < 0 ||
+              priv->buffer_height <= 0)
+            {
+              *natural_width_p = natural_width;
+            }
+          else
+            {
+              /* Set the natural width so as to preserve the aspect ratio */
+              gfloat ratio =  natural_width /  natural_height;
+
+              *natural_width_p = ratio * for_height;
+            }
+        }
+    }
+  else
+    {
+      if (natural_width_p)
+        *natural_width_p = 0;
+    }
+}
+
+static void
+clutter_gst_video_texture_get_preferred_height (ClutterActor *self,
+                                                gfloat        for_width,
+                                                gfloat       *min_height_p,
+                                                gfloat       *natural_height_p)
+{
+  ClutterGstVideoTexture *texture = CLUTTER_GST_VIDEO_TEXTURE (self);
+  ClutterGstVideoTexturePrivate *priv = texture->priv;
+  gboolean sync_size, keep_aspect_ratio;
+  gfloat natural_width, natural_height;
+
+  /* Min request is always 0 since we can scale down or clip */
+  if (min_height_p)
+    *min_height_p = 0;
+
+  sync_size = clutter_texture_get_sync_size (CLUTTER_TEXTURE (self));
+  keep_aspect_ratio =
+    clutter_texture_get_keep_aspect_ratio (CLUTTER_TEXTURE (self));
+
+  clutter_gst_video_texture_get_natural_size (texture,
+                                              &natural_width,
+                                              &natural_height);
+
+  if (sync_size)
+    {
+      if (natural_height_p)
+        {
+          if (!keep_aspect_ratio ||
+              for_width < 0 ||
+              priv->buffer_width <= 0)
+            {
+              *natural_height_p = natural_height;
+            }
+          else
+            {
+              /* Set the natural height so as to preserve the aspect ratio */
+              gfloat ratio = natural_height / natural_width;
+
+              *natural_height_p = ratio * for_width;
+            }
+        }
+    }
+  else
+    {
+      if (natural_height_p)
+        *natural_height_p = 0;
+    }
+}
+
 /*
  * ClutterTexture unconditionnaly sets the material color to:
  *    (opacity,opacity,opacity,opacity)
@@ -814,6 +1023,7 @@ clutter_gst_video_texture_class_init (ClutterGstVideoTextureClass *klass)
 {
   GObjectClass *object_class = G_OBJECT_CLASS (klass);
   ClutterActorClass *actor_class = CLUTTER_ACTOR_CLASS (klass);
+  ClutterTextureClass *texture_class = CLUTTER_TEXTURE_CLASS (klass);
   GParamSpec *pspec;
 
   g_type_class_add_private (klass, sizeof (ClutterGstVideoTexturePrivate));
@@ -824,6 +1034,12 @@ clutter_gst_video_texture_class_init (ClutterGstVideoTextureClass *klass)
   object_class->get_property = clutter_gst_video_texture_get_property;
 
   actor_class->paint = clutter_gst_video_texture_paint;
+  actor_class->get_preferred_width =
+    clutter_gst_video_texture_get_preferred_width;
+  actor_class->get_preferred_height =
+    clutter_gst_video_texture_get_preferred_height;
+
+  texture_class->size_change = clutter_gst_video_texture_size_change;
 
   g_object_class_override_property (object_class,
                                     PROP_URI, "uri");
@@ -1086,9 +1302,10 @@ clutter_gst_video_texture_init (ClutterGstVideoTexture *video_texture)
   create_black_idle_material (video_texture);
 
   priv->is_idle = TRUE;
-
   priv->in_seek = FALSE;
 
+  priv->par_n = priv->par_d = 1;
+
   bus = gst_pipeline_get_bus (GST_PIPELINE (priv->pipeline));
 
   gst_bus_add_signal_watch (bus);
@@ -1115,6 +1332,24 @@ clutter_gst_video_texture_init (ClutterGstVideoTexture *video_texture)
   gst_object_unref (GST_OBJECT (bus));
 }
 
+/*
+ * Private symbols
+ */
+
+/* This function is called from the sink set_caps(). we receive the first
+ * buffer way after this so are told about the par before size_changed has
+ * been fired */
+void
+_clutter_gst_video_texture_set_par (ClutterGstVideoTexture *texture,
+                                    guint                   par_n,
+                                    guint                   par_d)
+{
+  ClutterGstVideoTexturePrivate *priv = texture->priv;
+
+  priv->par_n = par_n;
+  priv->par_d = par_d;
+}
+
 /**
  * clutter_gst_video_texture_new:
  *
diff --git a/examples/video-player.c b/examples/video-player.c
index 0bf53fb..8563464 100644
--- a/examples/video-player.c
+++ b/examples/video-player.c
@@ -224,17 +224,23 @@ input_cb (ClutterStage *stage,
 
 static void
 size_change (ClutterTexture *texture, 
-             gint            width,
-             gint            height,
+             gint            base_width,
+             gint            base_height,
              VideoApp       *app)
 {
   ClutterActor *stage = app->stage;
   gfloat new_x, new_y, new_width, new_height;
   gfloat stage_width, stage_height;
+  gfloat frame_width, frame_height;
 
   clutter_actor_get_size (stage, &stage_width, &stage_height);
 
-  new_height = (height * stage_width) / width;
+  /* base_width and base_height are the actual dimensions of the buffers before
+   * taking the pixel aspect ratio into account. We need to get the actual
+   * size of the texture to display */
+  clutter_actor_get_size (CLUTTER_ACTOR (texture), &frame_width, &frame_height);
+
+  new_height = (frame_height * stage_width) / frame_width;
   if (new_height <= stage_height)
     {
       new_width = stage_width;
@@ -244,7 +250,7 @@ size_change (ClutterTexture *texture,
     }
   else
     {
-      new_width  = (width * stage_height) / height;
+      new_width  = (frame_width * stage_height) / frame_height;
       new_height = stage_height;
 
       new_x = (stage_width - new_width) / 2;
@@ -323,13 +329,10 @@ main (int argc, char *argv[])
                     G_CALLBACK (on_fullscreen),
                     app);
 
-  /* Dont let the underlying pixbuf dictate size */
-  g_object_set (G_OBJECT(app->vtexture), "sync-size", FALSE, NULL);
-
   /* Handle it ourselves so can scale up for fullscreen better */
-  g_signal_connect (CLUTTER_TEXTURE (app->vtexture),
-                    "size-change",
-                    G_CALLBACK (size_change), app);
+  g_signal_connect_after (CLUTTER_TEXTURE (app->vtexture),
+                          "size-change",
+                          G_CALLBACK (size_change), app);
 
   /* Load up out video texture */
   clutter_media_set_filename (CLUTTER_MEDIA (app->vtexture), argv[1]);
diff --git a/examples/video-sink.c b/examples/video-sink.c
index 0391572..1e7fefc 100644
--- a/examples/video-sink.c
+++ b/examples/video-sink.c
@@ -93,7 +93,6 @@ main (int argc, char *argv[])
    * efficient/corrent playback onto the texture (which sucks a bit)  
   */
   texture = g_object_new (CLUTTER_TYPE_TEXTURE, 
-			  "sync-size",       FALSE, 
 			  "disable-slicing", TRUE, 
 			  NULL);
 
diff --git a/tests/test-alpha.c b/tests/test-alpha.c
index 9214e90..b577eba 100644
--- a/tests/test-alpha.c
+++ b/tests/test-alpha.c
@@ -150,7 +150,6 @@ main (int argc, char *argv[])
    * efficient/corrent playback onto the texture (which sucks a bit)
    */
   texture = g_object_new (CLUTTER_TYPE_TEXTURE,
-                          "sync-size",       FALSE,
                           "disable-slicing", TRUE,
                           NULL);
   clutter_actor_set_opacity (texture, 0);
diff --git a/tests/test-rgb-upload.c b/tests/test-rgb-upload.c
index 273997d..388fa9f 100644
--- a/tests/test-rgb-upload.c
+++ b/tests/test-rgb-upload.c
@@ -136,7 +136,6 @@ main (int argc, char *argv[])
    * efficient/corrent playback onto the texture (which sucks a bit)
    */
   texture = g_object_new (CLUTTER_TYPE_TEXTURE,
-                          "sync-size",       FALSE,
                           "disable-slicing", TRUE,
                           NULL);
 
diff --git a/tests/test-yuv-upload.c b/tests/test-yuv-upload.c
index a15fc27..f890a82 100644
--- a/tests/test-yuv-upload.c
+++ b/tests/test-yuv-upload.c
@@ -134,7 +134,6 @@ main (int argc, char *argv[])
    * efficient/corrent playback onto the texture (which sucks a bit)
    */
   texture = g_object_new (CLUTTER_TYPE_TEXTURE,
-                          "sync-size",       FALSE,
                           "disable-slicing", TRUE,
                           NULL);