plugins: Add more check for allowed raw caps.

The gst_vaapi_plugin_base_get_allowed_raw_caps is used for both sink
pad and src pad, which cause some bugs. For sink pad, we need to verify
vaPutImage() while for the src pad we need to verify vaGetImage().

For vaapidecoderXXX kind of plugins, the case is more complex. We need
to verify whether the decoded result(in some surface, NV12 format most
of the time) can be vaGetImage to some raw image format. Add more check
to fix all these problems.

Fixes: #123

Signed-off-by: He Junyan's avatarHe Junyan <junyan.he@hotmail.com>

5 files changed, 107 insertions, 39 deletions

diff --git a/gst/vaapi/gstvaapidecode.c b/gst/vaapi/gstvaapidecode.c
index 68ec0851..76380405 100644
--- a/gst/vaapi/gstvaapidecode.c
+++ b/gst/vaapi/gstvaapidecode.c
@@ -238,8 +238,9 @@ gst_vaapidecode_ensure_allowed_srcpad_caps (GstVaapiDecode * decode)
   out_caps = gst_caps_make_writable (out_caps);
   gst_caps_append (out_caps, gst_caps_from_string (GST_VAAPI_MAKE_DMABUF_CAPS));
 
-  raw_caps = gst_vaapi_plugin_base_get_allowed_raw_caps
-      (GST_VAAPI_PLUGIN_BASE (decode));
+  raw_caps = gst_vaapi_plugin_base_get_allowed_srcpad_raw_caps
+      (GST_VAAPI_PLUGIN_BASE (decode),
+      GST_VIDEO_INFO_FORMAT (&decode->decoded_info));
   if (!raw_caps) {
     gst_caps_unref (out_caps);
     GST_WARNING_OBJECT (decode, "failed to create raw sink caps");
diff --git a/gst/vaapi/gstvaapipluginbase.c b/gst/vaapi/gstvaapipluginbase.c
index d23b519a..c9290515 100644
--- a/gst/vaapi/gstvaapipluginbase.c
+++ b/gst/vaapi/gstvaapipluginbase.c
@@ -1253,51 +1253,96 @@ no_valid_gl_display:
 #endif
 }
 
+static GArray *
+extract_allowed_surface_formats (GstVaapiDisplay * display,
+    GArray * img_formats, GstVideoFormat specified_format,
+    GstPadDirection direction)
+{
+  guint i;
+  GArray *out_formats;
+  GstVaapiSurface *surface = NULL;
+
+  g_assert (direction == GST_PAD_SRC || direction == GST_PAD_SINK);
+
+  out_formats =
+      g_array_sized_new (FALSE, FALSE, sizeof (GstVideoFormat),
+      img_formats->len);
+  if (!out_formats)
+    return NULL;
+
+  for (i = 0; i < img_formats->len; i++) {
+    const GstVideoFormat img_format =
+        g_array_index (img_formats, GstVideoFormat, i);
+    GstVaapiImage *image;
+    GstVideoInfo vi;
+    GstVideoFormat surface_format;
+    gboolean res;
+
+    if (img_format == GST_VIDEO_FORMAT_UNKNOWN)
+      continue;
+
+    surface_format =
+        (specified_format == GST_VIDEO_FORMAT_UNKNOWN) ?
+        img_format : specified_format;
+    if (!surface) {
+      gst_video_info_set_format (&vi, surface_format, 64, 64);
+      surface = gst_vaapi_surface_new_full (display, &vi, 0);
+      if (!surface)
+        continue;
+    }
+
+    image = gst_vaapi_image_new (display, img_format, 64, 64);
+    if (!image) {
+      /* Just reuse the surface if the format is specified */
+      if (specified_format == GST_VIDEO_FORMAT_UNKNOWN)
+        gst_vaapi_object_replace (&surface, NULL);
+
+      continue;
+    }
+
+    res = FALSE;
+    if (direction == GST_PAD_SRC) {
+      res = gst_vaapi_surface_get_image (surface, image);
+    } else {
+      res = gst_vaapi_surface_put_image (surface, image);
+    }
+    if (res)
+      g_array_append_val (out_formats, img_format);
+
+    gst_vaapi_object_unref (image);
+    /* Just reuse the surface if the format is specified */
+    if (specified_format == GST_VIDEO_FORMAT_UNKNOWN)
+      gst_vaapi_object_replace (&surface, NULL);
+  }
+
+  if (out_formats->len == 0) {
+    g_array_unref (out_formats);
+    return NULL;
+  }
+  return out_formats;
+}
+
 static gboolean
-ensure_allowed_raw_caps (GstVaapiPluginBase * plugin)
+ensure_allowed_raw_caps (GstVaapiPluginBase * plugin, GstVideoFormat format,
+    GstPadDirection direction)
 {
   GArray *formats, *out_formats;
-  GstVaapiSurface *surface;
   GstVaapiDisplay *display;
-  guint i;
   GstCaps *out_caps;
   gboolean ret = FALSE;
 
   if (plugin->allowed_raw_caps)
     return TRUE;
 
-  out_formats = formats = NULL;
-  surface = NULL;
-
+  out_formats = NULL;
   display = gst_vaapi_display_ref (plugin->display);
   formats = gst_vaapi_display_get_image_formats (display);
   if (!formats)
     goto bail;
-
   out_formats =
-      g_array_sized_new (FALSE, FALSE, sizeof (GstVideoFormat), formats->len);
+      extract_allowed_surface_formats (display, formats, format, direction);
   if (!out_formats)
     goto bail;
-
-  surface =
-      gst_vaapi_surface_new (display, GST_VAAPI_CHROMA_TYPE_YUV420, 64, 64);
-  if (!surface)
-    goto bail;
-
-  for (i = 0; i < formats->len; i++) {
-    const GstVideoFormat format = g_array_index (formats, GstVideoFormat, i);
-    GstVaapiImage *image;
-
-    if (format == GST_VIDEO_FORMAT_UNKNOWN)
-      continue;
-    image = gst_vaapi_image_new (display, format, 64, 64);
-    if (!image)
-      continue;
-    if (gst_vaapi_surface_put_image (surface, image))
-      g_array_append_val (out_formats, format);
-    gst_vaapi_object_unref (image);
-  }
-
   out_caps = gst_vaapi_video_format_new_template_caps_from_list (out_formats);
   if (!out_caps)
     goto bail;
@@ -1311,25 +1356,41 @@ bail:
     g_array_unref (formats);
   if (out_formats)
     g_array_unref (out_formats);
-  if (surface)
-    gst_vaapi_object_unref (surface);
   gst_vaapi_display_unref (display);
 
   return ret;
 }
 
 /**
- * gst_vaapi_plugin_base_get_allowed_raw_caps:
+ * gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps:
+ * @plugin: a #GstVaapiPluginBase
+ *
+ * Returns the raw #GstCaps allowed by the element.
+ *
+ * Returns: the allowed raw #GstCaps or %NULL
+ **/
+GstCaps *
+gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps (GstVaapiPluginBase * plugin)
+{
+  if (!ensure_allowed_raw_caps (plugin, GST_VIDEO_FORMAT_UNKNOWN, GST_PAD_SINK))
+    return NULL;
+  return plugin->allowed_raw_caps;
+}
+
+/**
+ * gst_vaapi_plugin_base_get_allowed_srcpad_raw_caps:
  * @plugin: a #GstVaapiPluginBase
+ * @format: a #GstVideoFormat, the format we need to check
  *
  * Returns the raw #GstCaps allowed by the element.
  *
  * Returns: the allowed raw #GstCaps or %NULL
  **/
 GstCaps *
-gst_vaapi_plugin_base_get_allowed_raw_caps (GstVaapiPluginBase * plugin)
+gst_vaapi_plugin_base_get_allowed_srcpad_raw_caps (GstVaapiPluginBase *
+    plugin, GstVideoFormat format)
 {
-  if (!ensure_allowed_raw_caps (plugin))
+  if (!ensure_allowed_raw_caps (plugin, format, GST_PAD_SRC))
     return NULL;
   return plugin->allowed_raw_caps;
 }
diff --git a/gst/vaapi/gstvaapipluginbase.h b/gst/vaapi/gstvaapipluginbase.h
index c0d07496..9faa0639 100644
--- a/gst/vaapi/gstvaapipluginbase.h
+++ b/gst/vaapi/gstvaapipluginbase.h
@@ -251,7 +251,12 @@ gst_vaapi_plugin_base_create_gl_context (GstVaapiPluginBase * plugin);
 
 G_GNUC_INTERNAL
 GstCaps *
-gst_vaapi_plugin_base_get_allowed_raw_caps (GstVaapiPluginBase * plugin);
+gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps (GstVaapiPluginBase * plugin);
+
+G_GNUC_INTERNAL
+GstCaps *
+gst_vaapi_plugin_base_get_allowed_srcpad_raw_caps (
+    GstVaapiPluginBase * plugin, GstVideoFormat format);
 
 G_GNUC_INTERNAL
 void
diff --git a/gst/vaapi/gstvaapipostproc.c b/gst/vaapi/gstvaapipostproc.c
index 3a770c7c..a2feff4d 100644
--- a/gst/vaapi/gstvaapipostproc.c
+++ b/gst/vaapi/gstvaapipostproc.c
@@ -1096,7 +1096,7 @@ ensure_allowed_sinkpad_caps (GstVaapiPostproc * postproc)
     return FALSE;
   }
 
-  raw_caps = gst_vaapi_plugin_base_get_allowed_raw_caps
+  raw_caps = gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps
       (GST_VAAPI_PLUGIN_BASE (postproc));
   if (!raw_caps) {
     gst_caps_unref (out_caps);
diff --git a/gst/vaapi/gstvaapisink.c b/gst/vaapi/gstvaapisink.c
index 9333f438..c1965833 100644
--- a/gst/vaapi/gstvaapisink.c
+++ b/gst/vaapi/gstvaapisink.c
@@ -1215,7 +1215,7 @@ gst_vaapisink_start (GstBaseSink * base_sink)
 
   /* Ensures possible raw caps earlier to avoid race conditions at
    * get_caps() */
-  if (!gst_vaapi_plugin_base_get_allowed_raw_caps (plugin))
+  if (!gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps (plugin))
     return FALSE;
 
   return TRUE;
@@ -1251,7 +1251,8 @@ gst_vaapisink_get_caps_impl (GstBaseSink * base_sink)
 
   out_caps = gst_caps_from_string (surface_caps_str);
   raw_caps =
-      gst_vaapi_plugin_base_get_allowed_raw_caps (GST_VAAPI_PLUGIN_BASE (sink));
+      gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps (GST_VAAPI_PLUGIN_BASE
+      (sink));
   if (!raw_caps)
     return out_caps;