libs: move get_surface_formats to utils_core

The query of all the supported formats for a VA config were only used by the
postprocessor (vaapifilter). But, in order to enable the vaapidecoder to
negotiate a suitable raw format with downstream, we need to query these
formats against the decoder's config.

This patch is the first step: moves the code in filter's ensure_image() to a
generic gst_vaapi_get_surface_formats() in vaapiutils_core, so it can be
shared later by the decoder.

https://bugzilla.gnome.org/show_bug.cgi?id=752958

3 files changed, 85 insertions, 54 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapifilter.c b/gst-libs/gst/vaapi/gstvaapifilter.c
index 1405a0ec..da6f733e 100644
--- a/gst-libs/gst/vaapi/gstvaapifilter.c
+++ b/gst-libs/gst/vaapi/gstvaapifilter.c
@@ -27,6 +27,7 @@
 #include "gstvaapiminiobject.h"
 #include "gstvaapidisplay_priv.h"
 #include "gstvaapisurface_priv.h"
+#include "gstvaapiutils_core.h"
 
 #if USE_VA_VPP
 # include <va/va_vpp.h>
@@ -992,63 +993,12 @@ deint_refs_clear_all (GstVaapiFilter * filter)
 static gboolean
 ensure_formats (GstVaapiFilter * filter)
 {
-#if VA_CHECK_VERSION(0,34,0)
-  VASurfaceAttrib *surface_attribs = NULL;
-  guint i, num_surface_attribs = 0;
-  VAStatus va_status;
-
   if (G_LIKELY (filter->formats))
     return TRUE;
 
-  GST_VAAPI_DISPLAY_LOCK (filter->display);
-  va_status = vaQuerySurfaceAttributes (filter->va_display, filter->va_config,
-      NULL, &num_surface_attribs);
-  GST_VAAPI_DISPLAY_UNLOCK (filter->display);
-  if (!vaapi_check_status (va_status, "vaQuerySurfaceAttributes()"))
-    return FALSE;
-
-  surface_attribs = g_malloc (num_surface_attribs * sizeof (*surface_attribs));
-  if (!surface_attribs)
-    return FALSE;
-
-  GST_VAAPI_DISPLAY_LOCK (filter->display);
-  va_status = vaQuerySurfaceAttributes (filter->va_display, filter->va_config,
-      surface_attribs, &num_surface_attribs);
-  GST_VAAPI_DISPLAY_UNLOCK (filter->display);
-  if (!vaapi_check_status (va_status, "vaQuerySurfaceAttributes()"))
-    return FALSE;
-
-  filter->formats = g_array_sized_new (FALSE, FALSE, sizeof (GstVideoFormat),
-      num_surface_attribs);
-  if (!filter->formats)
-    goto error;
-
-  for (i = 0; i < num_surface_attribs; i++) {
-    const VASurfaceAttrib *const surface_attrib = &surface_attribs[i];
-    GstVideoFormat format;
-
-    if (surface_attrib->type != VASurfaceAttribPixelFormat)
-      continue;
-    if (!(surface_attrib->flags & VA_SURFACE_ATTRIB_SETTABLE))
-      continue;
-
-    format =
-        gst_vaapi_video_format_from_va_fourcc (surface_attrib->value.value.i);
-    if (format == GST_VIDEO_FORMAT_UNKNOWN)
-      continue;
-    g_array_append_val (filter->formats, format);
-  }
-
-  g_free (surface_attribs);
-  return TRUE;
-
-  /* ERRORS */
-error:
-  {
-    g_free (surface_attribs);
-  }
-#endif
-  return FALSE;
+  filter->formats = gst_vaapi_get_surface_formats (filter->display,
+      filter->va_config);
+  return (filter->formats != NULL);
 }
 
 static inline gboolean
diff --git a/gst-libs/gst/vaapi/gstvaapiutils_core.c b/gst-libs/gst/vaapi/gstvaapiutils_core.c
index fe083265..42ebaff1 100644
--- a/gst-libs/gst/vaapi/gstvaapiutils_core.c
+++ b/gst-libs/gst/vaapi/gstvaapiutils_core.c
@@ -24,6 +24,7 @@
 
 #include "sysdeps.h"
 #include "gstvaapicompat.h"
+#include "gstvaapiimage.h"
 #include "gstvaapiutils.h"
 #include "gstvaapiutils_core.h"
 #include "gstvaapidisplay_priv.h"
@@ -75,3 +76,78 @@ gst_vaapi_get_config_attribute (GstVaapiDisplay * display, VAProfile profile,
     *out_value_ptr = attrib.value;
   return TRUE;
 }
+
+/**
+ * gst_vaapi_get_surface_formats:
+ * @display: a #GstVaapiDisplay
+ * @config: a #VAConfigID
+ *
+ * Gets surface formats for the supplied config.
+ *
+ * This function will query for all the supported formats for the
+ * supplied VA @config.
+ *
+ * Return value: (transfer full): a #GArray of #GstVideoFormats or %NULL
+ */
+GArray *
+gst_vaapi_get_surface_formats (GstVaapiDisplay * display, VAConfigID config)
+{
+#if VA_CHECK_VERSION(0,34,0)
+  VASurfaceAttrib *surface_attribs = NULL;
+  guint i, num_surface_attribs = 0;
+  VAStatus va_status;
+  GArray *formats;
+
+  if (config == VA_INVALID_ID)
+    return NULL;
+
+  GST_VAAPI_DISPLAY_LOCK (display);
+  va_status = vaQuerySurfaceAttributes (GST_VAAPI_DISPLAY_VADISPLAY (display),
+      config, NULL, &num_surface_attribs);
+  GST_VAAPI_DISPLAY_UNLOCK (display);
+  if (!vaapi_check_status (va_status, "vaQuerySurfaceAttributes()"))
+    return NULL;
+
+  surface_attribs = g_malloc (num_surface_attribs * sizeof (*surface_attribs));
+  if (!surface_attribs)
+    return NULL;
+
+  GST_VAAPI_DISPLAY_LOCK (display);
+  va_status = vaQuerySurfaceAttributes (GST_VAAPI_DISPLAY_VADISPLAY (display),
+      config, surface_attribs, &num_surface_attribs);
+  GST_VAAPI_DISPLAY_UNLOCK (display);
+  if (!vaapi_check_status (va_status, "vaQuerySurfaceAttributes()"))
+    return NULL;
+
+  formats = g_array_sized_new (FALSE, FALSE, sizeof (GstVideoFormat),
+      num_surface_attribs);
+  if (!formats)
+    goto error;
+
+  for (i = 0; i < num_surface_attribs; i++) {
+    const VASurfaceAttrib *const attrib = &surface_attribs[i];
+    GstVideoFormat fmt;
+
+    if (attrib->type != VASurfaceAttribPixelFormat)
+      continue;
+    if (!(attrib->flags & VA_SURFACE_ATTRIB_SETTABLE))
+      continue;
+
+    fmt = gst_vaapi_video_format_from_va_fourcc (attrib->value.value.i);
+    if (fmt == GST_VIDEO_FORMAT_UNKNOWN)
+      continue;
+    g_array_append_val (formats, fmt);
+  }
+
+  g_free (surface_attribs);
+  return formats;
+
+  /* ERRORS */
+error:
+  {
+    g_free (surface_attribs);
+  }
+#endif
+  return NULL;
+
+}
diff --git a/gst-libs/gst/vaapi/gstvaapiutils_core.h b/gst-libs/gst/vaapi/gstvaapiutils_core.h
index 7f9dcf9b..4fdaad5e 100644
--- a/gst-libs/gst/vaapi/gstvaapiutils_core.h
+++ b/gst-libs/gst/vaapi/gstvaapiutils_core.h
@@ -35,6 +35,11 @@ gboolean
 gst_vaapi_get_config_attribute (GstVaapiDisplay * display, VAProfile profile,
     VAEntrypoint entrypoint, VAConfigAttribType type, guint * out_value_ptr);
 
+/* Gets the available GstVideoFormats of a surface in a VAConfig */
+G_GNUC_INTERNAL
+GArray *
+gst_vaapi_get_surface_formats (GstVaapiDisplay * display, VAConfigID config);
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_UTILS_CORE_H */