glx: Iterate over FBConfig and select 8 bit color size

Texture upload mechanism used by gstreamer-vaapi relies on 8 bpc.
In latest mesa versions the first fbconfig might not be 8 bit, so iterate
over it to find the correct config with supported values.

This also adds 8 bit alpha size to the framebuffer configuration which is
required to get it working properly.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/410>

1 files changed, 38 insertions, 2 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapiutils_glx.c b/gst-libs/gst/vaapi/gstvaapiutils_glx.c
index ccd7832b..f73106c2 100644
--- a/gst-libs/gst/vaapi/gstvaapiutils_glx.c
+++ b/gst-libs/gst/vaapi/gstvaapiutils_glx.c
@@ -301,9 +301,17 @@ gl_create_context (Display * dpy, int screen, GLContextState * parent)
     GLX_RED_SIZE, 8,
     GLX_GREEN_SIZE, 8,
     GLX_BLUE_SIZE, 8,
+    GLX_ALPHA_SIZE, 8,
     None
   };
 
+  const GLint rgba_colors[4] = {
+    GLX_RED_SIZE,
+    GLX_GREEN_SIZE,
+    GLX_BLUE_SIZE,
+    GLX_ALPHA_SIZE
+  };
+
   cs = malloc (sizeof (*cs));
   if (!cs)
     goto error;
@@ -333,11 +341,38 @@ gl_create_context (Display * dpy, int screen, GLContextState * parent)
     if (!fbconfigs)
       goto error;
 
-    /* Find out a GLXFBConfig compatible with the parent context */
+    /* Find out a 8 bit GLXFBConfig compatible with the parent context */
     for (n = 0; n < n_fbconfigs; n++) {
+      gboolean sizes_correct = FALSE;
+      int cn;
+
       status = glXGetFBConfigAttrib (parent->display,
           fbconfigs[n], GLX_FBCONFIG_ID, &val);
-      if (status == Success && val == fbconfig_id)
+      if (status != Success)
+        goto error;
+      if (val != fbconfig_id)
+        continue;
+
+      /* Iterate over RGBA sizes in fbconfig */
+      for (cn = 0; cn < 4; cn++) {
+        int size = 0;
+
+        status = glXGetFBConfigAttrib (parent->display,
+            fbconfigs[n], rgba_colors[cn], &size);
+        if (status != Success)
+          goto error;
+
+        /* Last check is for alpha
+         * and alpha is optional */
+        if (cn == 3) {
+          if (size == 0 || size == 8) {
+            sizes_correct = TRUE;
+            break;
+          }
+        } else if (size != 8)
+          break;
+      }
+      if (sizes_correct)
         break;
     }
     if (n == n_fbconfigs)
@@ -809,6 +844,7 @@ gl_create_pixmap_object (Display * dpy, guint width, guint height)
     GLX_RED_SIZE, 8,
     GLX_GREEN_SIZE, 8,
     GLX_BLUE_SIZE, 8,
+    GLX_ALPHA_SIZE, 8,
     GL_NONE,
   };