glcolorbalance: Error out on unsupported texture target types

The issue can be reproduced on a computer with a Radeon graphics card
when trying to force GStreamer Editing Services to use GL for video
mixing in GESSmartMixer, instead of the GstCompositor that smart mixer
would normally use. This change causes the resulting video stream to
have "video/x-raw(memory:GLMemory) ... texture-target: 2D" caps (instead
of "video/x-raw ..." caps). At the PlaySink stage of the pipeline, a
GstGLImageSinkBin is plugged, with a GstGLColorBalance on it. For some
reason that is still to be debugged (and out of the scope of this
patch), gst_gl_filter_set_caps() is never called on that color balance
element, leaving filter->in_texture_target set to its default
GST_GL_TEXTURE_TARGET_NONE value. The incomplete _create_shader() logic
does the rest and silently generates a shader code that doesn't build.

This is the command I use to reproduce the issue (I'm not sure if I
would be able to isolate the issue in a simple pipeline, though):

 GST_PLUGIN_FEATURE_RANK=vaapih265enc:NONE,vaapih264enc:NONE,vaapisink:NONE,vaapidecodebin:NONE,vaapipostproc:NONE,vaapih265dec:NONE,vaapivc1dec:NONE,vaapih264dec:NONE,vaapimpeg2dec:NONE,vaapijpegdec:NONE,glvideomixer:260
 ges-launch-1.0 +clip /tmp/video.mp4

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/1159>

1 files changed, 14 insertions, 10 deletions

diff --git a/ext/gl/gstglcolorbalance.c b/ext/gl/gstglcolorbalance.c
index f543a4c07..3af45696c 100644
--- a/ext/gl/gstglcolorbalance.c
+++ b/ext/gl/gstglcolorbalance.c
@@ -282,7 +282,7 @@ _create_shader (GstGLColorBalance * balance)
   guint frag_i = 0;
 
   if (balance->shader)
-    gst_object_unref (balance->shader);
+    gst_clear_object (&balance->shader);
 
   if (filter->in_texture_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
     frags[frag_i++] = glsl_external_image_extension;
@@ -292,16 +292,20 @@ _create_shader (GstGLColorBalance * balance)
       GST_GLSL_VERSION_NONE,
       GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY);
 
-  /* Can support rectangle textures in the future if needed */
-  if (filter->in_texture_target == GST_GL_TEXTURE_TARGET_2D) {
-    frags[frag_i++] = glsl_2D_image_sampler;
-    frags[frag_i++] = frag_body =
-        g_strdup_printf (color_balance_frag_templ, "texture2D");
-  } else {
-    frags[frag_i++] = glsl_external_image_sampler;
-    frags[frag_i++] = frag_body =
-        g_strdup_printf (color_balance_frag_templ, "texture2D");
+  switch (filter->in_texture_target) {
+    case GST_GL_TEXTURE_TARGET_2D:
+      frags[frag_i++] = glsl_2D_image_sampler;
+      break;
+    case GST_GL_TEXTURE_TARGET_EXTERNAL_OES:
+      frags[frag_i++] = glsl_external_image_sampler;
+      break;
+    default:
+      GST_ERROR_OBJECT (balance, "Unsupported GstGLTextureTarget value: %d",
+          filter->in_texture_target);
+      return FALSE;
   }
+  frags[frag_i++] = frag_body =
+      g_strdup_printf (color_balance_frag_templ, "texture2D");
 
   g_assert (frag_i <= G_N_ELEMENTS (frags));