sobel: convolve only luma

Rework Sobel a little bit again making it work as the old one:
1. desaturate input texture
2. calculate horizontal convolution for x gradient and vertical
   convolution for y gradient at the same time (halves the number of
   needed texture lookups)
3. store results in a single texture (red and green channel)
4. calculate remaining convolution (same as above switching vertical and
   horizontal)
5. calculate length of gradient using red and green as x and y
   components.
Optimize wherever possible, store kernels as constants in the shaders,
remove unneeded uniforms. Restore invert property carefully avoiding
using IF.
Still not sure if "full color" convolution will be needed, glfiltersobel
is to be intended as a demo filter and xray, the only effect which uses
sobel only needs edge intensity. Dropping it for now.

4 files changed, 91 insertions, 114 deletions

diff --git a/gst/gl/effects/gstgleffectssources.c b/gst/gl/effects/gstgleffectssources.c
index 27b2c74..b26be16 100644
--- a/gst/gl/effects/gstgleffectssources.c
+++ b/gst/gl/effects/gstgleffectssources.c
@@ -308,60 +308,79 @@ const gchar *sobel_fragment_source =
   "  gl_FragColor = vec4(vec3(g), 1.0);"
   "}";
 
-const gchar *sobel_gradient_length_fragment_source =
+const gchar *sep_sobel_length_fragment_source =
   "#extension GL_ARB_texture_rectangle : enable\n"
-  "uniform sampler2DRect gx;"
-  "uniform sampler2DRect gy;"
+  "uniform sampler2DRect tex;"
+  "uniform bool invert;"
   "void main () {"
-  "  vec4 dx = texture2DRect (gx, gl_TexCoord[0].st);"
-  "  vec4 dy = texture2DRect (gy, gl_TexCoord[0].st);"
-  "  dx = (dx - 0.5);"
-  "  dy = (dy - 0.5);"
-  "  gl_FragColor = vec4(sqrt(dx*dx + dy*dy));"
+  "  vec4 g = texture2DRect (tex, gl_TexCoord[0].st);"
+  /* restore black background with grey edges */
+  "  g -= vec4(0.5, 0.5, 0.0, 0.0);"
+  "  float len = length (g);"
+  /* little trick to avoid IF operator */
+  /* TODO: test if a standalone inverting pass is worth */
+  "  gl_FragColor = abs(int(invert) - vec4(vec3(len), 1.0));"
+  "}";
+
+const gchar *desaturate_fragment_source =
+  "#extension GL_ARB_texture_rectangle : enable\n"
+  "uniform sampler2DRect tex;"
+  "void main () {"
+  "  vec4 color = texture2DRect (tex, gl_TexCoord[0].st);"
+  "  float luma = dot(color.rgb, vec3(0.2125, 0.7154, 0.0721));"
+  "  gl_FragColor = vec4(vec3(luma), color.a);"
   "}";
 
-/* horizontal convolution 3x3 */
-const gchar *hconv3_fragment_source =
+const gchar *sep_sobel_hconv3_fragment_source =
   "#extension GL_ARB_texture_rectangle : enable\n"
   "uniform sampler2DRect tex;"
-  "uniform float kernel[3];"
-  "uniform float offset;"
   "void main () {"
   "  vec2 texturecoord[3];"
-  "  float s = gl_TexCoord[0].s;"
-  "  float t = gl_TexCoord[0].t;"
-  "  texturecoord[0] = vec2(s-1.0, t);"
-  "  texturecoord[1] = vec2(s, t);"
-  "  texturecoord[2] = vec2(s+1.0, t);"
+  "  texturecoord[1] = gl_TexCoord[0].st;"
+  "  texturecoord[0] = texturecoord[1] - vec2(1.0, 0.0);"
+  "  texturecoord[2] = texturecoord[1] + vec2(1.0, 0.0);"
+  "  float grad_kern[3];"
+  "  grad_kern[0] = 1.0;"
+  "  grad_kern[1] = 0.0;"
+  "  grad_kern[2] = -1.0;"
+  "  float blur_kern[3];"
+  "  blur_kern[0] = 0.25;"
+  "  blur_kern[1] = 0.5;"
+  "  blur_kern[2] = 0.25;"
   "  int i;"
   "  vec4 sum = vec4 (0.0);"
   "  for (i = 0; i < 3; i++) { "
   "    vec4 neighbor = texture2DRect(tex, texturecoord[i]); "
-  "    sum += neighbor * kernel[i];"
+  "    sum.r = neighbor.r * blur_kern[i] + sum.r;"
+  "    sum.g = neighbor.g * grad_kern[i] + sum.g;"
   "  }"
-  "  gl_FragColor = sum + offset;"
+  "  gl_FragColor = sum + vec4(0.0, 0.5, 0.0, 0.0);"
   "}";
 
-/* vertical convolution 3x3 */
-const gchar *vconv3_fragment_source =
+const gchar *sep_sobel_vconv3_fragment_source =
   "#extension GL_ARB_texture_rectangle : enable\n"
   "uniform sampler2DRect tex;"
-  "uniform float kernel[3];"
-  "uniform float offset;"
   "void main () {"
   "  vec2 texturecoord[3];"
-  "  float s = gl_TexCoord[0].s;"
-  "  float t = gl_TexCoord[0].t;"
-  "  texturecoord[0] = vec2(s, t-1.0);"
-  "  texturecoord[1] = vec2(s, t);"
-  "  texturecoord[2] = vec2(s, t+1.0);"
+  "  texturecoord[1] = gl_TexCoord[0].st;"
+  "  texturecoord[0] = texturecoord[1] - vec2(0.0, 1.0);"
+  "  texturecoord[2] = texturecoord[1] + vec2(0.0, 1.0);"
+  "  float grad_kern[3];"
+  "  grad_kern[0] = 1.0;"
+  "  grad_kern[1] = 0.0;"
+  "  grad_kern[2] = -1.0;"
+  "  float blur_kern[3];"
+  "  blur_kern[0] = 0.25;"
+  "  blur_kern[1] = 0.5;"
+  "  blur_kern[2] = 0.25;"
   "  int i;"
   "  vec4 sum = vec4 (0.0);"
   "  for (i = 0; i < 3; i++) { "
-  "    vec4 neighbor = texture2DRect(tex, texturecoord[i]);"
-  "    sum += neighbor * kernel[i]; "
+  "    vec4 neighbor = texture2DRect(tex, texturecoord[i]); "
+  "    sum.r = neighbor.r * grad_kern[i] + sum.r;"
+  "    sum.g = neighbor.g * blur_kern[i] + sum.g;"
   "  }"
-  "  gl_FragColor = sum + offset;"
+  "  gl_FragColor = sum + vec4(0.5, 0.0, 0.0, 0.0);"
   "}";
 
 /* horizontal convolution 9x9 */
diff --git a/gst/gl/effects/gstgleffectssources.h b/gst/gl/effects/gstgleffectssources.h
index fc69b8b..eebb87d 100644
--- a/gst/gl/effects/gstgleffectssources.h
+++ b/gst/gl/effects/gstgleffectssources.h
@@ -33,9 +33,10 @@ extern const gchar *bulge_fragment_source;
 extern const gchar *square_fragment_source;
 extern const gchar *luma_threshold_fragment_source;
 extern const gchar *sobel_fragment_source;
-extern const gchar *sobel_gradient_length_fragment_source;
-extern const gchar *hconv3_fragment_source;
-extern const gchar *vconv3_fragment_source;
+extern const gchar *sep_sobel_length_fragment_source;
+extern const gchar *desaturate_fragment_source;
+extern const gchar *sep_sobel_hconv3_fragment_source;
+extern const gchar *sep_sobel_vconv3_fragment_source;
 extern const gchar *hconv9_fragment_source;
 extern const gchar *vconv9_fragment_source;
 extern const gchar *sum_fragment_source;
diff --git a/gst/gl/gstglfiltersobel.c b/gst/gl/gstglfiltersobel.c
index d2c2b93..535d971 100644
--- a/gst/gl/gstglfiltersobel.c
+++ b/gst/gl/gstglfiltersobel.c
@@ -65,24 +65,15 @@ static void gst_gl_filtersobel_draw_texture (GstGLFilterSobel * filtersobel,
 static void gst_gl_filtersobel_init_shader (GstGLFilter * filter);
 static gboolean gst_gl_filtersobel_filter (GstGLFilter * filter,
     GstGLBuffer * inbuf, GstGLBuffer * outbuf);
-static void gst_gl_filtersobel_step_one (gint width, gint height, guint texture,
+
+static void gst_gl_filtersobel_desaturate (gint width, gint height,
+    guint texture, gpointer stuff);
+static void gst_gl_filtersobel_hconv (gint width, gint height, guint texture,
     gpointer stuff);
-static void gst_gl_filtersobel_step_two (gint width, gint height, guint texture,
+static void gst_gl_filtersobel_vconv (gint width, gint height, guint texture,
+    gpointer stuff);
+static void gst_gl_filtersobel_length (gint width, gint height, guint texture,
     gpointer stuff);
-static void gst_gl_filtersobel_step_three (gint width, gint height,
-    guint texture, gpointer stuff);
-static void gst_gl_filtersobel_step_four (gint width, gint height,
-    guint texture, gpointer stuff);
-static void gst_gl_filtersobel_step_five (gint width, gint height,
-    guint texture, gpointer stuff);
-
-static gfloat grad_kern[3] = {
-  1.0, 0.0, -1.0,
-};
-
-static gfloat blur_kern[3] = {
-  1.0 / 4.0, 2.0 / 4.0, 1.0 / 4.0,
-};
 
 static void
 gst_gl_filtersobel_init_resources (GstGLFilter * filter)
@@ -90,7 +81,7 @@ gst_gl_filtersobel_init_resources (GstGLFilter * filter)
   GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter);
   int i;
 
-  for (i = 0; i < 5; i++) {
+  for (i = 0; i < 2; i++) {
     glGenTextures (1, &filtersobel->midtexture[i]);
     glBindTexture (GL_TEXTURE_RECTANGLE_ARB, filtersobel->midtexture[i]);
     glTexImage2D (GL_TEXTURE_RECTANGLE_ARB, 0, GL_RGBA8,
@@ -112,7 +103,7 @@ gst_gl_filtersobel_reset_resources (GstGLFilter * filter)
   GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter);
   int i;
 
-  for (i = 0; i < 5; i++) {
+  for (i = 0; i < 2; i++) {
     glDeleteTextures (1, &filtersobel->midtexture[i]);
   }
 }
@@ -160,7 +151,7 @@ gst_gl_filtersobel_init (GstGLFilterSobel * filtersobel,
   filtersobel->hconv = NULL;
   filtersobel->vconv = NULL;
   filtersobel->invert = FALSE;
-  for (i = 0; i < 5; i++) {
+  for (i = 0; i < 2; i++) {
     filtersobel->midtexture[i] = 0;
   }
 }
@@ -171,6 +162,7 @@ gst_gl_filter_filtersobel_reset (GstGLFilter * filter)
   GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter);
 
   //blocking call, wait the opengl thread has destroyed the shader
+  gst_gl_display_del_shader (filter->display, filtersobel->desat);
   gst_gl_display_del_shader (filter->display, filtersobel->hconv);
   gst_gl_display_del_shader (filter->display, filtersobel->vconv);
   gst_gl_display_del_shader (filter->display, filtersobel->len);
@@ -214,12 +206,14 @@ gst_gl_filtersobel_init_shader (GstGLFilter * filter)
   GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter);
 
   //blocking call, wait the opengl thread has compiled the shader
-  gst_gl_display_gen_shader (filter->display, 0, hconv3_fragment_source,
-      &filtersobel->hconv);
-  gst_gl_display_gen_shader (filter->display, 0, vconv3_fragment_source,
-      &filtersobel->vconv);
+  gst_gl_display_gen_shader (filter->display, 0, desaturate_fragment_source,
+      &filtersobel->desat);
+  gst_gl_display_gen_shader (filter->display, 0,
+      sep_sobel_hconv3_fragment_source, &filtersobel->hconv);
+  gst_gl_display_gen_shader (filter->display, 0,
+      sep_sobel_vconv3_fragment_source, &filtersobel->vconv);
   gst_gl_display_gen_shader (filter->display, 0,
-      sobel_gradient_length_fragment_source, &filtersobel->len);
+      sep_sobel_length_fragment_source, &filtersobel->len);
 }
 
 static void
@@ -252,21 +246,18 @@ gst_gl_filtersobel_filter (GstGLFilter * filter, GstGLBuffer * inbuf,
   GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter);
 
   gst_gl_filter_render_to_target (filter, inbuf->texture,
-      filtersobel->midtexture[0], gst_gl_filtersobel_step_one, filtersobel);
+      filtersobel->midtexture[0], gst_gl_filtersobel_desaturate, filtersobel);
   gst_gl_filter_render_to_target (filter, filtersobel->midtexture[0],
-      filtersobel->midtexture[1], gst_gl_filtersobel_step_two, filtersobel);
-  gst_gl_filter_render_to_target (filter, inbuf->texture,
-      filtersobel->midtexture[2], gst_gl_filtersobel_step_three, filtersobel);
-  gst_gl_filter_render_to_target (filter, filtersobel->midtexture[2],
-      filtersobel->midtexture[3], gst_gl_filtersobel_step_four, filtersobel);
-  gst_gl_filter_render_to_target (filter, filtersobel->midtexture[3],
-      outbuf->texture, gst_gl_filtersobel_step_five, filtersobel);
-
+      filtersobel->midtexture[1], gst_gl_filtersobel_hconv, filtersobel);
+  gst_gl_filter_render_to_target (filter, filtersobel->midtexture[1],
+      filtersobel->midtexture[0], gst_gl_filtersobel_vconv, filtersobel);
+  gst_gl_filter_render_to_target (filter, filtersobel->midtexture[0],
+      outbuf->texture, gst_gl_filtersobel_length, filtersobel);
   return TRUE;
 }
 
 static void
-gst_gl_filtersobel_step_one (gint width, gint height, guint texture,
+gst_gl_filtersobel_desaturate (gint width, gint height, guint texture,
     gpointer stuff)
 {
   GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff);
@@ -274,22 +265,20 @@ gst_gl_filtersobel_step_one (gint width, gint height, guint texture,
   glMatrixMode (GL_PROJECTION);
   glLoadIdentity ();
 
-  gst_gl_shader_use (filtersobel->hconv);
+  gst_gl_shader_use (filtersobel->desat);
 
   glActiveTexture (GL_TEXTURE1);
   glEnable (GL_TEXTURE_RECTANGLE_ARB);
   glBindTexture (GL_TEXTURE_RECTANGLE_ARB, texture);
   glDisable (GL_TEXTURE_RECTANGLE_ARB);
 
-  gst_gl_shader_set_uniform_1i (filtersobel->hconv, "tex", 1);
-  gst_gl_shader_set_uniform_1fv (filtersobel->hconv, "kernel", 3, blur_kern);
-  gst_gl_shader_set_uniform_1f (filtersobel->hconv, "offset", 0.0);
+  gst_gl_shader_set_uniform_1i (filtersobel->desat, "tex", 1);
 
   gst_gl_filtersobel_draw_texture (filtersobel, texture);
 }
 
 static void
-gst_gl_filtersobel_step_two (gint width, gint height, guint texture,
+gst_gl_filtersobel_hconv (gint width, gint height, guint texture,
     gpointer stuff)
 {
   GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff);
@@ -297,22 +286,20 @@ gst_gl_filtersobel_step_two (gint width, gint height, guint texture,
   glMatrixMode (GL_PROJECTION);
   glLoadIdentity ();
 
-  gst_gl_shader_use (filtersobel->vconv);
+  gst_gl_shader_use (filtersobel->hconv);
 
   glActiveTexture (GL_TEXTURE1);
   glEnable (GL_TEXTURE_RECTANGLE_ARB);
   glBindTexture (GL_TEXTURE_RECTANGLE_ARB, texture);
   glDisable (GL_TEXTURE_RECTANGLE_ARB);
 
-  gst_gl_shader_set_uniform_1i (filtersobel->vconv, "tex", 1);
-  gst_gl_shader_set_uniform_1fv (filtersobel->vconv, "kernel", 3, grad_kern);
-  gst_gl_shader_set_uniform_1f (filtersobel->vconv, "offset", 0.5);
+  gst_gl_shader_set_uniform_1i (filtersobel->hconv, "tex", 1);
 
   gst_gl_filtersobel_draw_texture (filtersobel, texture);
 }
 
 static void
-gst_gl_filtersobel_step_three (gint width, gint height, guint texture,
+gst_gl_filtersobel_vconv (gint width, gint height, guint texture,
     gpointer stuff)
 {
   GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff);
@@ -328,37 +315,12 @@ gst_gl_filtersobel_step_three (gint width, gint height, guint texture,
   glDisable (GL_TEXTURE_RECTANGLE_ARB);
 
   gst_gl_shader_set_uniform_1i (filtersobel->vconv, "tex", 1);
-  gst_gl_shader_set_uniform_1fv (filtersobel->vconv, "kernel", 3, blur_kern);
-  gst_gl_shader_set_uniform_1f (filtersobel->vconv, "offset", 0.0);
 
   gst_gl_filtersobel_draw_texture (filtersobel, texture);
 }
 
 static void
-gst_gl_filtersobel_step_four (gint width, gint height, guint texture,
-    gpointer stuff)
-{
-  GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff);
-
-  glMatrixMode (GL_PROJECTION);
-  glLoadIdentity ();
-
-  gst_gl_shader_use (filtersobel->hconv);
-
-  glActiveTexture (GL_TEXTURE1);
-  glEnable (GL_TEXTURE_RECTANGLE_ARB);
-  glBindTexture (GL_TEXTURE_RECTANGLE_ARB, texture);
-  glDisable (GL_TEXTURE_RECTANGLE_ARB);
-
-  gst_gl_shader_set_uniform_1i (filtersobel->hconv, "tex", 1);
-  gst_gl_shader_set_uniform_1fv (filtersobel->hconv, "kernel", 3, grad_kern);
-  gst_gl_shader_set_uniform_1f (filtersobel->vconv, "offset", 0.5);
-
-  gst_gl_filtersobel_draw_texture (filtersobel, texture);
-}
-
-static void
-gst_gl_filtersobel_step_five (gint width, gint height, guint texture,
+gst_gl_filtersobel_length (gint width, gint height, guint texture,
     gpointer stuff)
 {
   GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff);
@@ -370,18 +332,12 @@ gst_gl_filtersobel_step_five (gint width, gint height, guint texture,
 
   glActiveTexture (GL_TEXTURE1);
   glEnable (GL_TEXTURE_RECTANGLE_ARB);
-  glBindTexture (GL_TEXTURE_RECTANGLE_ARB, filtersobel->midtexture[1]);
-  glDisable (GL_TEXTURE_RECTANGLE_ARB);
-
-  gst_gl_shader_set_uniform_1i (filtersobel->len, "gx", 1);
-
-  glActiveTexture (GL_TEXTURE2);
-  glEnable (GL_TEXTURE_RECTANGLE_ARB);
-  glBindTexture (GL_TEXTURE_RECTANGLE_ARB, filtersobel->midtexture[3]);
+  glBindTexture (GL_TEXTURE_RECTANGLE_ARB, texture);
   glDisable (GL_TEXTURE_RECTANGLE_ARB);
 
-  gst_gl_shader_set_uniform_1i (filtersobel->len, "gy", 2);
-
+  gst_gl_shader_set_uniform_1i (filtersobel->len, "tex", 1);
+  gst_gl_shader_set_uniform_1i (filtersobel->len, "invert",
+      filtersobel->invert);
 
   gst_gl_filtersobel_draw_texture (filtersobel, texture);
 }
diff --git a/gst/gl/gstglfiltersobel.h b/gst/gl/gstglfiltersobel.h
index 0d614f3..faac6ac 100644
--- a/gst/gl/gstglfiltersobel.h
+++ b/gst/gl/gstglfiltersobel.h
@@ -39,6 +39,7 @@ struct _GstGLFilterSobel
   GstGLShader *hconv;
   GstGLShader *vconv;
   GstGLShader *len;
+  GstGLShader *desat;
 
   GLuint midtexture[5];