glamor: Use GL_RED instead of GL_ALPHA if we have texture_swizzle (v3)

GL_RED is supported by core profiles while GL_ALPHA is not; use GL_RED
for one channel objects (depth 1 to 8), and then swizzle them into the
alpha channel when used as a mask.

[airlied: updated to master, add swizzle to composited glyphs and xv paths]

v2: consolidate setting swizzle into the texture creation code, it
    should work fine there. Handle swizzle when setting color as well.
v3: Fix drawing to a8 with Render (changes by anholt, reviewed by airlied).

Signed-off-by: Keith Packard <keithp@keithp.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>
Reviewed-by: Eric Anholt <eric@anholt.net>

8 files changed, 118 insertions, 22 deletions

diff --git a/glamor/glamor.c b/glamor/glamor.c
index d4f3be2dd..25967b698 100644
--- a/glamor/glamor.c
+++ b/glamor/glamor.c
@@ -599,6 +599,10 @@ glamor_init(ScreenPtr screen, unsigned int flags)
     glamor_priv->max_fbo_size = MAX_FBO_SIZE;
 #endif
 
+    glamor_priv->one_channel_format = GL_ALPHA;
+    if (epoxy_has_gl_extension("GL_ARB_texture_rg") && epoxy_has_gl_extension("GL_ARB_texture_swizzle"))
+        glamor_priv->one_channel_format = GL_RED;
+
     glamor_set_debug_level(&glamor_debug_level);
 
     glamor_priv->saved_procs.create_screen_resources =
diff --git a/glamor/glamor_fbo.c b/glamor/glamor_fbo.c
index b1b584d21..5bfffe501 100644
--- a/glamor/glamor_fbo.c
+++ b/glamor/glamor_fbo.c
@@ -75,6 +75,8 @@ cache_format(GLenum format)
 {
     switch (format) {
     case GL_ALPHA:
+    case GL_LUMINANCE:
+    case GL_RED:
         return 2;
     case GL_RGB:
         return 1;
@@ -338,6 +340,8 @@ _glamor_create_tex(glamor_screen_private *glamor_priv,
     glBindTexture(GL_TEXTURE_2D, tex);
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+    if (format == glamor_priv->one_channel_format && format == GL_RED)
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_A, GL_RED);
     glamor_priv->suppress_gl_out_of_memory_logging = true;
     glTexImage2D(GL_TEXTURE_2D, 0, format, w, h, 0,
                  format, GL_UNSIGNED_BYTE, NULL);
diff --git a/glamor/glamor_picture.c b/glamor/glamor_picture.c
index 352858fac..b069ce56b 100644
--- a/glamor/glamor_picture.c
+++ b/glamor/glamor_picture.c
@@ -41,19 +41,21 @@
  * Return 0 if find a matched texture type. Otherwise return -1.
  **/
 static int
-glamor_get_tex_format_type_from_pictformat_gl(PictFormatShort format,
+glamor_get_tex_format_type_from_pictformat_gl(ScreenPtr pScreen,
+                                              PictFormatShort format,
                                               GLenum *tex_format,
                                               GLenum *tex_type,
                                               int *no_alpha,
                                               int *revert,
                                               int *swap_rb, int is_upload)
 {
+    glamor_screen_private *glamor_priv = glamor_get_screen_private(pScreen);
     *no_alpha = 0;
     *revert = REVERT_NONE;
     *swap_rb = is_upload ? SWAP_NONE_UPLOADING : SWAP_NONE_DOWNLOADING;
     switch (format) {
     case PICT_a1:
-        *tex_format = GL_ALPHA;
+        *tex_format = glamor_priv->one_channel_format;
         *tex_type = GL_UNSIGNED_BYTE;
         *revert = is_upload ? REVERT_UPLOADING_A1 : REVERT_DOWNLOADING_A1;
         break;
@@ -111,7 +113,7 @@ glamor_get_tex_format_type_from_pictformat_gl(PictFormatShort format,
         *tex_type = GL_UNSIGNED_SHORT_1_5_5_5_REV;
         break;
     case PICT_a8:
-        *tex_format = GL_ALPHA;
+        *tex_format = glamor_priv->one_channel_format;
         *tex_type = GL_UNSIGNED_BYTE;
         break;
     case PICT_x4r4g4b4:
@@ -137,13 +139,15 @@ glamor_get_tex_format_type_from_pictformat_gl(PictFormatShort format,
 #define IS_LITTLE_ENDIAN  (IMAGE_BYTE_ORDER == LSBFirst)
 
 static int
-glamor_get_tex_format_type_from_pictformat_gles2(PictFormatShort format,
+glamor_get_tex_format_type_from_pictformat_gles2(ScreenPtr pScreen,
+                                                 PictFormatShort format,
                                                  GLenum *tex_format,
                                                  GLenum *tex_type,
                                                  int *no_alpha,
                                                  int *revert,
                                                  int *swap_rb, int is_upload)
 {
+    glamor_screen_private *glamor_priv = glamor_get_screen_private(pScreen);
     int need_swap_rb = 0;
 
     *no_alpha = 0;
@@ -264,13 +268,13 @@ glamor_get_tex_format_type_from_pictformat_gles2(PictFormatShort format,
         break;
 
     case PICT_a1:
-        *tex_format = GL_ALPHA;
+        *tex_format = glamor_priv->one_channel_format;
         *tex_type = GL_UNSIGNED_BYTE;
         *revert = is_upload ? REVERT_UPLOADING_A1 : REVERT_DOWNLOADING_A1;
         break;
 
     case PICT_a8:
-        *tex_format = GL_ALPHA;
+        *tex_format = glamor_priv->one_channel_format;
         *tex_type = GL_UNSIGNED_BYTE;
         *revert = REVERT_NONE;
         break;
@@ -317,14 +321,16 @@ glamor_get_tex_format_type_from_pixmap(PixmapPtr pixmap,
         glamor_get_screen_private(pixmap->drawable.pScreen);
 
     if (glamor_priv->gl_flavor == GLAMOR_GL_DESKTOP) {
-        return glamor_get_tex_format_type_from_pictformat_gl(pict_format,
+        return glamor_get_tex_format_type_from_pictformat_gl(pixmap->drawable.pScreen,
+                                                             pict_format,
                                                              format, type,
                                                              no_alpha,
                                                              revert,
                                                              swap_rb,
                                                              is_upload);
     } else {
-        return glamor_get_tex_format_type_from_pictformat_gles2(pict_format,
+        return glamor_get_tex_format_type_from_pictformat_gles2(pixmap->drawable.pScreen,
+                                                                pict_format,
                                                                 format, type,
                                                                 no_alpha,
                                                                 revert,
diff --git a/glamor/glamor_priv.h b/glamor/glamor_priv.h
index e49aee5ab..d2ef2664a 100644
--- a/glamor/glamor_priv.h
+++ b/glamor/glamor_priv.h
@@ -116,10 +116,17 @@ enum shader_in {
     SHADER_IN_COUNT,
 };
 
+enum shader_dest_swizzle {
+    SHADER_DEST_SWIZZLE_DEFAULT,
+    SHADER_DEST_SWIZZLE_ALPHA_TO_RED,
+    SHADER_DEST_SWIZZLE_COUNT,
+};
+
 struct shader_key {
     enum shader_source source;
     enum shader_mask mask;
     enum shader_in in;
+    enum shader_dest_swizzle dest_swizzle;
 };
 
 struct blendinfo {
@@ -212,6 +219,8 @@ typedef struct glamor_screen_private {
     Bool has_dual_blend;
     int max_fbo_size;
 
+    GLuint one_channel_format;
+
     struct xorg_list
         fbo_cache[CACHE_FORMAT_COUNT][CACHE_BUCKET_WCOUNT][CACHE_BUCKET_HCOUNT];
     unsigned long fbo_cache_watermark;
@@ -282,7 +291,8 @@ typedef struct glamor_screen_private {
     int render_nr_quads;
     glamor_composite_shader composite_shader[SHADER_SOURCE_COUNT]
         [SHADER_MASK_COUNT]
-        [SHADER_IN_COUNT];
+        [SHADER_IN_COUNT]
+        [SHADER_DEST_SWIZZLE_COUNT];
 
     /* shaders to restore a texture to another texture. */
     GLint finish_access_prog[2];
diff --git a/glamor/glamor_render.c b/glamor/glamor_render.c
index 4e66f6d7e..5712cf841 100644
--- a/glamor/glamor_render.c
+++ b/glamor/glamor_render.c
@@ -177,33 +177,46 @@ glamor_create_composite_fs(struct shader_key *key)
         "		return rel_sampler(mask_sampler, mask_texture,\n"
         "				   mask_wh, mask_repeat_mode, 1);\n"
         "}\n";
+
+    const char *dest_swizzle_default =
+        "vec4 dest_swizzle(vec4 color)\n"
+        "{"
+        "	return color;"
+        "}";
+    const char *dest_swizzle_alpha_to_red =
+        "vec4 dest_swizzle(vec4 color)\n"
+        "{"
+        "	float undef;\n"
+        "	return vec4(color.a, undef, undef, undef);"
+        "}";
+
     const char *in_source_only =
         "void main()\n"
         "{\n"
-        "	gl_FragColor = get_source();\n"
+        "	gl_FragColor = dest_swizzle(get_source());\n"
         "}\n";
     const char *in_normal =
         "void main()\n"
         "{\n"
-        "	gl_FragColor = get_source() * get_mask().a;\n"
+        "	gl_FragColor = dest_swizzle(get_source() * get_mask().a);\n"
         "}\n";
     const char *in_ca_source =
         "void main()\n"
         "{\n"
-        "	gl_FragColor = get_source() * get_mask();\n"
+        "	gl_FragColor = dest_swizzle(get_source() * get_mask());\n"
         "}\n";
     const char *in_ca_alpha =
         "void main()\n"
         "{\n"
-        "	gl_FragColor = get_source().a * get_mask();\n"
+        "	gl_FragColor = dest_swizzle(get_source().a * get_mask());\n"
         "}\n";
     const char *in_ca_dual_blend =
         "out vec4 color0;\n"
         "out vec4 color1;\n"
         "void main()\n"
         "{\n"
-        "	color0 = get_source() * get_mask();\n"
-        "	color1 = get_source().a * get_mask();\n"
+        "	color0 = dest_swizzle(get_source() * get_mask());\n"
+        "	color1 = dest_swizzle(get_source().a * get_mask());\n"
         "}\n";
     const char *header_ca_dual_blend =
         "#version 130\n";
@@ -214,6 +227,7 @@ glamor_create_composite_fs(struct shader_key *key)
     const char *in;
     const char *header;
     const char *header_norm = "";
+    const char *dest_swizzle;
     GLuint prog;
 
     switch (key->source) {
@@ -246,6 +260,21 @@ glamor_create_composite_fs(struct shader_key *key)
         FatalError("Bad composite shader mask");
     }
 
+    /* If we're storing to an a8 texture but our texture format is
+     * GL_RED because of a core context, then we need to make sure to
+     * put the alpha into the red channel.
+     */
+    switch (key->dest_swizzle) {
+    case SHADER_DEST_SWIZZLE_DEFAULT:
+        dest_swizzle = dest_swizzle_default;
+        break;
+    case SHADER_DEST_SWIZZLE_ALPHA_TO_RED:
+        dest_swizzle = dest_swizzle_alpha_to_red;
+        break;
+    default:
+        FatalError("Bad composite shader dest swizzle");
+    }
+
     header = header_norm;
     switch (key->in) {
     case SHADER_IN_SOURCE_ONLY:
@@ -271,8 +300,8 @@ glamor_create_composite_fs(struct shader_key *key)
     XNFasprintf(&source,
                 "%s"
                 GLAMOR_DEFAULT_PRECISION
-                "%s%s%s%s%s%s", header, repeat_define, relocate_texture,
-                rel_sampler, source_fetch, mask_fetch, in);
+                "%s%s%s%s%s%s%s", header, repeat_define, relocate_texture,
+                rel_sampler, source_fetch, mask_fetch, dest_swizzle, in);
 
     prog = glamor_compile_glsl_prog(GL_FRAGMENT_SHADER, source);
     free(source);
@@ -386,18 +415,36 @@ glamor_lookup_composite_shader(ScreenPtr screen, struct
     glamor_screen_private *glamor_priv = glamor_get_screen_private(screen);
     glamor_composite_shader *shader;
 
-    shader = &glamor_priv->composite_shader[key->source][key->mask][key->in];
+    shader = &glamor_priv->composite_shader[key->source][key->mask][key->in][key->dest_swizzle];
     if (shader->prog == 0)
         glamor_create_composite_shader(screen, key, shader);
 
     return shader;
 }
 
+static GLenum
+glamor_translate_blend_alpha_to_red(GLenum blend)
+{
+    switch (blend) {
+    case GL_SRC_ALPHA:
+        return GL_SRC_COLOR;
+    case GL_DST_ALPHA:
+        return GL_DST_COLOR;
+    case GL_ONE_MINUS_SRC_ALPHA:
+        return GL_ONE_MINUS_SRC_COLOR;
+    case GL_ONE_MINUS_DST_ALPHA:
+        return GL_ONE_MINUS_DST_COLOR;
+    default:
+        return blend;
+    }
+}
+
 static Bool
 glamor_set_composite_op(ScreenPtr screen,
                         CARD8 op, struct blendinfo *op_info_result,
                         PicturePtr dest, PicturePtr mask,
-                        enum ca_state ca_state)
+                        enum ca_state ca_state,
+                        struct shader_key *key)
 {
     GLenum source_blend, dest_blend;
     struct blendinfo *op_info;
@@ -444,6 +491,14 @@ glamor_set_composite_op(ScreenPtr screen,
         }
     }
 
+    /* If we're outputting our alpha to the red channel, then any
+     * reads of alpha for blending need to come from the red channel.
+     */
+    if (key->dest_swizzle == SHADER_DEST_SWIZZLE_ALPHA_TO_RED) {
+        source_blend = glamor_translate_blend_alpha_to_red(source_blend);
+        dest_blend = glamor_translate_blend_alpha_to_red(dest_blend);
+    }
+
     op_info_result->source_blend = source_blend;
     op_info_result->dest_blend = dest_blend;
     op_info_result->source_alpha = op_info->source_alpha;
@@ -841,6 +896,13 @@ glamor_composite_choose_shader(CARD8 op,
         key.in = SHADER_IN_SOURCE_ONLY;
     }
 
+    if (dest_pixmap->drawable.bitsPerPixel <= 8 &&
+        glamor_priv->one_channel_format == GL_RED) {
+        key.dest_swizzle = SHADER_DEST_SWIZZLE_ALPHA_TO_RED;
+    } else {
+        key.dest_swizzle = SHADER_DEST_SWIZZLE_DEFAULT;
+    }
+
     if (source && source->alphaMap) {
         glamor_fallback("source alphaMap\n");
         goto fail;
@@ -970,8 +1032,10 @@ glamor_composite_choose_shader(CARD8 op,
         goto fail;
     }
 
-    if (!glamor_set_composite_op(screen, op, op_info, dest, mask, ca_state))
+    if (!glamor_set_composite_op(screen, op, op_info, dest, mask, ca_state,
+                                 &key)) {
         goto fail;
+    }
 
     *shader = glamor_lookup_composite_shader(screen, &key);
     if ((*shader)->prog == 0) {
diff --git a/glamor/glamor_transfer.c b/glamor/glamor_transfer.c
index 155d7e0fa..91e174734 100644
--- a/glamor/glamor_transfer.c
+++ b/glamor/glamor_transfer.c
@@ -42,7 +42,7 @@ glamor_format_for_pixmap(PixmapPtr pixmap, GLenum *format, GLenum *type)
         *type = GL_UNSIGNED_SHORT_1_5_5_5_REV;
         break;
     case 8:
-        *format = GL_ALPHA;
+        *format = glamor_get_screen_private(pixmap->drawable.pScreen)->one_channel_format;
         *type = GL_UNSIGNED_BYTE;
         break;
     default:
diff --git a/glamor/glamor_transform.c b/glamor/glamor_transform.c
index f476a99b7..ad069435f 100644
--- a/glamor/glamor_transform.c
+++ b/glamor/glamor_transform.c
@@ -111,12 +111,18 @@ glamor_set_color(PixmapPtr      pixmap,
                  CARD32         pixel,
                  GLint          uniform)
 {
+    glamor_screen_private *glamor_priv =
+        glamor_get_screen_private((pixmap)->drawable.pScreen);
     float       color[4];
 
     glamor_get_rgba_from_pixel(pixel,
                                &color[0], &color[1], &color[2], &color[3],
                                format_for_pixmap(pixmap));
 
+    if ((pixmap->drawable.depth == 1 || pixmap->drawable.depth == 8) &&
+	glamor_priv->one_channel_format == GL_RED)
+      color[0] = color[3];
+
     glUniform4fv(uniform, 1, color);
 }
 
diff --git a/glamor/glamor_utils.h b/glamor/glamor_utils.h
index e648af2de..d4366c146 100644
--- a/glamor/glamor_utils.h
+++ b/glamor/glamor_utils.h
@@ -757,7 +757,7 @@ gl_iformat_for_pixmap(PixmapPtr pixmap)
 
     if (glamor_priv->gl_flavor == GLAMOR_GL_DESKTOP &&
         ((pixmap)->drawable.depth == 1 || (pixmap)->drawable.depth == 8)) {
-        return GL_ALPHA;
+        return glamor_priv->one_channel_format;
     } else {
         return GL_RGBA;
     }
@@ -867,6 +867,8 @@ glamor_pict_format_is_compatible(PicturePtr picture)
         return (picture->format == PICT_a8r8g8b8 ||
                 picture->format == PICT_x8r8g8b8);
     case GL_ALPHA:
+    case GL_RED:
+    case GL_LUMINANCE:
         return (picture->format == PICT_a8);
     default:
         return FALSE;