glamor: Don't override source alpha to 1.0 if it's used for blending

It caused an incorrect result of the blend operation.

Use glColorMask to prevent non-1.0 alpha channel values in a depth 32
pixmap backing an effective depth 24 window. For blending operations,
the expectation is that the destination drawable contains valid pixel
values, so the alpha channel should already be 1.0.

Fixes: d1f142891ef3 ("glamor: Ignore destination alpha as necessary for composite operation")
Issue: https://gitlab.gnome.org/GNOME/mutter/-/issues/3104
(cherry picked from commit d1bbf82d72566e16e90800c77cf70fe5fc4ef2e8)

1 files changed, 27 insertions, 1 deletions

diff --git a/glamor/glamor_render.c b/glamor/glamor_render.c
index 0d233f27b..60e1c64ac 100644
--- a/glamor/glamor_render.c
+++ b/glamor/glamor_render.c
@@ -835,6 +835,20 @@ glamor_render_format_is_supported(PicturePtr picture)
 }
 
 static Bool
+render_op_uses_src_alpha(CARD8 op)
+{
+    struct blendinfo *info = &composite_op_info[op];
+
+    switch (info->dest_blend) {
+    case GL_ONE_MINUS_SRC_ALPHA:
+    case GL_SRC_ALPHA:
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
+static Bool
 glamor_composite_choose_shader(CARD8 op,
                                PicturePtr source,
                                PicturePtr mask,
@@ -947,7 +961,8 @@ glamor_composite_choose_shader(CARD8 op,
         key.dest_swizzle = SHADER_DEST_SWIZZLE_ALPHA_TO_RED;
     } else {
         if (dest_pixmap->drawable.depth == 32 &&
-            glamor_drawable_effective_depth(dest->pDrawable) == 24)
+            glamor_drawable_effective_depth(dest->pDrawable) == 24 &&
+            !render_op_uses_src_alpha(op))
             key.dest_swizzle = SHADER_DEST_SWIZZLE_IGNORE_ALPHA;
         else
             key.dest_swizzle = SHADER_DEST_SWIZZLE_DEFAULT;
@@ -1181,6 +1196,7 @@ glamor_composite_with_shader(CARD8 op,
     Bool ret = FALSE;
     glamor_composite_shader *shader = NULL, *shader_ca = NULL;
     struct blendinfo op_info, op_info_ca;
+    Bool restore_colormask = FALSE;
 
     if (!glamor_composite_choose_shader(op, source, mask, dest,
                                         source_pixmap, mask_pixmap, dest_pixmap,
@@ -1205,6 +1221,14 @@ glamor_composite_with_shader(CARD8 op,
 
     glamor_make_current(glamor_priv);
 
+    if (ca_state != CA_TWO_PASS &&
+        key.dest_swizzle == SHADER_DEST_SWIZZLE_DEFAULT &&
+        dest_pixmap->drawable.depth == 32 &&
+        glamor_drawable_effective_depth(dest->pDrawable) == 24) {
+        glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_FALSE);
+        restore_colormask = TRUE;
+    }
+
     glamor_set_destination_pixmap_priv_nc(glamor_priv, dest_pixmap, dest_pixmap_priv);
     glamor_composite_set_shader_blend(glamor_priv, dest_pixmap_priv, &key, shader, &op_info);
     glamor_set_alu(screen, GXcopy);
@@ -1347,6 +1371,8 @@ glamor_composite_with_shader(CARD8 op,
 
     glDisable(GL_SCISSOR_TEST);
 disable_va:
+    if (restore_colormask)
+        glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
     glDisableVertexAttribArray(GLAMOR_VERTEX_POS);
     glDisableVertexAttribArray(GLAMOR_VERTEX_SOURCE);
     glDisableVertexAttribArray(GLAMOR_VERTEX_MASK);