Add an option to use alpha channel in multiwindow mode

Add an option to turn on the use of the X windows alpha channel in
multiwindow mode.  This works on W7/Vista (using
DwmEnableBlurBehindWindow()), and Windows 10 (using the undocumented
SetWindowCompositionAttribute()), but not on Windows 8/8.1

-compositewm must be enabled for this to be useful, as the X window only
uses a pixmap with an alpha channel when the Composite extensions is
enabled.

v2:
Update meson.build

Future work:

A window property to control use of alpha?
Option to turn off blur on W7/Vista
Implement _NET_WM_WINDOW_OPACITY

8 files changed, 158 insertions, 3 deletions

diff --git a/hw/xwin/InitOutput.c b/hw/xwin/InitOutput.c
index 82082afec..39e5a2cf7 100644
--- a/hw/xwin/InitOutput.c
+++ b/hw/xwin/InitOutput.c
@@ -740,6 +740,8 @@ winUseMsg(void)
            "\tthe updated region when num_boxes, or more, are in the\n"
            "\tupdated region.\n");
 
+    ErrorF("-[no]compositealpha\n"
+           "\tX windows with per-pixel alpha are composited into the Windows desktop.\n");
     ErrorF("-[no]compositewm\n"
            "\tUse the Composite extension to keep a bitmap image of each top-level\n"
            "\tX window, so window contents which are occluded show correctly in\n"
@@ -829,7 +831,7 @@ winUseMsg(void)
            "\tSpecify an optional refresh rate to use in fullscreen mode\n"
            "\twith a DirectDraw engine.\n");
 
-    ErrorF("-resize=none|scrollbars|randr"
+    ErrorF("-resize=none|scrollbars|randr\n"
            "\tIn windowed mode, [don't] allow resizing of the window. 'scrollbars'\n"
            "\tmode gives the window scrollbars as needed, 'randr' mode uses the RANR\n"
            "\textension to resize the X screen.  'randr' is the default.\n");
diff --git a/hw/xwin/Makefile.am b/hw/xwin/Makefile.am
index faf7a8f73..5e1637ab1 100644
--- a/hw/xwin/Makefile.am
+++ b/hw/xwin/Makefile.am
@@ -27,7 +27,7 @@ SRCS_MULTIWINDOW = \
 	winmultiwindowwndproc.c \
 	propertystore.h \
 	winSetAppUserModelID.c
-MULTIWINDOW_SYS_LIBS = -lshlwapi -lole32
+MULTIWINDOW_SYS_LIBS = -lshlwapi -lole32 -ldwmapi
 MULTIWINDOW_LIBS = $(top_builddir)/hw/xwin/wmutil/libXWinWMUtil.la
 
 SRCS_RANDR = \
diff --git a/hw/xwin/man/XWin.man b/hw/xwin/man/XWin.man
index 2ed7217d4..050b4d1b4 100644
--- a/hw/xwin/man/XWin.man
+++ b/hw/xwin/man/XWin.man
@@ -180,6 +180,15 @@ Use Composite extension redirection to maintain a bitmap image of each top-level
 X window, so window contents which are occluded show correctly in task bar and
 task switcher previews.
 The default is enabled.
+.TP 8
+.B \-[no]compositealpha
+X windows with per-pixel alpha are composited into the \fIWindows\fP desktop
+(i.e. a \fIWindows\fP window can be seen through any transparency in an X window
+placed over it).
+
+This option has no effect on Windows 8 and 8.1.
+This option has no effect if \fB-compositewm\fP is disabled.
+The default is disabled.
 
 .SH OPTIONS CONTROLLING WINDOWS INTEGRATION
 .TP 8
diff --git a/hw/xwin/meson.build b/hw/xwin/meson.build
index 280af9414..b6bf6a6f7 100644
--- a/hw/xwin/meson.build
+++ b/hw/xwin/meson.build
@@ -44,7 +44,7 @@ srcs_windows += [
      'propertystore.h',
      'winSetAppUserModelID.c',
 ]
-xwin_sys_libs += ['-lshlwapi', '-lole32']
+xwin_sys_libs += ['-lshlwapi', '-lole32', '-ldwmapi']
 
 srcs_windows += [
      'winrandr.c',
diff --git a/hw/xwin/winglobals.c b/hw/xwin/winglobals.c
index a97db959b..b0086f871 100644
--- a/hw/xwin/winglobals.c
+++ b/hw/xwin/winglobals.c
@@ -59,6 +59,7 @@ HWND g_hDlgAbout = NULL;
 const char *g_pszQueryHost = NULL;
 Bool g_fXdmcpEnabled = FALSE;
 Bool g_fAuthEnabled = FALSE;
+Bool g_fCompositeAlpha = FALSE;
 HICON g_hIconX = NULL;
 HICON g_hSmallIconX = NULL;
 
diff --git a/hw/xwin/winglobals.h b/hw/xwin/winglobals.h
index af8c7f896..fee507683 100644
--- a/hw/xwin/winglobals.h
+++ b/hw/xwin/winglobals.h
@@ -47,6 +47,7 @@ extern int g_iLogVerbose;
 
 extern Bool g_fAuthEnabled;
 extern Bool g_fXdmcpEnabled;
+extern Bool g_fCompositeAlpha;
 
 extern Bool g_fNoHelpMessageBox;
 extern Bool g_fSilentDupError;
diff --git a/hw/xwin/winmultiwindowwndproc.c b/hw/xwin/winmultiwindowwndproc.c
index c58de0ef3..ee3d1637e 100644
--- a/hw/xwin/winmultiwindowwndproc.c
+++ b/hw/xwin/winmultiwindowwndproc.c
@@ -35,6 +35,7 @@
 #ifdef HAVE_XWIN_CONFIG_H
 #include <xwin-config.h>
 #endif
+
 #include "win.h"
 #include "dixevents.h"
 #include "winmultiwindowclass.h"
@@ -42,6 +43,11 @@
 #include "winmsg.h"
 #include "inputstr.h"
 #include "wmutil/keyboard.h"
+#include <dwmapi.h>
+
+#ifndef WM_DWMCOMPOSITIONCHANGED
+#define WM_DWMCOMPOSITIONCHANGED 0x031e
+#endif
 
 extern void winUpdateWindowPosition(HWND hWnd, HWND * zstyle);
 
@@ -324,6 +330,115 @@ winAdjustXWindowState(winPrivScreenPtr s_pScreenPriv, winWMMessageRec *wmMsg)
     }
 }
 
+/* Undocumented */
+typedef struct _ACCENTPOLICY
+{
+    ULONG AccentState;
+    ULONG AccentFlags;
+    ULONG GradientColor;
+    ULONG AnimationId;
+} ACCENTPOLICY;
+
+#define ACCENT_ENABLE_BLURBEHIND 3
+
+typedef struct _WINCOMPATTR
+{
+    DWORD attribute;
+    PVOID pData;
+    ULONG dataSize;
+} WINCOMPATTR;
+
+#define WCA_ACCENT_POLICY 19
+
+typedef WINBOOL WINAPI (*PFNSETWINDOWCOMPOSITIONATTRIBUTE)(HWND, WINCOMPATTR *);
+
+static void
+CheckForAlpha(HWND hWnd, WindowPtr pWin, winScreenInfo *pScreenInfo)
+{
+    /* Check (once) which API we should use */
+    static Bool doOnce = TRUE;
+    static PFNSETWINDOWCOMPOSITIONATTRIBUTE pSetWindowCompositionAttribute = NULL;
+    static Bool useDwmEnableBlurBehindWindow = FALSE;
+
+    if (doOnce)
+        {
+            /* XXX: when mingw-w64-headers 5.0 is available, just include
+               versionhelper.h and use the appropriate macros here */
+            OSVERSIONINFOEX osvi = {0};
+            osvi.dwOSVersionInfoSize = sizeof(osvi);
+            GetVersionEx((LPOSVERSIONINFO)&osvi);
+
+            /* SetWindowCompositionAttribute() exists on Windows 7 and later,
+               but doesn't work for this purpose, so first check for Windows 10
+               or later */
+            if (osvi.dwMajorVersion >= 10)
+                {
+                    HMODULE hUser32 = GetModuleHandle("user32");
+
+                    if (hUser32)
+                        pSetWindowCompositionAttribute = (PFNSETWINDOWCOMPOSITIONATTRIBUTE) GetProcAddress(hUser32, "SetWindowCompositionAttribute");
+                    winDebug("SetWindowCompositionAttribute %s\n", pSetWindowCompositionAttribute ? "found" : "not found");
+                }
+            /* On Windows 7 and Windows Vista, use DwmEnableBlurBehindWindow() */
+            else if ((osvi.dwMajorVersion == 6) && (osvi.dwMinorVersion <= 1))
+                {
+                    useDwmEnableBlurBehindWindow = TRUE;
+                }
+            /* On Windows 8 and Windows 8.1, using the alpha channel on those
+               seems near impossible, so we don't do anything. */
+
+            doOnce = FALSE;
+        }
+
+    /* alpha-channel use is wanted */
+    if (!g_fCompositeAlpha || !pScreenInfo->fCompositeWM)
+        return;
+
+    /* Image has alpha ... */
+    if (pWin->drawable.depth != 32)
+        return;
+
+    /* ... and we can do something useful with it? */
+    if (pSetWindowCompositionAttribute)
+        {
+            WINBOOL rc;
+            /* Use the (undocumented) SetWindowCompositionAttribute, if
+               available, to turn on alpha channel use on Windows 10. */
+            ACCENTPOLICY policy = { ACCENT_ENABLE_BLURBEHIND, 0, 0, 0 } ;
+            WINCOMPATTR data = { WCA_ACCENT_POLICY,  &policy, sizeof(ACCENTPOLICY) };
+
+            /* This turns on DWM looking at the alpha-channel of this window */
+            winDebug("enabling alpha for XID %08x hWnd %p, using SetWindowCompositionAttribute()\n", (unsigned int)pWin->drawable.id, hWnd);
+            rc = pSetWindowCompositionAttribute(hWnd, &data);
+            if (!rc)
+                ErrorF("SetWindowCompositionAttribute failed: %d\n", (int)GetLastError());
+        }
+    else if (useDwmEnableBlurBehindWindow)
+        {
+            HRESULT rc;
+            WINBOOL enabled;
+
+            rc = DwmIsCompositionEnabled(&enabled);
+            if ((rc == S_OK) && enabled)
+                {
+                    /* Use DwmEnableBlurBehindWindow, to turn on alpha channel
+                       use on Windows Vista and Windows 7 */
+                    DWM_BLURBEHIND bbh;
+                    bbh.dwFlags = DWM_BB_ENABLE | DWM_BB_BLURREGION | DWM_BB_TRANSITIONONMAXIMIZED;
+                    bbh.fEnable = TRUE;
+                    bbh.hRgnBlur = NULL;
+                    bbh.fTransitionOnMaximized = TRUE; /* What does this do ??? */
+
+                    /* This terribly-named function actually controls if DWM
+                       looks at the alpha channel of this window */
+                    winDebug("enabling alpha for XID %08x hWnd %p, using DwmEnableBlurBehindWindow()\n", (unsigned int)pWin->drawable.id, hWnd);
+                    rc = DwmEnableBlurBehindWindow(hWnd, &bbh);
+                    if (rc != S_OK)
+                        ErrorF("DwmEnableBlurBehindWindow failed: %x, %d\n", (int)rc, (int)GetLastError());
+                }
+        }
+}
+
 /*
  * winTopLevelWindowProc - Window procedure for all top-level Windows windows.
  */
@@ -451,6 +566,8 @@ winTopLevelWindowProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
         if (fWMMsgInitialized)
             winSendMessageToWM(s_pScreenPriv->pWMInfo, &wmMsg);
 
+        CheckForAlpha(hwnd, pWin, s_pScreenInfo);
+
         return 0;
 
     case WM_INIT_SYS_MENU:
@@ -1152,6 +1269,12 @@ winTopLevelWindowProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
         winAdjustWindowsWindow(pWin, hwnd);
         break;
 
+    case WM_DWMCOMPOSITIONCHANGED:
+        /* This message is only sent on Vista/W7 */
+        CheckForAlpha(hwnd, pWin, s_pScreenInfo);
+
+        return 0;
+
     default:
         break;
     }
diff --git a/hw/xwin/winprocarg.c b/hw/xwin/winprocarg.c
index b01302eb9..d5ccdf01b 100644
--- a/hw/xwin/winprocarg.c
+++ b/hw/xwin/winprocarg.c
@@ -603,6 +603,25 @@ ddxProcessArgument(int argc, char *argv[], int i)
     }
 
     /*
+     * Look for the '-compositealpha' argument
+     */
+    if (IS_OPTION("-compositealpha")) {
+        g_fCompositeAlpha = TRUE;
+
+        /* Indicate that we have processed this argument */
+        return 1;
+    }
+    /*
+     * Look for the '-nocompositealpha' argument
+     */
+    if (IS_OPTION("-nocompositealpha")) {
+        g_fCompositeAlpha  = FALSE;
+
+        /* Indicate that we have processed this argument */
+        return 1;
+    }
+
+    /*
      * Look for the '-multiplemonitors' argument
      */
     if (IS_OPTION("-multiplemonitors")