hw/xwin: Add an option to use alpha channel in multiwindow mode

Add an option to turn on the use of the X window's alpha channel in
multiwindow mode, i.e. this uses the X window's alpha channel for
compositing into the native desktop.

This works on W7/Vista (using DwmEnableBlurBehindWindow()), and Windows
10 (using the undocumented SetWindowCompositionAttribute()), but not on
Windows 8/8.1

-compositewm must be enabled for this to be useful, as we only have a
pixmap with an alpha channel for the X window in that case.  The
framebuffer/root window doesn't have one (unless perhaps you are using
the rootless extension, maybe...).

v2:
Update meson.build

Future work:

A window property to control use of alpha?
Option to turn off blur on W7/Vista
Implement _NET_WM_WINDOW_OPACITY

9 files changed, 157 insertions, 4 deletions

diff --git a/doc/c-extensions b/doc/c-extensions
index e5fde3440..7dc58a215 100644
--- a/doc/c-extensions
+++ b/doc/c-extensions
@@ -62,7 +62,7 @@ TODO: Solaris.
 
 TODO: *BSD.
 
-Windows-dependent code assumes at least NT 5.1.
+Windows-dependent code assumes at least NT 6.0.
 
 OSX support is generally limited to the most recent version.  Currently
 that means 10.5.
diff --git a/hw/xwin/InitOutput.c b/hw/xwin/InitOutput.c
index 00053beb0..250051425 100644
--- a/hw/xwin/InitOutput.c
+++ b/hw/xwin/InitOutput.c
@@ -716,6 +716,8 @@ winUseMsg(void)
            "\tthe updated region when num_boxes, or more, are in the\n"
            "\tupdated region.\n");
 
+    ErrorF("-[no]compositealpha\n"
+           "\tX windows with per-pixel alpha are composited into the Windows desktop.\n");
     ErrorF("-[no]compositewm\n"
            "\tUse the Composite extension to keep a bitmap image of each top-level\n"
            "\tX window, so window contents which are occluded show correctly in\n"
@@ -803,7 +805,7 @@ winUseMsg(void)
            "\tSpecify an optional refresh rate to use in fullscreen mode\n"
            "\twith a DirectDraw engine.\n");
 
-    ErrorF("-resize=none|scrollbars|randr"
+    ErrorF("-resize=none|scrollbars|randr\n"
            "\tIn windowed mode, [don't] allow resizing of the window. 'scrollbars'\n"
            "\tmode gives the window scrollbars as needed, 'randr' mode uses the RANR\n"
            "\textension to resize the X screen.  'randr' is the default.\n");
diff --git a/hw/xwin/Makefile.am b/hw/xwin/Makefile.am
index 42fc9dd68..6ef89196a 100644
--- a/hw/xwin/Makefile.am
+++ b/hw/xwin/Makefile.am
@@ -27,7 +27,7 @@ SRCS_MULTIWINDOW = \
 	winmultiwindowwndproc.c \
 	propertystore.h \
 	winSetAppUserModelID.c
-MULTIWINDOW_SYS_LIBS = -lshlwapi -lole32
+MULTIWINDOW_SYS_LIBS = -lshlwapi -lole32 -ldwmapi
 
 SRCS_RANDR = \
 	winrandr.c
diff --git a/hw/xwin/man/XWin.man b/hw/xwin/man/XWin.man
index e61589e8e..f9305f592 100644
--- a/hw/xwin/man/XWin.man
+++ b/hw/xwin/man/XWin.man
@@ -176,6 +176,15 @@ Use Composite extension redirection to maintain a bitmap image of each top-level
 X window, so window contents which are occluded show correctly in task bar and
 task switcher previews.
 The default is enabled.
+.TP 8
+.B \-[no]compositealpha
+X windows with per-pixel alpha are composited into the \fIWindows\fP desktop
+(i.e. a \fIWindows\fP window can be seen through any transparency in an X window
+placed over it).
+
+This option has no effect on Windows 8 and 8.1.
+This option has no effect if \fB-compositewm\fP is disabled.
+The default is disabled.
 
 .SH OPTIONS CONTROLLING WINDOWS INTEGRATION
 .TP 8
diff --git a/hw/xwin/meson.build b/hw/xwin/meson.build
index 3462c396d..c1a2917b8 100644
--- a/hw/xwin/meson.build
+++ b/hw/xwin/meson.build
@@ -44,7 +44,7 @@ srcs_windows += [
      'propertystore.h',
      'winSetAppUserModelID.c',
 ]
-xwin_sys_libs += ['-lshlwapi', '-lole32']
+xwin_sys_libs += ['-lshlwapi', '-lole32', '-ldwmapi']
 
 srcs_windows += [
      'winrandr.c',
diff --git a/hw/xwin/winglobals.c b/hw/xwin/winglobals.c
index d1a0d4587..3212c6beb 100644
--- a/hw/xwin/winglobals.c
+++ b/hw/xwin/winglobals.c
@@ -59,6 +59,7 @@ HWND g_hDlgAbout = NULL;
 const char *g_pszQueryHost = NULL;
 Bool g_fXdmcpEnabled = FALSE;
 Bool g_fAuthEnabled = FALSE;
+Bool g_fCompositeAlpha = FALSE;
 HICON g_hIconX = NULL;
 HICON g_hSmallIconX = NULL;
 
diff --git a/hw/xwin/winglobals.h b/hw/xwin/winglobals.h
index 82fc1c5ec..77af6125a 100644
--- a/hw/xwin/winglobals.h
+++ b/hw/xwin/winglobals.h
@@ -50,6 +50,7 @@ extern Bool g_fLogInited;
 
 extern Bool g_fAuthEnabled;
 extern Bool g_fXdmcpEnabled;
+extern Bool g_fCompositeAlpha;
 
 extern Bool g_fNoHelpMessageBox;
 extern Bool g_fNativeGl;
diff --git a/hw/xwin/winmultiwindowwndproc.c b/hw/xwin/winmultiwindowwndproc.c
index 48e9f9dd8..587884d11 100644
--- a/hw/xwin/winmultiwindowwndproc.c
+++ b/hw/xwin/winmultiwindowwndproc.c
@@ -35,12 +35,18 @@
 #ifdef HAVE_XWIN_CONFIG_H
 #include <xwin-config.h>
 #endif
+
 #include "win.h"
 #include "dixevents.h"
 #include "winmultiwindowclass.h"
 #include "winprefs.h"
 #include "winmsg.h"
 #include "inputstr.h"
+#include <dwmapi.h>
+
+#ifndef WM_DWMCOMPOSITIONCHANGED
+#define WM_DWMCOMPOSITIONCHANGED 0x031e
+#endif
 
 extern void winUpdateWindowPosition(HWND hWnd, HWND * zstyle);
 
@@ -294,6 +300,113 @@ winStartMousePolling(winPrivScreenPtr s_pScreenPriv)
                                             MOUSE_POLLING_INTERVAL, NULL);
 }
 
+/* Undocumented */
+typedef struct _ACCENTPOLICY
+{
+    ULONG AccentState;
+    ULONG AccentFlags;
+    ULONG GradientColor;
+    ULONG AnimationId;
+} ACCENTPOLICY;
+
+#define ACCENT_ENABLE_BLURBEHIND 3
+
+typedef struct _WINCOMPATTR
+{
+    DWORD attribute;
+    PVOID pData;
+    ULONG dataSize;
+} WINCOMPATTR;
+
+#define WCA_ACCENT_POLICY 19
+
+typedef WINBOOL WINAPI (*PFNSETWINDOWCOMPOSITIONATTRIBUTE)(HWND, WINCOMPATTR *);
+
+static void
+CheckForAlpha(HWND hWnd, WindowPtr pWin, winScreenInfo *pScreenInfo)
+{
+    /* Check (once) which API we should use */
+    static Bool doOnce = TRUE;
+    static PFNSETWINDOWCOMPOSITIONATTRIBUTE pSetWindowCompositionAttribute = NULL;
+    static Bool useDwmEnableBlurBehindWindow = FALSE;
+
+    if (doOnce)
+        {
+            OSVERSIONINFOEX osvi = {0};
+            osvi.dwOSVersionInfoSize = sizeof(osvi);
+            GetVersionEx((LPOSVERSIONINFO)&osvi);
+
+            /* SetWindowCompositionAttribute() exists on Windows 7 and later,
+               but doesn't work for this purpose, so first check for Windows 10
+               or later */
+            if (osvi.dwMajorVersion >= 10)
+                {
+                    HMODULE hUser32 = GetModuleHandle("user32");
+
+                    if (hUser32)
+                        pSetWindowCompositionAttribute = (PFNSETWINDOWCOMPOSITIONATTRIBUTE) GetProcAddress(hUser32, "SetWindowCompositionAttribute");
+                    winDebug("SetWindowCompositionAttribute %s\n", pSetWindowCompositionAttribute ? "found" : "not found");
+                }
+            /* On Windows 7 and Windows Vista, use DwmEnableBlurBehindWindow() */
+            else if ((osvi.dwMajorVersion == 6) && (osvi.dwMinorVersion <= 1))
+                {
+                    useDwmEnableBlurBehindWindow = TRUE;
+                }
+            /* On Windows 8 and Windows 8.1, using the alpha channel on those
+               seems near impossible, so we don't do anything. */
+
+            doOnce = FALSE;
+        }
+
+    /* alpha-channel use is wanted */
+    if (!g_fCompositeAlpha || !pScreenInfo->fCompositeWM)
+        return;
+
+    /* Image has alpha ... */
+    if (pWin->drawable.depth != 32)
+        return;
+
+    /* ... and we can do something useful with it? */
+    if (pSetWindowCompositionAttribute)
+        {
+            WINBOOL rc;
+            /* Use the (undocumented) SetWindowCompositionAttribute, if
+               available, to turn on alpha channel use on Windows 10. */
+            ACCENTPOLICY policy = { ACCENT_ENABLE_BLURBEHIND, 0, 0, 0 } ;
+            WINCOMPATTR data = { WCA_ACCENT_POLICY,  &policy, sizeof(ACCENTPOLICY) };
+
+            /* This turns on DWM looking at the alpha-channel of this window */
+            winDebug("enabling alpha for XID %08x hWnd %p, using SetWindowCompositionAttribute()\n", (unsigned int)pWin->drawable.id, hWnd);
+            rc = pSetWindowCompositionAttribute(hWnd, &data);
+            if (!rc)
+                ErrorF("SetWindowCompositionAttribute failed: %d\n", (int)GetLastError());
+        }
+    else if (useDwmEnableBlurBehindWindow)
+        {
+            HRESULT rc;
+            WINBOOL enabled;
+
+            rc = DwmIsCompositionEnabled(&enabled);
+            if ((rc == S_OK) && enabled)
+                {
+                    /* Use DwmEnableBlurBehindWindow, to turn on alpha channel
+                       use on Windows Vista and Windows 7 */
+                    DWM_BLURBEHIND bbh;
+                    bbh.dwFlags = DWM_BB_ENABLE | DWM_BB_BLURREGION | DWM_BB_TRANSITIONONMAXIMIZED;
+                    bbh.fEnable = TRUE;
+                    bbh.hRgnBlur = NULL;
+                    bbh.fTransitionOnMaximized = TRUE; /* What does this do ??? */
+
+                    /* This terribly-named function actually controls if DWM
+                       looks at the alpha channel of this window */
+                    winDebug("enabling alpha for XID %08x hWnd %p, using DwmEnableBlurBehindWindow()\n", (unsigned int)pWin->drawable.id, hWnd);
+                    rc = DwmEnableBlurBehindWindow(hWnd, &bbh);
+                    if (rc != S_OK)
+                        ErrorF("DwmEnableBlurBehindWindow failed: %x, %d\n", (int)rc, (int)GetLastError());
+                }
+        }
+}
+
 /*
  * winTopLevelWindowProc - Window procedure for all top-level Windows windows.
  */
@@ -416,6 +529,8 @@ winTopLevelWindowProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
 
         SetWindowLongPtr(hwnd, GWLP_USERDATA, (LONG_PTR) XMING_SIGNATURE);
 
+        CheckForAlpha(hwnd, pWin, s_pScreenInfo);
+
         return 0;
 
     case WM_INIT_SYS_MENU:
@@ -1114,6 +1229,12 @@ winTopLevelWindowProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
         }
         break;
 
+
+    case WM_DWMCOMPOSITIONCHANGED:
+        /* This message is only sent on Vista/W7 */
+        CheckForAlpha(hwnd, pWin, s_pScreenInfo);
+
+        return 0;
     default:
         break;
     }
diff --git a/hw/xwin/winprocarg.c b/hw/xwin/winprocarg.c
index e849cbc97..29e91abc8 100644
--- a/hw/xwin/winprocarg.c
+++ b/hw/xwin/winprocarg.c
@@ -592,6 +592,25 @@ ddxProcessArgument(int argc, char *argv[], int i)
     }
 
     /*
+     * Look for the '-compositealpha' argument
+     */
+    if (IS_OPTION("-compositealpha")) {
+        g_fCompositeAlpha = TRUE;
+
+        /* Indicate that we have processed this argument */
+        return 1;
+    }
+    /*
+     * Look for the '-nocompositealpha' argument
+     */
+    if (IS_OPTION("-nocompositealpha")) {
+        g_fCompositeAlpha  = FALSE;
+
+        /* Indicate that we have processed this argument */
+        return 1;
+    }
+
+    /*
      * Look for the '-multiplemonitors' argument
      */
     if (IS_OPTION("-multiplemonitors")