Use SetWindowCompositionAttribute for alpha-awareness on W10

Using DwmEnableBlurBehindWindow() as we do works fine on Vista and Windows 7
but doesn't render properly on Windows 8.1, 10

Use the (undocumented) SetWindowCompositionAttribute function to get the
same appearance on Windows 10.

For Windows 8.1, we seem to be out of luck.

3 files changed, 86 insertions, 10 deletions

diff --git a/src/main.c b/src/main.c
index b50eaa4..58935d1 100644
--- a/src/main.c
+++ b/src/main.c
@@ -33,6 +33,29 @@
 #include "wndproc.h"
 #include "wincursor.h"
 
+// When mingw-w64-headers 5.0 is available, just include versionhelper.h for these
+static
+BOOL IsWindowsVersionOrGreater(WORD major, WORD minor, WORD servpack)
+{
+  OSVERSIONINFOEXW vi = {sizeof(vi),major,minor,0,0,{0},servpack,0,0,0,0};
+  return VerifyVersionInfoW(&vi, VER_MAJORVERSION|VER_MINORVERSION|VER_SERVICEPACKMAJOR,
+         VerSetConditionMask(VerSetConditionMask(VerSetConditionMask(0,
+            VER_MAJORVERSION,VER_GREATER_EQUAL),
+            VER_MINORVERSION,VER_GREATER_EQUAL),
+            VER_SERVICEPACKMAJOR, VER_GREATER_EQUAL));
+}
+
+static
+BOOL IsWindowsVistaOrGreater(void) {
+  return IsWindowsVersionOrGreater(HIBYTE(_WIN32_WINNT_VISTA), LOBYTE(_WIN32_WINNT_VISTA), 0);
+}
+
+static
+BOOL IsWindows8OrGreater(void) {
+  return IsWindowsVersionOrGreater(HIBYTE(_WIN32_WINNT_WIN8), LOBYTE(_WIN32_WINNT_WIN8), 0);
+}
+
+//
 #define WM_XCWM_CREATE  WM_USER
 #define WM_XCWM_DESTROY (WM_USER+1)
 #define WM_XCWM_CURSOR (WM_USER+2)
@@ -202,16 +225,34 @@ int main(int argc, char **argv)
   // initialize semaphore used for synchronization
   sem_init(&semaphore, 0, 0);
 
-  // Get access to DwmEnableBlurBehindWindow, if available, so we can take advantage of it
-  // on Vista and later, but still run on earlier versions of Windows
+  // The way to tell Windows that it should use the alpha channel from a GDI
+  // window in DWM desktop composition depends on the Windows version
   HMODULE hDwmApiLib = LoadLibraryEx("dwmapi.dll", NULL, 0);
-  if (hDwmApiLib)
-    pDwmEnableBlurBehindWindow = (PFNDWMENABLEBLURBEHINDWINDOW)GetProcAddress(hDwmApiLib, "DwmEnableBlurBehindWindow");
-  DEBUG("DwmEnableBlurBehindWindow %s\n", pDwmEnableBlurBehindWindow ? "found" : "not found");
+  HMODULE hUser32 = LoadLibraryEx("user32.dll", NULL, 0);
+
+  if (IsWindowsVistaOrGreater() && !IsWindows8OrGreater())
+    {
+      // Use DwmEnableBlurBehindWindow, if available, to turn on alpha channel
+      // use on Windows Vista and Windows 7
+
+      if (hDwmApiLib)
+        pDwmEnableBlurBehindWindow = (PFNDWMENABLEBLURBEHINDWINDOW)GetProcAddress(hDwmApiLib, "DwmEnableBlurBehindWindow");
+      DEBUG("DwmEnableBlurBehindWindow %s\n", pDwmEnableBlurBehindWindow ? "found" : "not found");
+    }
+  else if (IsWindows8OrGreater())
+    {
+      // Use SetWindowCompositionAttribute, if available, to turn on alpha
+      // channel use on Windows 8, 8.1, 10
+      if (hUser32)
+        pSetWindowCompositionAttribute = (PFNSETWINDOWCOMPOSITIONATTRIBUTE) GetProcAddress(hUser32, "SetWindowCompositionAttribute");
+      DEBUG("SetWindowCompositionAttribute %s\n", pSetWindowCompositionAttribute ? "found" : "not found");
+    }
+
   if (nodwm)
     {
       DEBUG("DWM disabled by --nodwm option\n");
       pDwmEnableBlurBehindWindow = NULL;
+      pSetWindowCompositionAttribute = NULL;
     }
 
   xcwm_context_flags_t flags = 0;
@@ -361,6 +402,7 @@ int main(int argc, char **argv)
   xcwm_context_close(context);
 
   FreeLibrary(hDwmApiLib);
+  FreeLibrary(hUser32);
 
   sem_destroy(&semaphore);
 }
diff --git a/src/wndproc.c b/src/wndproc.c
index cf75463..f639f4b 100644
--- a/src/wndproc.c
+++ b/src/wndproc.c
@@ -54,6 +54,7 @@
 
 int blur = 0;
 PFNDWMENABLEBLURBEHINDWINDOW pDwmEnableBlurBehindWindow = NULL;
+PFNSETWINDOWCOMPOSITIONATTRIBUTE pSetWindowCompositionAttribute = NULL;
 
 /*
  * ValidateSizing - Ensures size request respects hints
@@ -293,9 +294,18 @@ static void
 CheckForAlpha(HWND hWnd, xcwm_image_t *image)
 {
   /* image has alpha and we can do something useful with it? */
-  if ((image->image->depth == 32) && pDwmEnableBlurBehindWindow)
+  /* XXX: only need to do this once per window */
+  if (image->image->depth == 32)
+  {
+    if (pSetWindowCompositionAttribute)
+    {
+      /* This turns on DWM looking at the alpha-channel of this window */
+      ACCENTPOLICY policy = { ACCENT_ENABLE_BLURBEHIND, 0, 0, 0 } ;
+      WINCOMPATTR data = { WCA_ACCENT_POLICY,  &policy, sizeof(ACCENTPOLICY) };
+      pSetWindowCompositionAttribute(hWnd, &data);
+    }
+    else if(pDwmEnableBlurBehindWindow)
     {
-      /* XXX: only do this once, for each window */
       HRGN dummyRegion = NULL;
 
       /* restricting the blur effect to a dummy region means the rest is unblurred */
@@ -303,7 +313,7 @@ CheckForAlpha(HWND hWnd, xcwm_image_t *image)
         dummyRegion = CreateRectRgn(-1, -1, 0, 0);
 
       DWM_BLURBEHIND bbh;
-      bbh.dwFlags = DWM_BB_ENABLE | DWM_BB_BLURREGION |DWM_BB_TRANSITIONONMAXIMIZED;
+      bbh.dwFlags = DWM_BB_ENABLE | DWM_BB_BLURREGION | DWM_BB_TRANSITIONONMAXIMIZED;
       bbh.fEnable = TRUE;
       bbh.hRgnBlur = dummyRegion;
       bbh.fTransitionOnMaximized = TRUE; /* What does this do ??? */
@@ -313,11 +323,12 @@ CheckForAlpha(HWND hWnd, xcwm_image_t *image)
       // This terribly-named function actually controls if DWM looks at the alpha channel of this window
       HRESULT rc = pDwmEnableBlurBehindWindow(hWnd, &bbh);
       if (rc != S_OK)
-        fprintf(stderr, "DwmEnableBlurBehindWindow failed: %d\n", (int) GetLastError());
+        fprintf(stderr, "DwmEnableBlurBehindWindow failed: %x, %d\n", rc, (int) GetLastError());
 
       if (dummyRegion)
         DeleteObject(dummyRegion);
     }
+  }
 }
 
 static void
@@ -650,7 +661,7 @@ winApplyStyle(xcwm_window_t *window)
   if (zstyle == HWND_NOTOPMOST)
     flags |= SWP_NOZORDER | SWP_NOOWNERZORDER;
 
-  if (!pDwmEnableBlurBehindWindow)
+  if (!pDwmEnableBlurBehindWindow && !pSetWindowCompositionAttribute)
     {
       /*
         On XP, it seems we have to do an elaborate, performance killing
diff --git a/src/wndproc.h b/src/wndproc.h
index 5e3aac1..00e850c 100644
--- a/src/wndproc.h
+++ b/src/wndproc.h
@@ -35,6 +35,29 @@ void winAdjustWindowsWindow(xcwm_window_t *window);
 typedef HRESULT WINAPI (*PFNDWMENABLEBLURBEHINDWINDOW)(HWND hWnd, const DWM_BLURBEHIND *pBlurBehind);
 extern PFNDWMENABLEBLURBEHINDWINDOW pDwmEnableBlurBehindWindow;
 
+/* Undocumented */
+typedef struct _ACCENTPOLICY
+{
+  ULONG AccentState;
+  ULONG AccentFlags;
+  ULONG GradientColor;
+  ULONG AnimationId;
+} ACCENTPOLICY;
+
+#define ACCENT_ENABLE_BLURBEHIND 3
+
+typedef struct _WINCOMPATTR
+{
+    DWORD attribute;
+    PVOID pData;
+    ULONG dataSize;
+}  WINCOMPATTR;
+
+#define WCA_ACCENT_POLICY 19
+
+typedef HRESULT WINAPI (*PFNSETWINDOWCOMPOSITIONATTRIBUTE)(HWND, WINCOMPATTR *);
+extern PFNSETWINDOWCOMPOSITIONATTRIBUTE pSetWindowCompositionAttribute;
+
 extern int blur;
 
 #endif /* WNDPROC_H */