Transparency support for D3D swap chain (#837)

* Transparency support for D3D swap chain

* Store composition objects in DirectXDevice

* Fix AWT transparent sample

samples/SkiaAwtSample/src/main/kotlin/SkiaAwtSample/App.kt

@@ -40,7 +40,7 @@ fun createWindow(title: String, exitOnClose: Boolean) = SwingUtilities.invokeLat
     window.defaultCloseOperation =
         if (exitOnClose) WindowConstants.EXIT_ON_CLOSE else WindowConstants.DISPOSE_ON_CLOSE
     window.background = Color.GREEN
-    window.contentPane = skiaLayer
+    window.contentPane.add(skiaLayer)
 
     // Create menu.
     val menuBar = JMenuBar()
@@ -120,11 +120,33 @@ fun createWindow(title: String, exitOnClose: Boolean) = SwingUtilities.invokeLat
     // Window transparency
     if (System.getProperty("skiko.transparency") == "true") {
         window.isUndecorated = true
-        // On Windows we don't set transparent background to avoid event input issues (JDK specific)
-        if (hostOs != OS.Windows) {
+
+        /**
+         * There is a hack inside skiko OpenGL and Software redrawers for Windows that makes current
+         * window transparent without setting `background` to JDK's window. It's done by getting native
+         * component parent and calling `DwmEnableBlurBehindWindow`.
+         *
+         * FIXME: Make OpenGL work inside transparent window (background == Color(0, 0, 0, 0)) without this hack.
+         *
+         * See `enableTransparentWindow` (skiko/src/awtMain/cpp/windows/window_util.cc)
+         */
+        if (hostOs != OS.Windows || skiaLayer.renderApi == GraphicsApi.DIRECT3D) {
             window.background = Color(0, 0, 0, 0)
         }
         skiaLayer.transparency = true
+
+        /*
+         * Windows makes clicks on transparent pixels fall through, but it doesn't work
+         * with GPU accelerated rendering since this check requires having access to pixels from CPU.
+         *
+         * JVM doesn't allow override this behaviour with low-level windows methods, so hack this in this way.
+         * Based on tests, it doesn't affect resulting pixel color.
+         */
+        if (hostOs == OS.Windows) {
+            val contentPane = window.contentPane as JComponent
+            contentPane.background = Color(0, 0, 0, 1)
+            contentPane.isOpaque = true
+        }
     } else {
         skiaLayer.background = Color.LIGHT_GRAY
     }

skiko/src/awtMain/cpp/windows/directXRedrawer.cc

@@ -20,6 +20,9 @@
 #include <dxgi1_4.h>
 #include <dxgi1_6.h>
 
+#include <dcomp.h>
+#pragma comment(lib, "dcomp.lib")
+
 const int BuffersCount = 2;
 
 class DirectXDevice
@@ -32,6 +35,9 @@ public:
     gr_cp<ID3D12CommandQueue> queue;
     gr_cp<ID3D12Resource> buffers[BuffersCount];
     gr_cp<ID3D12Fence> fence;
+    gr_cp<IDCompositionDevice> dcDevice;
+    gr_cp<IDCompositionTarget> dcTarget;
+    gr_cp<IDCompositionVisual> dcVisual;
     uint64_t fenceValues[BuffersCount];
     HANDLE fenceEvent = NULL;
     unsigned int bufferIndex;
@@ -53,24 +59,36 @@ public:
     }
 
     void initSwapChain() {
+        RECT windowRect;
+        GetClientRect(window, &windowRect);
+        UINT width = windowRect.right - windowRect.left;
+        UINT height = windowRect.bottom - windowRect.top;
+
         gr_cp<IDXGIFactory4> swapChainFactory4;
         gr_cp<IDXGISwapChain1> swapChain1;
         CreateDXGIFactory2(0, IID_PPV_ARGS(&swapChainFactory4));
         DXGI_SWAP_CHAIN_DESC1 swapChainDesc = {};
-        swapChainDesc.BufferCount = BuffersCount;
+        swapChainDesc.Width = width;
+        swapChainDesc.Height = height;
         swapChainDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
+        swapChainDesc.SampleDesc.Count = 1;
+        swapChainDesc.SampleDesc.Quality = 0;
         swapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
+        swapChainDesc.BufferCount = BuffersCount;
+        swapChainDesc.Scaling = DXGI_SCALING_STRETCH;
         swapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD;
-        swapChainDesc.SampleDesc.Count = 1;
-        swapChainDesc.Scaling = DXGI_SCALING_NONE;
-        swapChainFactory4->CreateSwapChainForHwnd(queue.get(), window, &swapChainDesc, nullptr, nullptr, &swapChain1);
+        swapChainDesc.AlphaMode = DXGI_ALPHA_MODE_PREMULTIPLIED;
+        swapChainFactory4->CreateSwapChainForComposition(queue.get(), &swapChainDesc, nullptr, &swapChain1);
+
+        DCompositionCreateDevice(0, IID_PPV_ARGS(&dcDevice));
+        dcDevice->CreateTargetForHwnd(window, true, &dcTarget);
+        dcDevice->CreateVisual(&dcVisual);
+        dcVisual->SetContent(swapChain1.get());
+        dcTarget->SetRoot(dcVisual.get());
+        dcDevice->Commit();
+
         swapChainFactory4->MakeWindowAssociation(window, DXGI_MWA_NO_ALT_ENTER);
         swapChain1->QueryInterface(IID_PPV_ARGS(&swapChain));
-        RECT windowRect;
-        GetWindowRect(window, &windowRect);
-        unsigned int w = windowRect.right - windowRect.left;
-        unsigned int h = windowRect.bottom - windowRect.top;
-        swapChain->ResizeBuffers(BuffersCount, w, h, DXGI_FORMAT_R8G8B8A8_UNORM, 0);
         swapChainFactory4.reset(nullptr);
     }
 };
@@ -292,6 +310,7 @@ extern "C"
             return 0;
         }
 
+        HWND hWnd = fromJavaPointer<HWND>(contentHandle);
         DirectXDevice *d3dDevice = new DirectXDevice();
         d3dDevice->backendContext.fAdapter = adapter;
         d3dDevice->backendContext.fDevice = device;
@@ -300,13 +319,11 @@ extern "C"
 
         d3dDevice->device = device;
         d3dDevice->queue = queue;
-        d3dDevice->window = (HWND)contentHandle;
+        d3dDevice->window = hWnd;
 
         if (transparency) {
-            //TODO: current swapChain does not support transparency
-            return 0;
-            // HWND wnd = GetAncestor(d3dDevice->window, GA_PARENT);
-            // enableTransparentWindow(wnd);
+            const LONG style = GetWindowLong(hWnd, GWL_EXSTYLE);
+            SetWindowLong(hWnd, GWL_EXSTYLE, style | WS_EX_TRANSPARENT);
         }
 
         return toJavaPointer(d3dDevice);

skiko/src/awtMain/kotlin/org/jetbrains/skiko/redrawer/Direct3DRedrawer.kt

 package org.jetbrains.skiko.redrawer
 
-import kotlinx.coroutines.asCoroutineDispatcher
 import kotlinx.coroutines.withContext
 import org.jetbrains.skia.DirectContext
 import org.jetbrains.skia.Surface
 import org.jetbrains.skia.SurfaceProps
-import org.jetbrains.skia.impl.interopScope
 import org.jetbrains.skia.impl.InteropPointer
+import org.jetbrains.skia.impl.interopScope
 import org.jetbrains.skiko.*
 import org.jetbrains.skiko.context.Direct3DContextHandler
 

skiko/src/commonMain/kotlin/org/jetbrains/skiko/SkiaLayer.kt

@@ -27,6 +27,7 @@ expect open class SkiaLayer {
      * If rendering is full screen.
      */
     var fullscreen: Boolean
+
     /**
      * If transparency is enabled.
      */

skiko/src/jvmMain/cpp/common/stubs.cc

@@ -59,6 +59,11 @@ JNIEXPORT void JNICALL Java_org_jetbrains_skiko_context_Direct3DContextHandler_f
     skikoUnimplemented("Java_org_jetbrains_skiko_context_Direct3DContextHandler_flush");
 }
 
+JNIEXPORT jlong JNICALL Java_org_jetbrains_skiko_redrawer_Direct3DRedrawer_chooseAdapter(JNIEnv *env, jobject redrawer, jint adapterPriority) {
+    skikoUnimplemented("Java_org_jetbrains_skiko_redrawer_Direct3DRedrawer_chooseAdapter");
+    return 0;
+}
+
 JNIEXPORT jlong JNICALL Java_org_jetbrains_skiko_redrawer_Direct3DRedrawer_createDirectXDevice(
         JNIEnv *env, jobject redrawer, jint adapterPriority, jlong contentHandle, jboolean transparency) {
     skikoUnimplemented("Java_org_jetbrains_skiko_redrawer_Direct3DRedrawer_createDirectXDevice");