Introduce Swing interop with offscreen rendering on Linux with OpenGL (#776)

skiko/build.gradle.kts

@@ -887,6 +887,7 @@ fun createCompileJvmBindingsTask(
             includeHeadersNonRecursive(runPkgConfig("dbus-1"))
             osFlags = arrayOf(
                 *buildType.clangFlags,
+                "-DGL_GLEXT_PROTOTYPES",
                 "-fPIC",
                 "-fno-rtti",
                 "-fno-exceptions",

skiko/src/awtMain/cpp/linux/swingRedrawer.cc

+#include <jawt_md.h>
+#include <GL/gl.h>
+#include <GL/glx.h>
+#include <X11/X.h>
+#include <X11/Xlib.h>
+#include <X11/Xresource.h>
+#include <cstdlib>
+#include <unistd.h>
+#include <stdio.h>
+#include "jni_helpers.h"
+
+class OffScreenContext
+{
+public:
+    Display* display;
+    GLXContext context;
+    GLXFBConfig* fbConfigs;
+
+    OffScreenContext(Display* _display, GLXContext _context, GLXFBConfig* _fbConfigs) {
+        display = _display;
+        context = _context;
+        fbConfigs = _fbConfigs;
+    }
+
+    ~OffScreenContext() {
+        XFree(fbConfigs);
+        glXDestroyContext(display, context);
+        XCloseDisplay(display);
+    }
+
+    static OffScreenContext* create() {
+        const int glxContextAttribs[] {
+            GLX_DRAWABLE_TYPE, GLX_WINDOW_BIT,
+            GLX_RENDER_TYPE, GLX_RGBA_BIT,
+            GLX_DOUBLEBUFFER, False,
+            GLX_RED_SIZE, 8,
+            GLX_GREEN_SIZE, 8,
+            GLX_BLUE_SIZE, 8,
+            None
+        };
+
+        Display* display = XOpenDisplay(nullptr);
+
+        if (!display) {
+            return nullptr;
+        }
+
+        int numConfigs = 0;
+        GLXFBConfig* fbConfigs = glXChooseFBConfig(display, DefaultScreen(display), glxContextAttribs, &numConfigs);
+
+        if (!fbConfigs) {
+            XCloseDisplay(display);
+            return nullptr;
+        }
+
+        XVisualInfo* visual = glXGetVisualFromFBConfig(display, fbConfigs[0]);
+
+        if (!visual) {
+            XFree(fbConfigs);
+            XCloseDisplay(display);
+            return nullptr;
+        }
+
+        GLXContext context = glXCreateContext(display, visual, nullptr, True);
+
+        XFree(visual);
+
+        if (!context) {
+            XFree(fbConfigs);
+            XCloseDisplay(display);
+            return nullptr;
+        }
+
+        return new OffScreenContext(display, context, fbConfigs);
+    }
+};
+
+class OffScreenBuffer
+{
+public:
+    Display* display;
+    GLXPbuffer pbuffer;
+    int width;
+    int height;
+
+    OffScreenBuffer(Display* _display, GLXPbuffer _pbuffer, int _width, int _height) {
+        display = _display;
+        pbuffer = _pbuffer;
+        width = _width;
+        height = _height; 
+    }
+
+    ~OffScreenBuffer() {
+        glXDestroyPbuffer(display, pbuffer);
+    }
+
+    static OffScreenBuffer* create(OffScreenContext* context, int width, int height) {
+        int pbufferAttribs[] = {
+            GLX_PBUFFER_WIDTH, width,
+            GLX_PBUFFER_HEIGHT, height,
+            None  
+        };
+
+        GLXPbuffer pbuffer = glXCreatePbuffer(context->display, context->fbConfigs[0], pbufferAttribs);
+
+        if (!pbuffer) {
+            return nullptr; 
+        }
+
+        return new OffScreenBuffer(context->display, pbuffer, width, height);
+    }
+};
+
+class OffScreenTexture
+{
+public:
+    GLuint tex;
+    GLuint fbo;
+
+    OffScreenTexture(GLuint _tex, GLuint _fbo) {
+        tex = _tex;
+        fbo = _fbo;
+    }
+
+    ~OffScreenTexture() {
+        glDeleteFramebuffers(1, &fbo);
+        glDeleteTextures(1, &tex);
+    }
+};
+
+extern "C"
+{
+    JNIEXPORT jlong JNICALL Java_org_jetbrains_skiko_swing_LinuxOpenGLSwingRedrawer_makeOffScreenContext(
+        JNIEnv *env, jobject redrawer)
+    {
+        OffScreenContext* context = OffScreenContext::create();
+        return toJavaPointer(context);
+    }
+
+    JNIEXPORT void JNICALL Java_org_jetbrains_skiko_swing_LinuxOpenGLSwingRedrawer_disposeOffScreenContext(
+        JNIEnv *env, jobject redrawer, jlong contextPtr)
+    {
+        OffScreenContext* context = fromJavaPointer<OffScreenContext *>(contextPtr);
+
+        if (context) {
+            delete context;
+        }
+    }
+
+    JNIEXPORT jlong JNICALL Java_org_jetbrains_skiko_swing_LinuxOpenGLSwingRedrawer_makeOffScreenBuffer(
+        JNIEnv *env, jobject redrawer, jlong contextPtr, jlong oldBufferPtr, jint width, jint height)
+    {
+        OffScreenContext* context = fromJavaPointer<OffScreenContext *>(contextPtr);
+        OffScreenBuffer* oldBuffer = fromJavaPointer<OffScreenBuffer *>(oldBufferPtr);
+        
+        OffScreenBuffer* buffer;
+        if (oldBuffer == nullptr || oldBuffer->width != width || oldBuffer->height != height) {
+            if (oldBuffer != nullptr) {
+                delete oldBuffer;
+            }
+            buffer = OffScreenBuffer::create(context, width, height);
+        } else {
+            buffer = oldBuffer;
+        }
+
+        return toJavaPointer(buffer);
+    }
+
+    JNIEXPORT void JNICALL Java_org_jetbrains_skiko_swing_LinuxOpenGLSwingRedrawer_disposeOffScreenBuffer(
+        JNIEnv *env, jobject redrawer, jlong bufferPtr)
+    {
+        OffScreenBuffer* buffer = fromJavaPointer<OffScreenBuffer *>(bufferPtr);
+        if (buffer) {
+            delete buffer;
+        }
+    }
+
+    JNIEXPORT void JNICALL Java_org_jetbrains_skiko_swing_LinuxOpenGLSwingRedrawer_startRendering(
+        JNIEnv *env, jobject redrawer, jlong contextPtr, jlong bufferPtr)
+    {
+        OffScreenContext* context = fromJavaPointer<OffScreenContext *>(contextPtr);
+        OffScreenBuffer* buffer = fromJavaPointer<OffScreenBuffer *>(bufferPtr);
+
+        glXMakeCurrent(context->display, buffer->pbuffer, context->context);
+    }
+
+    JNIEXPORT void JNICALL Java_org_jetbrains_skiko_swing_LinuxOpenGLSwingRedrawer_finishRendering(
+        JNIEnv *env, jobject redrawer, jlong contextPtr)
+    {
+        OffScreenContext* context = fromJavaPointer<OffScreenContext *>(contextPtr);
+        glXMakeCurrent(context->display, None, nullptr);
+    }
+
+    JNIEXPORT jlong JNICALL Java_org_jetbrains_skiko_swing_LinuxOpenGLSwingRedrawer_createAndBindTexture(
+        JNIEnv *env, jobject redrawer, jint width, jint height)
+    {
+        GLuint tex;
+        glGenTextures(1, &tex);
+        glBindTexture(GL_TEXTURE_2D, tex);
+        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
+
+        GLuint fbo;
+        glGenFramebuffers(1, &fbo);
+        glBindFramebuffer(GL_FRAMEBUFFER, fbo);
+
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, tex, 0);
+
+        OffScreenTexture *texture = new OffScreenTexture(tex, fbo);
+
+        return toJavaPointer(texture);
+    }
+
+    JNIEXPORT GLuint JNICALL Java_org_jetbrains_skiko_swing_LinuxOpenGLSwingRedrawer_getFboId(
+        JNIEnv *env, jobject redrawer, jlong texturePtr)
+    {
+        OffScreenTexture *texture = fromJavaPointer<OffScreenTexture *>(texturePtr);
+
+        return texture->fbo;
+    }
+
+    JNIEXPORT void JNICALL Java_org_jetbrains_skiko_swing_LinuxOpenGLSwingRedrawer_unbindAndDisposeTexture(
+        JNIEnv *env, jobject redrawer, jlong texturePtr)
+    {
+        glBindFramebuffer(GL_FRAMEBUFFER, 0);
+        glBindTexture(GL_TEXTURE_2D, 0);
+        OffScreenTexture *texture = fromJavaPointer<OffScreenTexture *>(texturePtr);
+
+        if (texture) {
+            delete texture;
+        }
+    }
+} // extern "C"
\ No newline at end of file

skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/LinuxOpenGLSwingRedrawer.kt

+package org.jetbrains.skiko.swing
+
+import org.jetbrains.skia.*
+import org.jetbrains.skiko.*
+import java.awt.Graphics2D
+
+internal class LinuxOpenGLSwingRedrawer(
+    swingLayerProperties: SwingLayerProperties,
+    private val skikoView: SkikoView,
+    analytics: SkiaLayerAnalytics
+) : SwingRedrawerBase(swingLayerProperties, analytics, GraphicsApi.OPENGL) {
+    init {
+        onDeviceChosen("OpenGL OffScreen") // TODO: properly choose device
+    }
+
+    private val swingOffscreenDrawer = SwingOffscreenDrawer(swingLayerProperties)
+
+    private val offScreenContextPtr: Long = makeOffScreenContext()
+
+    private var offScreenBufferPtr: Long = 0L
+
+    private val storage = Bitmap()
+
+    private var bytesToDraw = ByteArray(0)
+
+    init {
+        onContextInit()
+    }
+
+    override fun dispose() {
+        bytesToDraw = ByteArray(0)
+        storage.close()
+        disposeOffScreenBuffer(offScreenBufferPtr)
+        disposeOffScreenContext(offScreenContextPtr)
+        super.dispose()
+    }
+
+    override fun onRender(g: Graphics2D, width: Int, height: Int, nanoTime: Long) {
+        offScreenBufferPtr = makeOffScreenBuffer(offScreenContextPtr, offScreenBufferPtr, width, height)
+        if (offScreenBufferPtr == 0L) {
+            throw RenderException("Cannot create offScreen OpenGL buffer")
+        }
+        startRendering(offScreenContextPtr, offScreenBufferPtr)
+        try {
+            autoCloseScope {
+                // TODO: reuse texture
+                val texturePtr = createAndBindTexture(width, height)
+                if (texturePtr == 0L) {
+                    throw RenderException("Cannot create offScreen OpenGL texture")
+                }
+                val fbId = getFboId(texturePtr)
+                val renderTarget = makeGLRenderTarget(
+                    width,
+                    height,
+                    0,
+                    8,
+                    fbId,
+                    FramebufferFormat.GR_GL_RGBA8
+                ).autoClose()
+
+                // TODO: may be it is possible to reuse [makeGLContext]
+                val directContext = makeGLContext().autoClose()
+                val surface = Surface.makeFromBackendRenderTarget(
+                    directContext,
+                    renderTarget,
+                    SurfaceOrigin.TOP_LEFT,
+                    SurfaceColorFormat.BGRA_8888,
+                    ColorSpace.sRGB,
+                    SurfaceProps(pixelGeometry = PixelGeometry.UNKNOWN)
+                )?.autoClose() ?: throw RenderException("Cannot create surface")
+
+                val canvas = surface.canvas
+                canvas.clear(Color.TRANSPARENT)
+                skikoView.onRender(canvas, width, height, nanoTime)
+                flush(surface, g)
+                unbindAndDisposeTexture(texturePtr)
+            }
+        } finally {
+            finishRendering(offScreenContextPtr)
+        }
+    }
+
+    private fun flush(surface: Surface, g: Graphics2D) {
+        surface.flushAndSubmit(syncCpu = true)
+
+        val width = surface.width
+        val height = surface.height
+
+        val dstRowBytes = width * 4
+        if (storage.width != width || storage.height != height) {
+            storage.allocPixelsFlags(ImageInfo.makeS32(width, height, ColorAlphaType.PREMUL), false)
+            bytesToDraw = ByteArray(storage.getReadPixelsArraySize(dstRowBytes = dstRowBytes))
+        }
+        // TODO: it copies pixels from GPU to CPU, so it is really slow
+        surface.readPixels(storage, 0, 0)
+
+        val successfulRead = storage.readPixels(bytesToDraw, dstRowBytes = dstRowBytes)
+        if (successfulRead) {
+            swingOffscreenDrawer.draw(g, bytesToDraw, width, height)
+        }
+    }
+
+    /**
+     * Creates new OpenGL context and opens X11 connection.
+     * This context can be used together with buffer ([makeOffScreenBuffer]) for offscreen rendering.
+     *
+     * Should be manually disposed using [disposeOffScreenContext] when no longer needed.
+     */
+    private external fun makeOffScreenContext(): Long
+    private external fun disposeOffScreenContext(contextPtr: Long): Long
+
+    /**
+     * Provides offscreen pixels GPU buffer.
+     * If size of [oldBufferPtr] same as [width] and [height], it will be reused
+     * or created new one otherwise ([oldBufferPtr] will be disposed in this case automatically).
+     *
+     * Should be manually disposed using [disposeOffScreenBuffer] when no longer needed.
+     *
+     * @see [makeOffScreenContext]
+     */
+    private external fun makeOffScreenBuffer(contextPtr: Long, oldBufferPtr: Long, width: Int, height: Int): Long
+    private external fun disposeOffScreenBuffer(bufferPtr: Long)
+
+    /**
+     * Sets current OpenGL context to [contextPtr] and [bufferPtr],
+     * so OpenGL will render into offscreen texture not on screen.
+     *
+     * Make sure to call [finishRendering] to reset context and wait for all Open GL commands to apply.
+     */
+    private external fun startRendering(contextPtr: Long, bufferPtr: Long)
+    private external fun finishRendering(contextPtr: Long)
+
+    private external fun createAndBindTexture(width: Int, height: Int): Long
+    private external fun getFboId(texturePtr: Long): Int
+    private external fun unbindAndDisposeTexture(texturePtr: Long)
+}
\ No newline at end of file

skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/SwingRedrawer.kt

@@ -40,6 +40,7 @@ internal fun createSwingRedrawer(
     return when (hostOs) {
         OS.MacOS -> MetalSwingRedrawer(swingLayerProperties, skikoView, analytics)
         OS.Windows -> Direct3DSwingRedrawer(swingLayerProperties, skikoView, analytics)
+        OS.Linux -> LinuxOpenGLSwingRedrawer(swingLayerProperties, skikoView, analytics)
         else -> SoftwareSwingRedrawer(swingLayerProperties, skikoView, analytics)
     }
 }
\ No newline at end of file