metal: accelerate swing rendering (#1031)

This PR contain:
1. Using shared metal textures with this JBR API -
[SharedTextures](https://github.com/JetBrains/JetBrainsRuntimeApi/blob/main/src/com/jetbrains/SharedTextures.java).
See `AcceleratedSwingPainter`.

2. Not ultimate, but visible improvement of the not accelerated painting
for Windows(~%25), and MacOS(~20%). Unfortunately, didn't manage to see
a positive change on Linux with XRender pipeline(about the same,
probably few percents down). With opengl pipeline it also got ~20%
faster. But currently our pipeline on linux is XRender
I got rid of make one extra copy of the raster image. See
`SoftwareSwingPainter`

`com.jetbrains:jbr-api` size is about 44kb

Benchmarks:
`Redraw` - time to prepare the offscreen image including sync, but not
including fetching texture to the CPU RAM
`Paint` - time to get the image from GPU(if needed) and draw onto
`Graphics2D`
`Total` - total time to deliver the frame

```
MetalSwingRedrawer:
Image size: 3200x2344
Test: ClocksAwt
MacBook Pro M1 Max
java2d pipeline = Metal
               Current   SoftwareSwingPainter   AcceleratedSwingPainter
FPS              66             80                      141
Redraw(ms)      7.13           7.13                     6.92
Paint(ms)       7.65           6.07                     0.006
Total(ms)      14.78          12.16                     6.92


Direct3DSwingRedrawer
Image size: 3176x2284
Dell Prescision 5570, i9-12900H + GPU Nvidia
Test: ClocksAwt
java2d pipeline = GDI
               Current   SoftwareSwingPainter
FPS              32             41
Redraw(ms)     12.58          11.57
Paint(ms)      17.43          11.54
Total(ms)      29.93          23.11


LinuxOpenGLSwingRedrawer
Image size: 3192x2230
Dell Prescision 5570, i9-12900H + GPU Nvidia
Test: ClocksAwt
java2d pipeline = XRender
               Current   SoftwareSwingPainter
FPS              24             23
Redraw(ms)     20.07          19.64
Paint(ms)      19.69          22.73
Total(ms)      39.77          42.39
```

---------
Co-authored-by: Igor Demin <igordmn@users.noreply.github.com>

samples/SkiaAwtSample/build.gradle.kts

@@ -39,6 +39,7 @@ dependencies {
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-swing:1.5.0")
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.5.0")
     implementation("org.jetbrains.skiko:skiko-awt-runtime-$target:$version")
+    implementation("org.jetbrains.runtime:jbr-api:1.5.0")
     testImplementation("org.jetbrains.kotlin:kotlin-test")
     testImplementation("org.jetbrains.kotlin:kotlin-test-junit")
 }

skiko/build.gradle.kts

@@ -167,6 +167,7 @@ kotlin {
             dependencies {
                 implementation(kotlin("stdlib"))
                 implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+                implementation("org.jetbrains.runtime:jbr-api:1.5.0")
             }
         }
         val commonTest by getting {

skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/AcceleratedSwingPainter.kt

+package org.jetbrains.skiko.swing
+
+import org.jetbrains.skia.Surface
+import java.awt.Graphics2D
+import com.jetbrains.JBR
+import com.jetbrains.SharedTextures
+import org.jetbrains.skiko.RenderException
+import java.awt.GraphicsConfiguration
+import java.awt.GraphicsEnvironment
+import java.awt.Image
+
+internal class AcceleratedSwingPainter : SwingPainter {
+    private val sharedTextures =
+        if (JBR.isSharedTexturesSupported() &&
+            JBR.getSharedTextures().textureType == SharedTextures.METAL_TEXTURE_TYPE
+        ) JBR.getSharedTextures()
+        else throw RenderException("Shared textures are not supported")
+
+    private var imageWrapper: Image? = null
+    private var texturePtr: Long = 0L
+    private var gc: GraphicsConfiguration = GraphicsEnvironment.getLocalGraphicsEnvironment()
+        .defaultScreenDevice.defaultConfiguration
+
+    override fun paint(g: Graphics2D, surface: Surface, texture: Long) {
+        if (g.deviceConfiguration != gc || texturePtr != texture || imageWrapper == null) {
+            gc = g.deviceConfiguration
+            texturePtr = texture
+            imageWrapper = sharedTextures.wrapTexture(gc, texturePtr)
+        }
+
+        g.drawImage(imageWrapper, 0, 0, null)
+    }
+
+    override fun dispose() {
+    }
+}

skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/Direct3DSwingRedrawer.kt

@@ -10,7 +10,6 @@ import org.jetbrains.skiko.graphicapi.InternalDirectXApi.disposeDevice
 import org.jetbrains.skiko.graphicapi.InternalDirectXApi.makeDirectXContext
 import org.jetbrains.skiko.graphicapi.InternalDirectXApi.makeDirectXRenderTargetOffScreen
 import org.jetbrains.skiko.graphicapi.InternalDirectXApi.makeDirectXTexture
-import org.jetbrains.skiko.graphicapi.InternalDirectXApi.readPixels
 import org.jetbrains.skiko.graphicapi.InternalDirectXApi.waitForCompletion
 import java.awt.Graphics2D
 
@@ -32,7 +31,7 @@ internal class Direct3DSwingRedrawer(
 
     private val device = createDirectXOffscreenDevice(adapter)
 
-    private val swingOffscreenDrawer = SwingOffscreenDrawer(swingLayerProperties)
+    private val painter: SwingPainter = SoftwareSwingPainter(swingLayerProperties)
 
     private val context = if (device == 0L) {
         throw RenderException("Failed to create DirectX12 device.")
@@ -54,6 +53,7 @@ internal class Direct3DSwingRedrawer(
         context.close()
         disposeDirectXTexture(texturePtr)
         disposeDevice(device)
+        painter.dispose()
         super.dispose()
     }
 
@@ -87,18 +87,9 @@ internal class Direct3DSwingRedrawer(
 
     fun flush(surface: Surface, g: Graphics2D) {
         surface.flushAndSubmit(syncCpu = false)
-
-        val bytesArraySize = surface.width * surface.height * 4
-        if (bytesToDraw.size != bytesArraySize) {
-            bytesToDraw = ByteArray(bytesArraySize)
-        }
-
         waitForCompletion(device, texturePtr)
-        if(!readPixels(texturePtr, bytesToDraw)) {
-            throw RenderException("Couldn't read pixels")
-        }
 
-        swingOffscreenDrawer.draw(g, bytesToDraw, surface.width, surface.height)
+        painter.paint(g, surface, texturePtr)
     }
 
     private fun makeRenderTarget() = BackendRenderTarget(

skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/LinuxOpenGLSwingRedrawer.kt

@@ -13,7 +13,7 @@ internal class LinuxOpenGLSwingRedrawer(
         onDeviceChosen("OpenGL OffScreen") // TODO: properly choose device
     }
 
-    private val swingOffscreenDrawer = SwingOffscreenDrawer(swingLayerProperties)
+    private val painter: SwingPainter = SoftwareSwingPainter(swingLayerProperties)
 
     private val offScreenContextPtr: Long = makeOffScreenContext().also {
         if (it == 0L) {
@@ -37,6 +37,7 @@ internal class LinuxOpenGLSwingRedrawer(
         storage.close()
         disposeOffScreenBuffer(offScreenBufferPtr)
         disposeOffScreenContext(offScreenContextPtr)
+        painter.dispose()
         super.dispose()
     }
 
@@ -87,22 +88,7 @@ internal class LinuxOpenGLSwingRedrawer(
 
     private fun flush(surface: Surface, g: Graphics2D) {
         surface.flushAndSubmit(syncCpu = true)
-
-        val width = surface.width
-        val height = surface.height
-
-        val dstRowBytes = width * 4
-        if (storage.width != width || storage.height != height) {
-            storage.allocPixelsFlags(ImageInfo.makeS32(width, height, ColorAlphaType.PREMUL), false)
-            bytesToDraw = ByteArray(storage.getReadPixelsArraySize(dstRowBytes = dstRowBytes))
-        }
-        // TODO: it copies pixels from GPU to CPU, so it is really slow
-        surface.readPixels(storage, 0, 0)
-
-        val successfulRead = storage.readPixels(bytesToDraw, dstRowBytes = dstRowBytes)
-        if (successfulRead) {
-            swingOffscreenDrawer.draw(g, bytesToDraw, width, height)
-        }
+        painter.paint(g, surface, 0)
     }
 
     /**

skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/MetalSwingRedrawer.kt

@@ -15,7 +15,7 @@ import java.awt.Graphics2D
  * For on-screen rendering see [org.jetbrains.skiko.redrawer.MetalRedrawer].
  *
  * @see SwingRedrawerBase
- * @see SwingOffscreenDrawer
+ * @see SoftwareSwingPainter
  */
 internal class MetalSwingRedrawer(
     swingLayerProperties: SwingLayerProperties,
@@ -26,6 +26,12 @@ internal class MetalSwingRedrawer(
         init {
             Library.load()
         }
+
+        private fun createSwingPainter(swingLayerProperties: SwingLayerProperties): SwingPainter = try {
+            AcceleratedSwingPainter()
+        } catch (_ : RenderException) {
+            SoftwareSwingPainter(swingLayerProperties)
+        }
     }
 
     private val adapter: MetalAdapter = chooseMetalAdapter(swingLayerProperties.adapterPriority).also {
@@ -35,22 +41,17 @@ internal class MetalSwingRedrawer(
 
     private var texturePtr: Long = 0
 
-    private val storage = Bitmap()
-
-    private var bytesToDraw = ByteArray(0)
-
     init {
         onContextInit()
     }
 
-    private val swingOffscreenDrawer = SwingOffscreenDrawer(swingLayerProperties)
+    private val painter: SwingPainter = createSwingPainter(swingLayerProperties)
 
     override fun dispose() {
-        bytesToDraw = ByteArray(0)
-        storage.close()
         disposeMetalTexture(texturePtr)
         context.close()
         adapter.dispose()
+        painter.dispose()
         super.dispose()
     }
 
@@ -78,22 +79,7 @@ internal class MetalSwingRedrawer(
 
     private fun flush(surface: Surface, g: Graphics2D) {
         surface.flushAndSubmit(syncCpu = true)
-
-        val width = surface.width
-        val height = surface.height
-
-        val dstRowBytes = width * 4
-        if (storage.width != width || storage.height != height) {
-            storage.allocPixelsFlags(ImageInfo.makeS32(width, height, ColorAlphaType.PREMUL), false)
-            bytesToDraw = ByteArray(storage.getReadPixelsArraySize(dstRowBytes = dstRowBytes))
-        }
-        // TODO: it copies pixels from GPU to CPU, so it is really slow
-        surface.readPixels(storage, 0, 0)
-
-        val successfulRead = storage.readPixels(bytesToDraw, dstRowBytes = dstRowBytes)
-        if (successfulRead) {
-            swingOffscreenDrawer.draw(g, bytesToDraw, width, height)
-        }
+        painter.paint(g, surface, texturePtr)
     }
 
     override fun rendererInfo(): String {

skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/SwingOffscreenDrawer.kt → skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/SoftwareSwingPainter.kt

 package org.jetbrains.skiko.swing
 
+import org.jetbrains.skia.Bitmap
+import org.jetbrains.skia.ColorAlphaType
+import org.jetbrains.skia.ImageInfo
+import org.jetbrains.skia.Surface
+import org.jetbrains.skia.impl.BufferUtil
+import org.jetbrains.skiko.RenderException
 import java.awt.*
 import java.awt.geom.AffineTransform
 import java.awt.image.*
-import java.nio.ByteBuffer
 import java.nio.ByteOrder
 import java.nio.IntBuffer
 import kotlin.math.*
 
-// TODO: extract this code to a library and share it with JCEF implementation in IntelliJ
-//  since this code is mostly taken from intellij repository with some small changes
-internal class SwingOffscreenDrawer(
+internal class SoftwareSwingPainter(
     private val swingLayerProperties: SwingLayerProperties
-) {
-    @Volatile
-    private var volatileImage: VolatileImage? = null
-    private var bufferedImage: BufferedImage? = null
-    private var bufferedImageGraphics: Graphics2D? = null
+) : SwingPainter {
+    private var bufferedImage = BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB_PRE)
+    private var bitmap = Bitmap()
 
-    /**
-     * Draws rendered image that is represented by [bytes] on [g].
-     *
-     * If size of the rendered image is bigger than size from [swingLayerProperties]
-     * then only part of the image will be drawn on [g].
-     *
-     * @param g graphics where rendered picture given in [bytes] should be drawn
-     * @param bytes bytes of rendered picture in little endian order
-     * @param width width of rendered picture in real pixels
-     * @param height height of rendered picture in real pixels
-     */
-    fun draw(g: Graphics2D, bytes: ByteArray, width: Int, height: Int) {
-        val dirtyRectangles = listOf(
-            Rectangle(0, 0, width, height)
-        )
-        val image = createImageFromBytes(bytes, width, height, dirtyRectangles)
-        var vi = volatileImage
-
-        do {
-            if (vi == null || vi.width != swingLayerProperties.width || vi.height != swingLayerProperties.height) {
-                vi = createVolatileImage(image)
+    override fun paint(g: Graphics2D, surface: Surface, texture: Long) {
+        val width = surface.width
+        val height = surface.height
+        if (bitmap.width != width || bitmap.height != height) {
+            bitmap.allocPixelsFlags(ImageInfo.makeS32(width, height, ColorAlphaType.PREMUL), false)
         }
-            drawVolatileImage(vi, image)
-            when (vi.validate(swingLayerProperties.graphicsConfiguration)) {
-                VolatileImage.IMAGE_RESTORED -> drawVolatileImage(vi, image)
-                VolatileImage.IMAGE_INCOMPATIBLE -> vi = createVolatileImage(image)
+
+        surface.readPixels(bitmap, 0, 0)
+        val bufferPtr = bitmap.peekPixels()?.addr ?: throw RenderException("Can't get pixels address")
+        bufferedImage = createImageFromBytes(bufferPtr, width, height)
+        drawImage(g, bufferedImage)
     }
-            g.drawImage(vi, 0, 0, null)
-        } while (vi!!.contentsLost())
 
-        volatileImage = vi
+    override fun dispose() {
+        bitmap.close()
     }
 
     private fun createImageFromBytes(
-        bytes: ByteArray,
+        pBytes: Long,
         width: Int,
         height: Int,
-        dirtyRectangles: List<Rectangle>
     ): BufferedImage {
-        val src = ByteBuffer.wrap(bytes)
-        if (bufferedImage == null || bufferedImage?.width != width || bufferedImage?.height != height) {
-            bufferedImage?.flush()
+        if (bufferedImage.width != width || bufferedImage.height != height) {
             bufferedImage = BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB_PRE)
-            bufferedImageGraphics = bufferedImage?.createGraphics()
-        } else {
-            bufferedImageGraphics?.clearRect(0,0, width, height)
         }
-        val image = bufferedImage!!
+        val image = bufferedImage
 
         val dstData = (image.raster.dataBuffer as DataBufferInt).data
+        val src = BufferUtil.getByteBufferFromPointer(pBytes, width * height * 4)
         val srcData: IntBuffer = src.order(ByteOrder.LITTLE_ENDIAN).asIntBuffer()
-        for (rect in dirtyRectangles) {
-            if (rect.width < image.width) {
-                for (line in rect.y until rect.y + rect.height) {
-                    val offset: Int = line * image.width + rect.x
-                    srcData.position(offset)[dstData, offset, min(
-                        rect.width.toDouble(),
-                        (src.capacity() - offset).toDouble()
-                    ).toInt()]
-                }
-            } else { // optimized for a buffer wide dirty rect
-                val offset: Int = rect.y * image.width
-                srcData.position(offset)[dstData, offset, min(
-                    (rect.height * image.width).toDouble(),
-                    (src.capacity() - offset).toDouble()
-                ).toInt()]
-            }
-        }
+        srcData.position(0).get(dstData, 0, min(image.height * image.width, srcData.capacity()))
 
         return image
     }
 
-    private fun createVolatileImage(image: BufferedImage): VolatileImage {
-        val vi = swingLayerProperties.graphicsConfiguration.createCompatibleVolatileImage(
-            swingLayerProperties.width,
-            swingLayerProperties.height,
-            Transparency.TRANSLUCENT
-        )
-        drawVolatileImage(vi, image)
-        return vi
-    }
-
-    private fun drawVolatileImage(vi: VolatileImage, image: BufferedImage) {
-        val g = vi.graphics.create() as Graphics2D
-        try {
-            g.background = Color(0, 0, 0, 0)
-            g.composite = AlphaComposite.Src
-            g.clearRect(0, 0, swingLayerProperties.width, swingLayerProperties.height)
-            val imageClipRectangle = Rectangle(0, 0, swingLayerProperties.width, swingLayerProperties.height)
-            drawImage(g, image, sourceBounds = imageClipRectangle)
-        } finally {
-            g.dispose()
-        }
-    }
-
     private fun drawImage(
         g: Graphics,
         image: Image,

skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/SoftwareSwingRedrawer.kt

@@ -13,7 +13,7 @@ import java.awt.Graphics2D
  * Content to draw is provided by [SkikoRenderDelegate].
  *
  * @see SwingRedrawerBase
- * @see SwingOffscreenDrawer
+ * @see SoftwareSwingPainter
  */
 internal class SoftwareSwingRedrawer(
     swingLayerProperties: SwingLayerProperties,
@@ -28,7 +28,7 @@ internal class SoftwareSwingRedrawer(
         onDeviceChosen("Software")
     }
 
-    private val swingOffscreenDrawer = SwingOffscreenDrawer(swingLayerProperties)
+    private val painter: SwingPainter = SoftwareSwingPainter(swingLayerProperties)
 
     private val storage = Bitmap()
 
@@ -39,6 +39,7 @@ internal class SoftwareSwingRedrawer(
     override fun dispose() {
         super.dispose()
         storage.close()
+        painter.dispose()
     }
 
     override fun onRender(g: Graphics2D, width: Int, height: Int, nanoTime: Long) = autoCloseScope {
@@ -46,19 +47,20 @@ internal class SoftwareSwingRedrawer(
             storage.allocPixelsFlags(ImageInfo.makeS32(width, height, ColorAlphaType.PREMUL), false)
         }
 
-        val canvas = Canvas(storage, SurfaceProps(pixelGeometry = PixelGeometry.UNKNOWN)).autoClose()
-        canvas.clear(Color.TRANSPARENT)
-        renderDelegate.onRender(canvas, width, height, nanoTime)
+        val pixelsPointer = storage.peekPixels()?.addr!!
+        val surface = Surface.makeRasterDirect(
+            imageInfo = storage.imageInfo,
+            pixelsPtr = pixelsPointer,
+            rowBytes = storage.rowBytes
+        ).autoClose()
 
-        flush(g)
-    }
+        surface.canvas.clear(Color.TRANSPARENT)
+        renderDelegate.onRender(surface.canvas, width, height, nanoTime)
 
-    private fun flush(g: Graphics2D) {
-        val width = storage.width
-        val height = storage.height
-        val bytes = storage.readPixels(storage.imageInfo, (width * 4), 0, 0)
-        if (bytes != null) {
-            swingOffscreenDrawer.draw(g, bytes, width, height)
+        flush(g, surface)
     }
+
+    private fun flush(g: Graphics2D, surface: Surface) = autoCloseScope() {
+        painter.paint(g, surface, 0)
     }
 }
\ No newline at end of file

skiko/src/awtMain/kotlin/org/jetbrains/skiko/swing/SwingPainter.kt

+package org.jetbrains.skiko.swing
+
+import org.jetbrains.skia.Surface
+import java.awt.Graphics2D
+
+/**
+ * Interface for rendering Skia surfaces onto an AWT/Swing `Graphics2D` instance.
+ *
+ * @see SoftwareSwingDrawer
+ */
+internal interface SwingPainter {
+    fun paint(g: Graphics2D, surface: Surface, texture: Long)
+    fun dispose()
+}