Introduce vsync-based and overcommitment throttling to metal redrawer on macOS (#753)

* Add repro

* Add ignore

* Remove redundant retain

* Add DisplayLinkThrottler implementation

* Create Kotlin wrapper and JNI wrapper.

* Integrate throttler into redrawer

* Release CVDisplayLink in case of error

* Fix wrong screen dictionary retrieval.

* Remove temporary changes.

* Rename test.

* Add overcommitment prevention mechanism (#754)

* Add overcommitment prevention mechanism

* Replace hardcoded value with CAMetalLayer property

* Update skiko/src/awtMain/objectiveC/macos/MetalContextHandler.mm
Co-authored-by: Igor Demin <igordmn@users.noreply.github.com>

---------
Co-authored-by: Igor Demin <igordmn@users.noreply.github.com>

* Remove _displayLinkOk (#756)

* Remove _displayLinkOk

* Discussions

* Fix build

---------
Co-authored-by: Igor Demin <igordmn@users.noreply.github.com>

skiko/src/awtMain/kotlin/org/jetbrains/skiko/redrawer/DisplayLinkThrottler.kt

+package org.jetbrains.skiko.redrawer
+
+import org.jetbrains.skiko.Library
+
+internal class DisplayLinkThrottler {
+    private val implPtr = create()
+
+    internal fun dispose() = dispose(implPtr)
+
+    /*
+     * Creates a DisplayLink if needed with refresh rate matching NSScreen of NSWindow passed in [windowPtr].
+     * If DisplayLink is already active, blocks until next vsync for physical screen of NSWindow passed in [windowPtr].
+     */
+    internal fun waitVSync(windowPtr: Long) = waitVSync(implPtr, windowPtr)
+
+    private external fun create(): Long
+
+    private external fun dispose(implPtr: Long)
+
+    private external fun waitVSync(implPtr: Long, windowPtr: Long)
+
+    companion object {
+        init {
+            Library.load()
+        }
+    }
+}
\ No newline at end of file

skiko/src/awtMain/kotlin/org/jetbrains/skiko/redrawer/MetalRedrawer.kt

@@ -62,6 +62,7 @@ internal class MetalRedrawer(
         }
 
     private val adapter = chooseMetalAdapter(properties.adapterPriority)
+    private val displayLinkThrottler = DisplayLinkThrottler()
 
     init {
         onDeviceChosen(adapter.name)
@@ -88,11 +89,17 @@ internal class MetalRedrawer(
         onContextInit()
     }
 
+    fun drawSync() {
+        layer.update(System.nanoTime())
+        performDraw()
+    }
+
     override fun dispose() = synchronized(drawLock) {
         frameDispatcher.cancel()
         contextHandler.dispose()
         disposeDevice(device.ptr)
         adapter.dispose()
+        displayLinkThrottler.dispose()
         _device = null
         super.dispose()
     }
@@ -142,6 +149,11 @@ internal class MetalRedrawer(
 
     private fun performDraw() = synchronized(drawLock) {
         if (!isDisposed) {
+            // Wait for vsync because:
+            // - macOS drops the second/next drawables if they are sent in the same vsync
+            // - it makes frames consistent and limits FPS
+            displayLinkThrottler.waitVSync(windowPtr = layer.windowHandle)
+
             val handle = startRendering()
             try {
                 contextHandler.draw()

skiko/src/awtMain/objectiveC/macos/DisplayLinkThrottler.mm

+#ifdef SK_METAL
+
+#import <jawt.h>
+#import <jawt_md.h>
+#import <QuartzCore/QuartzCore.h>
+#import <AppKit/AppKit.h>
+#import <stdatomic.h>
+
+@interface DisplayLinkThrottler : NSObject
+
+- (void)onVSync;
+
+@end
+
+static CVReturn displayLinkCallback(CVDisplayLinkRef displayLink, const CVTimeStamp *now, const CVTimeStamp *outputTime, CVOptionFlags flagsIn, CVOptionFlags *flagsOut, void *displayLinkContext) {
+    DisplayLinkThrottler *throttler = (__bridge DisplayLinkThrottler *)displayLinkContext;
+
+    [throttler onVSync];
+
+    return kCVReturnSuccess;
+}
+
+@implementation DisplayLinkThrottler {
+    NSScreen *_displayLinkScreen;
+    CVDisplayLinkRef _displayLink;
+    NSConditionLock *_vsyncConditionLock;
+}
+
+- (instancetype)init {
+    self = [super init];
+
+    if (self) {
+        _displayLinkScreen = nil;
+        _displayLink = nil;
+        _vsyncConditionLock = [[NSConditionLock alloc] initWithCondition: 1];
+    }
+
+    return self;
+}
+
+- (void)onVSync {
+    /// Lock condition lock and immediately unlock setting condition variable to 1 (can render now)
+    [_vsyncConditionLock lock];
+    [_vsyncConditionLock unlockWithCondition:1];
+}
+
+- (void)waitVSync {
+    /// If display link construction was corrupted, don't perform any waiting
+    if (!_displayLink) {
+        return;
+    }
+
+    /// Wait until `onVSync` signals 1, then immediately lock, set it to 0 and unlock again.
+    [_vsyncConditionLock lockWhenCondition:1];
+    [_vsyncConditionLock unlockWithCondition:0];
+}
+
+- (void)invalidateDisplayLink {
+    if (_displayLink) {
+        CVDisplayLinkStop(_displayLink);
+        CVDisplayLinkRelease(_displayLink);
+        _displayLink = nil;
+        _displayLinkScreen = nil;
+    }
+}
+
+- (void)setupDisplayLinkForWindow:(NSWindow *)window {
+    NSScreen *screen = window.screen;
+
+    if (!screen) {
+        /// window is not yet attached to screen (or window is nil)
+        return;
+    }
+
+    if ([screen isEqualTo: _displayLinkScreen]) {
+        /// display link is already active for this screen, do nothing
+        return;
+    }
+
+    [self invalidateDisplayLink];
+
+    NSDictionary* screenDescription = [screen deviceDescription];
+    NSNumber* screenID = [screenDescription objectForKey:@"NSScreenNumber"];
+
+    _displayLink = [self createDisplayLinkForScreen:screenID];
+    _displayLinkScreen = screen;
+}
+
+- (CVDisplayLinkRef)createDisplayLinkForScreen:(NSNumber *)screenID {
+    CVReturn result;
+    CVDisplayLinkRef displayLink;
+
+    result = CVDisplayLinkCreateWithCGDisplay([screenID unsignedIntValue], &displayLink);
+
+    if (result != kCVReturnSuccess) {
+        return nil;
+    }
+
+    result = CVDisplayLinkSetOutputCallback(displayLink, &displayLinkCallback, (__bridge void *)(self));
+
+    if (result != kCVReturnSuccess) {
+        CVDisplayLinkRelease(displayLink);
+        return nil;
+    }
+
+    result = CVDisplayLinkStart(displayLink);
+
+    if (result != kCVReturnSuccess) {
+        CVDisplayLinkRelease(displayLink);
+        return nil;
+    }
+
+    return displayLink;
+}
+
+- (void)dealloc {
+    [self invalidateDisplayLink];
+}
+
+@end
+
+extern "C" {
+
+JNIEXPORT jlong JNICALL Java_org_jetbrains_skiko_redrawer_DisplayLinkThrottler_create(JNIEnv *env, jobject obj) {
+    DisplayLinkThrottler *throttler = [[DisplayLinkThrottler alloc] init];
+
+    return (jlong) (__bridge_retained void *) throttler;
+}
+
+JNIEXPORT void JNICALL Java_org_jetbrains_skiko_redrawer_DisplayLinkThrottler_dispose(JNIEnv *env, jobject obj, jlong throttlerPtr) {
+    DisplayLinkThrottler *throttler = (__bridge_transfer DisplayLinkThrottler *) (void *) throttlerPtr;
+    /// throttler will be released by ARC and deallocated in the end of this scope.
+}
+
+JNIEXPORT void JNICALL Java_org_jetbrains_skiko_redrawer_DisplayLinkThrottler_waitVSync(JNIEnv *env, jobject obj, jlong throttlerPtr, jlong windowPtr) {
+    DisplayLinkThrottler *throttler = (__bridge DisplayLinkThrottler *) (void *) throttlerPtr;
+    NSWindow *window = (__bridge NSWindow *) (void *) windowPtr;
+
+    [throttler setupDisplayLinkForWindow:window];
+    [throttler waitVSync];
+}
+
+}
+
+#endif // SK_METAL
\ No newline at end of file

skiko/src/awtMain/objectiveC/macos/MetalContextHandler.mm

@@ -32,8 +32,14 @@ JNIEXPORT jlong JNICALL Java_org_jetbrains_skiko_context_MetalContextHandler_mak
         MetalDevice *device = (__bridge MetalDevice *) (void *) devicePtr;
         GrBackendRenderTarget* renderTarget = NULL;
 
+        /// If we have more than `maximumDrawableCount` command buffers inflight, wait until one of them finishes work.
+        dispatch_semaphore_wait(device.inflightSemaphore, DISPATCH_TIME_FOREVER);
+
         id<CAMetalDrawable> currentDrawable = [device.layer nextDrawable];
         if (!currentDrawable) {
+            /// Signal semaphore immediately, no command buffer will be commited
+            dispatch_semaphore_signal(device.inflightSemaphore);
+
             return NULL;
         }
         device.drawableHandle = currentDrawable;
@@ -56,6 +62,10 @@ JNIEXPORT void JNICALL Java_org_jetbrains_skiko_context_MetalContextHandler_fini
             id<MTLCommandBuffer> commandBuffer = [device.queue commandBuffer];
             commandBuffer.label = @"Present";
             [commandBuffer presentDrawable:currentDrawable];
+            [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
+                /// commands have completed, allow next waiting (if any) to start encoding new work to gpu
+                dispatch_semaphore_signal(device.inflightSemaphore);
+            }];
             [commandBuffer commit];
             device.drawableHandle = nil;
         }

skiko/src/awtMain/objectiveC/macos/MetalDevice.h

@@ -9,10 +9,11 @@
 @interface MetalDevice : NSObject
 
 @property (weak) CALayer *container;
-@property (retain, strong) AWTMetalLayer *layer;
-@property (retain, strong) id<MTLDevice> adapter;
-@property (retain, strong) id<MTLCommandQueue> queue;
-@property (retain, strong) id<CAMetalDrawable> drawableHandle;
+@property (strong) AWTMetalLayer *layer;
+@property (strong) id<MTLDevice> adapter;
+@property (strong) id<MTLCommandQueue> queue;
+@property (strong) id<CAMetalDrawable> drawableHandle;
+@property (strong) dispatch_semaphore_t inflightSemaphore;
 
 @end
 

skiko/src/awtMain/objectiveC/macos/MetalRedrawer.mm

@@ -104,6 +104,9 @@ JNIEXPORT jlong JNICALL Java_org_jetbrains_skiko_redrawer_MetalRedrawer_createMe
         device.layer.opaque = NO;
         device.layer.framebufferOnly = NO;
 
+        /// max inflight command buffers count matches swapchain size to avoid overcommitment
+        device.inflightSemaphore = dispatch_semaphore_create(device.layer.maximumDrawableCount);
+
         if (transparency)
         {
             NSWindow* window = (__bridge NSWindow*) (void *) windowPtr;

skiko/src/awtTest/kotlin/org/jetbrains/skiko/SkiaLayerTest.kt

@@ -2,6 +2,7 @@ package org.jetbrains.skiko
 
 import kotlinx.coroutines.CompletableDeferred
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
 import kotlinx.coroutines.yield
 import org.jetbrains.skia.Canvas
 import org.jetbrains.skia.FontMgr
@@ -12,6 +13,7 @@ import org.jetbrains.skia.paragraph.ParagraphBuilder
 import org.jetbrains.skia.paragraph.ParagraphStyle
 import org.jetbrains.skia.paragraph.TextStyle
 import org.jetbrains.skiko.context.JvmContextHandler
+import org.jetbrains.skiko.redrawer.MetalRedrawer
 import org.jetbrains.skiko.redrawer.Redrawer
 import org.jetbrains.skiko.util.ScreenshotTestRule
 import org.jetbrains.skiko.util.UiTestScope
@@ -19,14 +21,13 @@ import org.jetbrains.skiko.util.UiTestWindow
 import org.jetbrains.skiko.util.uiTest
 import org.junit.Assert.assertEquals
 import org.junit.Assume.assumeTrue
+import org.junit.Ignore
 import org.junit.Rule
 import org.junit.Test
 import java.awt.BorderLayout
 import java.awt.Color
 import java.awt.Dimension
-import java.awt.event.ComponentAdapter
-import java.awt.event.ComponentEvent
-import java.awt.event.WindowEvent
+import java.awt.event.*
 import javax.swing.JFrame
 import javax.swing.JLayeredPane
 import javax.swing.JPanel
@@ -35,6 +36,8 @@ import javax.swing.WindowConstants
 import kotlin.random.Random
 import kotlin.test.assertNotNull
 import kotlin.test.assertTrue
+import kotlin.time.Duration
+import kotlin.time.ExperimentalTime
 
 @Suppress("BlockingMethodInNonBlockingContext", "SameParameterValue")
 class SkiaLayerTest {
@@ -56,6 +59,69 @@ class SkiaLayerTest {
     @get:Rule
     val screenshots = ScreenshotTestRule()
 
+    @OptIn(ExperimentalTime::class)
+    @Ignore
+    @Test
+    fun `metal drawables not lost`() = uiTest {
+        val window = UiTestWindow(
+            properties = SkiaLayerProperties(
+                isVsyncEnabled = true
+            )
+        )
+        val colors = arrayOf(
+            Color.RED,
+            Color.GREEN,
+            Color.BLUE,
+            Color.YELLOW
+        )
+
+        var counter1 = 0
+        var counter2 = 0
+        val paint = Paint()
+
+        try {
+            window.setLocation(200, 200)
+            window.setSize(400, 600)
+            window.defaultCloseOperation = WindowConstants.EXIT_ON_CLOSE
+            window.layer.skikoView = object : SkikoView {
+                override fun onRender(canvas: Canvas, width: Int, height: Int, nanoTime: Long) {
+                    val c1 = counter1
+                    val c2 = counter2
+
+                    paint.color = colors[c1.mod(colors.size)].rgb
+                    canvas.drawRect(Rect(0f, 0f, width.toFloat(), height / 2f), paint)
+
+                    paint.color = colors[c2.mod(colors.size)].rgb
+                    canvas.drawRect(Rect(0f, height / 2f, width.toFloat(), height.toFloat()), paint)
+                }
+            }
+            window.isVisible = true
+
+            window.addKeyListener(object : KeyAdapter() {
+                override fun keyTyped(e: KeyEvent?) {
+                    launch {
+                        val redrawer = window.layer.redrawer as MetalRedrawer
+                        redrawer.drawSync()
+                        counter1 += 1
+                        redrawer.drawSync()
+                        counter2 += 1
+                        redrawer.drawSync()
+                    }
+                }
+            });
+
+            window.addWindowListener(object : WindowAdapter() {
+                override fun windowActivated(e: WindowEvent?) {
+                    window.requestFocus()
+                }
+            })
+
+            delay(Duration.INFINITE)
+        } finally {
+            window.close()
+        }
+    }
+
     @Test
     fun `should not leak native windows`() = uiTest {
         assumeTrue(hostOs.isMacOS)