Cross compile native bridges (#227)

skiko/build.gradle.kts

@@ -62,6 +62,8 @@ val skiaCrossDownloadTasks: Map<Pair<OS, Arch>, Provider<File>> = crossTargets.k
     val out = skiko.dependenciesDir.resolve("skia/skia-$suffix.zip")
 
     tasks.register<Download>("skiaCrossDownload$suffix") {
+        onlyIf { skiko.skiaDir == null && !out.exists() }
+
         val release = skiko.skiaReleaseFor(osArch.first, osArch.second, buildType)
         src("https://github.com/JetBrains/skia-pack/releases/download/$release.zip")
         dest(out.absolutePath)
@@ -92,14 +94,15 @@ val skiaDirProviderForCrossTargets: Map<Pair<OS, Arch>, Provider<File>> = crossT
     }
 }
 
-val wasmCrossCompile = tasks.register<WasmCrossCompileTask>("wasmCrossCompile") {
+val wasmCrossCompile = tasks.register<CrossCompileTask>("wasmCrossCompile") {
     val osArch = OS.Wasm to Arch.Wasm
 
     val unzipper = skiaDirProviderForCrossTargets[osArch]!!
     dependsOn(unzipper)
     val unpackedSkia = unzipper.get()
 
-    targetArch.set(osArch.second)
+    compiler.set("emcc")
+    crossCompileTargetArch.set(osArch.second)
     buildVariant.set(buildType)
 
     sourceFiles =
@@ -108,38 +111,52 @@ val wasmCrossCompile = tasks.register<WasmCrossCompileTask>("wasmCrossCompile")
     if (skiko.includeTestHelpers) {
         sourceFiles += project.fileTree("src/commonTest/cpp") { include("**/*.cc") }
     }
+    outDir.set(project.layout.buildDirectory.dir("out/compile/${buildType.id}-${osArch.first.id}-${osArch.second.id}"))
+
+    includeHeadersNonRecursive(projectDir.resolve("src/commonMain/cpp"))
+    includeHeadersNonRecursive(skiaHeadersDirs(unpackedSkia))
+
+    flags.set(listOf(
+        *skiaPreprocessorFlags(),
+        "-DSK_SUPPORT_GPU=1"
+    ))
+}
+
+val linkWasm = tasks.register<LinkWasmTask>("linkWasm") {
+    val osArch = OS.Wasm to Arch.Wasm
+
+    dependsOn(wasmCrossCompile)
+
+    val unzipper = skiaDirProviderForCrossTargets[osArch]!!
+    dependsOn(unzipper)
+    val unpackedSkia = unzipper.get()
 
-    val skiaAFilesDir = unpackedSkia.resolve("out/${buildType.id}-${osArch.first.id}-${osArch.second.id}")
-    libFiles = project.fileTree(skiaAFilesDir)  { include("**/*.a") }
+    libFiles = project.fileTree(unpackedSkia)  { include("**/*.a") }
+    objectFiles = project.fileTree(wasmCrossCompile.map { it.outDir.get() }) { include("**/*.o") }
 
-    outDir.set(project.layout.buildDirectory.dir("out/${buildType.id}-${osArch.first.id}-${osArch.second.id}"))
     wasmFileName.set("skiko.wasm")
     jsFileName.set("skiko.js")
 
     skikoJsPrefix.set(project.layout.projectDirectory.file("src/jsMain/resources/setup.js"))
-
-    includeHeadersNonRecursive(projectDir.resolve("src/commonMain/cpp"))
-    includeHeadersNonRecursive(skiaHeadersDirs(unpackedSkia))
+    outDir.set(project.layout.buildDirectory.dir("out/link/${buildType.id}-${osArch.first.id}-${osArch.second.id}"))
 
     flags.set(listOf(
-        *skiaPreprocessorFlags(),
-        "-s", "USE_WEBGL2=1",
         "-l", "GL",
+        "-s", "USE_WEBGL2=1",
         "-s", "OFFSCREEN_FRAMEBUFFER=1",
-        "-DSK_SUPPORT_GPU=1",
-        "--extern-post-js",
-        skikoJsPrefix.get().asFile.absolutePath,
     ))
 
-    finalizedBy(replaceSymbolsInSkikoJsOutput)
-}
-
-val replaceSymbolsInSkikoJsOutput by project.tasks.registering {
     doLast {
-        val wasmTask = project.tasks.getByName("wasmCrossCompile") as WasmCrossCompileTask
-        val skikoJsFile: File = wasmTask.outDir.asFile.get().resolve("skiko.js")
-        val replacedContent = skikoJsFile.readText().replace("_org_jetbrains", "org_jetbrains")
-        skikoJsFile.writeText(replacedContent)
+        // skiko.js file is directly referenced in karma.config.d/wasm.js
+        // so symbols must be replaced right after linking
+        val jsFiles = outDir.asFile.get().walk()
+            .filter { it.isFile && it.name.endsWith(".js") }
+
+        for (jsFile in jsFiles) {
+            val originalContent = jsFile.readText()
+            val newContent = originalContent.replace("_org_jetbrains", "org_jetbrains")
+            jsFile.writeText(newContent)
+        }
     }
 }
 
@@ -180,7 +197,7 @@ kotlin {
         browser() {
             testTask {
                 testLogging.showStandardStreams = true
-                dependsOn(wasmCrossCompile)
+                dependsOn(linkWasm)
                 useKarma {
                     useChromeHeadless()
                 }
@@ -436,63 +453,56 @@ val generateVersion = project.tasks.register("generateVersion") {
 
 // Very hacky way to compile native bridges and add the
 // resulting object files into the final native klib.
-project.tasks.register<Exec>("nativeBridgesCompile") {
+val compileNativeBridges = project.tasks.register<CrossCompileTask>("compileNativeBridges") {
     dependsOn(skiaDir)
-    val inputDirs = mutableListOf(
-        "$projectDir/src/nativeMain/cpp/common",
-        "$projectDir/src/commonMain/cpp"
-    ).apply {
-        if (skiko.includeTestHelpers) add("$projectDir/src/commonTest/cpp/TestHelpers.cc")
-    }
-    val outDir = "$buildDir/nativeBridges/obj/$target"
-    val srcs = inputDirs
-        .findAllFiles(".cc")
-        .toTypedArray()
-    val outs = srcs
-        .map { it.substringAfterLast("/") }
-        .map { File(it).nameWithoutExtension }
-        .map { "$outDir/$it.o" }
-        .toTypedArray()
 
-    workingDir = File(outDir)
-    commandLine = listOf(
-        "clang++",
-        *targetArch.clangFlags,
-        *targetOs.clangFlags,
-        *buildType.clangFlags,
-        "-c",
+    compiler.set("clang++")
+    crossCompileTargetOS.set(targetOs)
+    crossCompileTargetArch.set(targetArch)
+    buildVariant.set(buildType)
+
+    sourceFiles =
+        project.fileTree("src/nativeMain/cpp") { include("**/*.cc") } +
+                project.fileTree("src/commonMain/cpp") { include("**/*.cc") }
+    if (skiko.includeTestHelpers) {
+        sourceFiles += project.fileTree("src/commonTest/cpp") { include("**/*.cc") }
+    }
+
+    outDir.set(project.layout.buildDirectory.dir("nativeBridges/obj/$target"))
+
+    includeHeadersNonRecursive(projectDir.resolve("src/commonMain/cpp"))
+    includeHeadersNonRecursive(skiaHeadersDirs(skiaDir.get()))
+
+    flags.set(listOf(
         "-DSK_SHAPER_CORETEXT_AVAILABLE",
         "-DSK_BUILD_FOR_MAC",
         "-DSK_METAL",
-        "-I$projectDir/src/commonMain/cpp",
-        *includeHeadersFlags(skiaHeadersDirs(skiaDir.get())),
         *skiaPreprocessorFlags(),
-        *srcs
-    )
-    file(outDir).mkdirs()
-    inputs.files(srcs)
-    outputs.files(outs)
+    ))
 }
 
-project.tasks.register<Exec>("nativeBridgesLink") {
-    dependsOn(project.tasks.getByName("nativeBridgesCompile"))
-    inputs.files(project.tasks.getByName("nativeBridgesCompile").outputs)
+val linkNativeBridges = project.tasks.register<Exec>("linkNativeBridges") {
+    dependsOn(compileNativeBridges)
+    val objectFilesDir = compileNativeBridges.map { it.outDir.get() }
+    val objectFiles = project.fileTree(objectFilesDir) {
+        include("**/*.o")
+    }
+    inputs.files(objectFiles)
 
     val outDir = "$buildDir/nativeBridges/static/$target"
-    val srcs = inputs.files.files
-        .map { it.absolutePath }
-        .toTypedArray()
     val staticLib = "$outDir/skiko-native-bridges-$target.a"
     workingDir = File(outDir)
-    commandLine = listOf(
-        "libtool",
+    executable = "libtool"
+    argumentProviders.add {
+        listOf(
             "-static",
             "-o",
-        staticLib,
-        *srcs
+            staticLib
         )
+    }
+    argumentProviders.add { objectFiles.files.map { it.absolutePath } }
     file(outDir).mkdirs()
-    outputs.files(staticLib)
+    outputs.dir(outDir)
 }
 
 fun localSign(signer: String, lib: File): File {
@@ -749,17 +759,13 @@ val skikoJvmRuntimeJar by project.tasks.registering(Jar::class) {
 }
 
 val skikoWasmJar by project.tasks.registering(Jar::class) {
-    dependsOn(wasmCrossCompile)
+    dependsOn(linkWasm)
     // We produce jar that contains .js of wrapper/bindings and .wasm with Skia + bindings.
-    val wasmOutDir = wasmCrossCompile.map { it.outDir }
+    val wasmOutDir = linkWasm.map { it.outDir }
 
     from(wasmOutDir) {
         include("*.wasm")
-    }
-
-    from(wasmOutDir) {
         include("*.js")
-        filter { line -> line.replace("_org_jetbrains_", "org_jetbrains_") }
     }
 
     archiveBaseName.set("skiko-wasm")
@@ -930,7 +936,7 @@ afterEvaluate {
         source(generatedKotlin)
 
         if (supportNative) {
-            dependsOn(tasks.getByName("nativeBridgesLink"))
+            dependsOn(linkNativeBridges)
         }
     }
 }

skiko/buildSrc/src/main/kotlin/WasmCrossCompileTask.kt → skiko/buildSrc/src/main/kotlin/CrossCompileTask.kt

 import org.gradle.api.DefaultTask
-import org.gradle.api.file.*
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.file.FileCollection
 import org.gradle.api.provider.ListProperty
 import org.gradle.api.provider.Property
 import org.gradle.api.tasks.*
 import org.gradle.process.ExecOperations
-
-import javax.inject.Inject
-
+import org.gradle.work.ChangeType
+import org.gradle.work.FileChange
+import org.gradle.work.Incremental
+import org.gradle.work.InputChanges
 import java.io.File
 import java.util.LinkedHashSet
+import java.util.concurrent.Callable
+import javax.inject.Inject
 
-abstract class WasmCrossCompileTask : DefaultTask() {
+abstract class CrossCompileTask : DefaultTask() {
     @get:Inject
     abstract val execOperations: ExecOperations
 
     @get:Input
-    abstract val targetArch: Property<Arch>
+    @get:Optional
+    abstract val crossCompileTargetOS: Property<OS>
+
+    @get:Input
+    abstract val crossCompileTargetArch: Property<Arch>
 
     @get:Input
     abstract val buildVariant: Property<SkiaBuildType>
@@ -24,28 +32,14 @@ abstract class WasmCrossCompileTask : DefaultTask() {
     abstract val flags: ListProperty<String>
 
     @get:InputFiles
+    @get:Incremental
     lateinit var sourceFiles: FileCollection
 
-    @get:InputFiles
-    lateinit var libFiles: FileCollection
-
-    @get:Optional
-    @get:Input
-    abstract val wasmFileName: Property<String>
-
-    @get:Optional
-    @get:Input
-    abstract val jsFileName: Property<String>
-
     @get:OutputDirectory
     abstract val outDir: DirectoryProperty
 
-    @get:InputFile
-    @get:Optional
-    abstract val skikoJsPrefix: RegularFileProperty
-
     @get:Input
-    var compiler: String = "emcc"
+    abstract val compiler: Property<String>
 
     /**
      * Used only for up-to-date checks of headers' content
@@ -54,8 +48,9 @@ abstract class WasmCrossCompileTask : DefaultTask() {
      */
     @Suppress("UNUSED")
     @get:InputFiles
-    val headerFiles: FileCollection
-        get() {
+    @get:Incremental
+    val headerFiles: FileCollection =
+        project.files(Callable {
             fun File.isHeaderFile(): Boolean =
                 isFile && name.endsWith(".h", ignoreCase = true)
 
@@ -68,8 +63,8 @@ abstract class WasmCrossCompileTask : DefaultTask() {
                     }
                 }
             }
-            return project.files(headers)
-        }
+            headers
+        })
 
     @get:Internal
     internal val headersDirs = LinkedHashSet<File>()
@@ -81,24 +76,76 @@ abstract class WasmCrossCompileTask : DefaultTask() {
     }
 
     @TaskAction
-    fun exec() {
+    open fun run(inputChanges: InputChanges) {
         execOperations.exec {
-            executable = compiler
-            val args = argumentProviders
+            val sourcesToCompile = determineSourcesToCompile(inputChanges)
+            executable = compiler.get()
+            args = arrayListOf<String>().also { args ->
+                configureCompilerArgs(args)
+                args.addAll(sourcesToCompile.map { it.absolutePath })
+            }
 
-            args.add { headersDirs.map { "-I${it.absolutePath}" } }
+            workingDir = outDir.get().asFile
+            // todo: log args to file system
+        }
+    }
 
-            args.add {
-                // todo: ensure that flags do not start with '-I' (all headers should be added via [headersDirs])
-                val bt = buildVariant.get()
-                (flags.get() + targetArch.get().clangFlags + bt.flags + bt.clangFlags).asIterable()
+    private sealed class Mode {
+        class Incremental(val changes: Iterable<FileChange>) : Mode()
+        class NonIncremental(val reason: String) : Mode()
+    }
+
+    private fun determineSourcesToCompile(inputChanges: InputChanges): Collection<File> {
+        val compilationMode = analyzeIncrementalChanges(inputChanges)
+        val outDir = outDir.get().asFile
+        return when (compilationMode) {
+            is Mode.Incremental -> {
+                val sourcesToCompile = arrayListOf<File>()
+                for (change in compilationMode.changes) {
+                    val outdatedOutputFile = outDir.resolve(change.file.nameWithoutExtension + ".o")
+                    if (outdatedOutputFile.exists()) {
+                        // `outdatedOutputFile` might not exist,
+                        // when `change.file` is a new file, which was not compiled
+                        // todo: log in verbose mode
+                        outdatedOutputFile.delete()
+                    }
+                    if (change.changeType != ChangeType.REMOVED) {
+                        sourcesToCompile.add(change.file)
+                    }
+                }
+                logger.warn("Compiling ${sourcesToCompile.size} files incrementally")
+                sourcesToCompile
+            }
+            is Mode.NonIncremental -> {
+                outDir.deleteRecursively()
+                outDir.mkdirs()
+                logger.warn("Recompiling all files: ${compilationMode.reason}")
+                sourceFiles.files
+            }
+        }
+    }
+    private fun analyzeIncrementalChanges(inputChanges: InputChanges): Mode {
+        if (!inputChanges.isIncremental) {
+            return Mode.NonIncremental("input changes are not incremental")
         }
 
-            args.add { sourceFiles.files.map { it.absolutePath } }
-            args.add { libFiles.files.map { it.absolutePath } }
+        if (inputChanges.getFileChanges(headerFiles).any()) {
+            return Mode.NonIncremental("header files are modified or removed")
+        }
 
-            args.add { listOf("-o", outDir.resolveToAbsolutePath(wasmFileName)) }
-            args.add { listOf("-o", outDir.resolveToAbsolutePath(jsFileName)) }
+        return Mode.Incremental(inputChanges.getFileChanges(sourceFiles))
     }
+
+    open fun configureCompilerArgs(args: MutableList<String>) {
+        args.add("-c")
+        args.addAll(headersDirs.map { "-I${it.absolutePath}" })
+
+        // todo: ensure that flags do not start with '-I' (all headers should be added via [headersDirs])
+        args.addAll(flags.get())
+        args.addAll(crossCompileTargetOS.orNull?.clangFlags ?: arrayOf())
+        args.addAll(crossCompileTargetArch.get().clangFlags)
+        val bt = buildVariant.get()
+        args.addAll(bt.flags)
+        args.addAll(bt.clangFlags)
     }
 }
\ No newline at end of file

skiko/buildSrc/src/main/kotlin/LinkWasmTask.kt

+import org.gradle.api.DefaultTask
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.file.FileCollection
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.provider.ListProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.*
+import org.gradle.process.ExecOperations
+import javax.inject.Inject
+
+abstract class LinkWasmTask : DefaultTask() {
+    @get:Inject
+    abstract val execOperations: ExecOperations
+
+    @get:InputFiles
+    lateinit var libFiles: FileCollection
+
+    @get:InputFiles
+    lateinit var objectFiles: FileCollection
+
+    @get:Input
+    abstract val wasmFileName: Property<String>
+
+    @get:Input
+    abstract val jsFileName: Property<String>
+
+    @get:InputFile
+    @get:Optional
+    abstract val skikoJsPrefix: RegularFileProperty
+
+    @get:OutputDirectory
+    abstract val outDir: DirectoryProperty
+
+    @get:Input
+    abstract val flags: ListProperty<String>
+
+    @TaskAction
+    fun run() {
+        outDir.get().asFile.apply {
+            deleteRecursively()
+            mkdirs()
+        }
+
+        execOperations.exec {
+            executable = "emcc"
+            args = arrayListOf<String>().also { configureCompilerArgs(it) }
+
+            workingDir = outDir.get().asFile
+            // todo: log args to file system
+        }
+    }
+
+    fun configureCompilerArgs(args: MutableList<String>) {
+        args.addAll(flags.get())
+        args.addAll(objectFiles.files.map { it.absolutePath })
+        args.addAll(libFiles.files.map { it.absolutePath })
+        args.add("--extern-post-js")
+        args.add(skikoJsPrefix.get().asFile.absolutePath)
+        args.add("-o")
+        args.add(outDir.resolveToAbsolutePath(wasmFileName))
+        args.add("-o")
+        args.add(outDir.resolveToAbsolutePath(jsFileName))
+    }
+}
\ No newline at end of file

skiko/buildSrc/src/main/kotlin/utils.kt

+import org.gradle.api.Task
 import org.gradle.api.file.FileSystemLocation
+import org.gradle.api.model.ObjectFactory
+import org.gradle.api.provider.Property
 import org.gradle.api.provider.Provider
 
+internal val Provider<out FileSystemLocation>.absolutePath: String
+        get() = get().asFile.absolutePath
+
 internal fun Provider<out FileSystemLocation>.resolveToAbsolutePath(path: Provider<String>): String =
     get().asFile.absoluteFile.resolve(path.get()).absolutePath
+
+inline fun <reified T : Any> ObjectFactory.nullableProperty(): Property<T?> =
+    property(T::class.java)
+
+inline fun <reified T : Any> ObjectFactory.notNullProperty(): Property<T> =
+    property(T::class.java)
+
+inline fun <reified T : Any> ObjectFactory.notNullProperty(defaultValue: T): Property<T> =
+    property(T::class.java).value(defaultValue)
+
+inline fun <reified T> Task.provider(noinline fn: () -> T): Provider<T> =
+    project.provider(fn)

skiko/karma.config.d/wasm.js

@@ -6,7 +6,7 @@ config.browserConsoleLogOptions.level = "debug";
 
 const basePath = config.basePath;
 const projectPath = path.resolve(basePath, "..", "..", "..", "..", "..");
-const wasmPath = path.resolve(projectPath, "build", "out", "Release-wasm-wasm")
+const wasmPath = path.resolve(projectPath, "build", "out", "link", "Release-wasm-wasm")
 
 const debug = message => console.log(`[karma-config] ${message}`);