Cross-compilation effort (#210)

* Cross-compilation effort

* Missing dep

* Polish wash cross compile task

* Fix code review issues
Co-authored-by: Alexey Tsvetkov <alexey.tsvetkov@jetbrains.com>

skiko/build.gradle.kts

@@ -2,6 +2,7 @@ import de.undercouch.gradle.tasks.download.Download
 import org.gradle.crypto.checksum.Checksum
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import org.jetbrains.kotlin.gradle.tasks.KotlinNativeCompile
+import org.jetbrains.kotlin.utils.keysToMap
 
 plugins {
     kotlin("multiplatform") version "1.5.10"
@@ -13,6 +14,10 @@ plugins {
 
 val coroutinesVersion = "1.5.0"
 
+fun targetSuffix(os: OS, arch: Arch): String {
+    return "${os.id}_${arch.id}"
+}
+
 buildscript {
     dependencies {
         classpath("org.kohsuke:github-api:1.116")
@@ -36,56 +41,96 @@ repositories {
 }
 
 val skiaZip = run {
-    val zipName = skiko.skiaReleaseFor(targetOs, targetArch) + ".zip"
+    val zipName = skiko.skiaReleaseFor(targetOs, targetArch, buildType) + ".zip"
     val zipFile = skiko.dependenciesDir.resolve("skia/${zipName.substringAfterLast('/')}")
 
     tasks.register("downloadSkia", Download::class) {
         onlyIf { skiko.skiaDir == null && !zipFile.exists() }
-        inputs.property("skia.release.for.target.os", skiko.skiaReleaseFor(targetOs, targetArch))
+        inputs.property("skia.release.for.target.os", skiko.skiaReleaseFor(targetOs, targetArch, buildType))
         src("https://github.com/JetBrains/skia-pack/releases/download/$zipName")
         dest(zipFile)
         onlyIfModified(true)
     }.map { zipFile }
 }
 
-val skiaWasmZip = run {
-    val release = skiko.skiaReleaseFor(OS.Wasm, Arch.Wasm)
-    val zipName = "$release.zip"
-    val zipFile = skiko.dependenciesDir.resolve("skia/${zipName.substringAfterLast('/')}")
-    tasks.register("downloadSkiaWasm", Download::class) {
-        onlyIf { skiko.skiaDir == null && !zipFile.exists() }
-        inputs.property("skia.release.for.wasm", release)
-        src("https://github.com/JetBrains/skia-pack/releases/download/$zipName")
-        dest(zipFile)
+val crossTargets = listOf(
+    OS.Wasm to Arch.Wasm
+)
+
+val skiaCrossDownloadTasks: Map<Pair<OS, Arch>, Provider<File>> = crossTargets.keysToMap { osArch ->
+    val suffix = targetSuffix(osArch.first, osArch.second)
+    val out = skiko.dependenciesDir.resolve("skia/skia-$suffix.zip")
+
+    tasks.register<Download>("skiaCrossDownload$suffix") {
+        val release = skiko.skiaReleaseFor(osArch.first, osArch.second, buildType)
+        src("https://github.com/JetBrains/skia-pack/releases/download/$release.zip")
+        dest(out.absolutePath)
         onlyIfModified(true)
-    }.map { zipFile }
+    }.map { out.absoluteFile }
 }
 
-fun AbstractCopyTask.configureSkiaCopy(targetDir: File) {
-    into(targetDir)
-}
+val skiaDirProviderForCrossTargets: Map<Pair<OS, Arch>, Provider<File>> = crossTargets.keysToMap { osArch ->
+    val suffix = targetSuffix(osArch.first, osArch.second)
 
-val skiaDir = run {
     if (skiko.skiaDir != null) {
-        tasks.register("skiaDir", DefaultTask::class) {
+        tasks.register("skiaCrossDir$suffix", DefaultTask::class) {
             // dummy task to simplify usage of the resulting provider (see `else` branch)
             // if a file provider is not created from a task provider,
             // then it cannot be used instead of a task in `dependsOn` clauses of other tasks.
             // e.g. the resulting `skiaDir` could not be used in `dependsOn` of CppCompile configuration
             enabled = false
-        }.map { skiko.skiaDir!! }
+        }.map { skiko.skiaDir!!.absoluteFile }
     } else {
-        val targetDir = skiko.dependenciesDir.resolve("skia/skia")
-        tasks.register("unzipSkia", Copy::class) {
-            from(skiaZip.map { zipTree(it) })
-            configureSkiaCopy(targetDir)
-        }.map { targetDir }
+        val targetDir = skiko.dependenciesDir.resolve("skia/skia-$suffix")
+        tasks.register("skiaCrossUnzip$suffix", Copy::class) {
+            val downloader = skiaCrossDownloadTasks[osArch]!!
+            dependsOn(downloader)
+            from(downloader.map { zipTree(it) })
+            into(targetDir)
+            destinationDir = targetDir.absoluteFile
+        }.map { it.destinationDir.absoluteFile }
     }
 }
 
-val skiaWasmDir = run {
+val wasmCrossCompile = tasks.register<WasmCrossCompileTask>("wasmCrossCompile") {
+    val osArch = OS.Wasm to Arch.Wasm
+
+    val unzipper = skiaDirProviderForCrossTargets[osArch]!!
+    dependsOn(unzipper)
+    val unpackedSkia = unzipper.get()
+
+    targetArch.set(osArch.second)
+    buildVariant.set(buildType)
+
+    sourceFiles =
+        project.fileTree("src/jsMain/cpp") { include("**/*.cc") } +
+        project.fileTree("src/commonMain/cpp") { include("**/*.cc") }
+    val skiaAFilesDir = unpackedSkia.resolve("out/${buildType.id}-${osArch.first.id}-${osArch.second.id}")
+    libFiles = project.fileTree(skiaAFilesDir)  { include("**/*.a") }
+
+    outDir.set(project.layout.buildDirectory.dir("out/${buildType.id}-${osArch.first.id}-${osArch.second.id}"))
+    wasmFileName.set("skiko.wasm")
+    jsFileName.set("skiko.js")
+
+    skikoJsPrefix.set(project.layout.projectDirectory.file("src/jsMain/resources/setup.js"))
+
+    includeHeadersNonRecursive(projectDir.resolve("src/commonMain/cpp"))
+    includeHeadersNonRecursive(skiaHeadersDirs(unpackedSkia))
+
+    flags.set(listOf(
+        *skiaPreprocessorFlags(),
+        "-s", "USE_WEBGL2=1",
+        "-l", "GL",
+        "-s", "OFFSCREEN_FRAMEBUFFER=1",
+        "-DSK_SUPPORT_GPU=1",
+        "--extern-post-js",
+        skikoJsPrefix.get().asFile.absolutePath,
+    ))
+}
+
+val skiaDir: Provider<File> = run {
     if (skiko.skiaDir != null) {
-        tasks.register("skiaWasmDir", DefaultTask::class) {
+        tasks.register("skiaDir", DefaultTask::class) {
             // dummy task to simplify usage of the resulting provider (see `else` branch)
             // if a file provider is not created from a task provider,
             // then it cannot be used instead of a task in `dependsOn` clauses of other tasks.
@@ -93,10 +138,10 @@ val skiaWasmDir = run {
             enabled = false
         }.map { skiko.skiaDir!! }
     } else {
-        val targetDir = skiko.dependenciesDir.resolve("skia/skia-wasm")
-        tasks.register("unzipSkiaWasm", Copy::class) {
-            from(skiaWasmZip.map { zipTree(it) })
-            configureSkiaCopy(targetDir)
+        val targetDir = skiko.dependenciesDir.resolve("skia/skia")
+        tasks.register("unzipSkia", Copy::class) {
+            from(skiaZip.map { zipTree(it) })
+            into(targetDir)
         }.map { targetDir }
     }
 }
@@ -120,7 +165,7 @@ kotlin {
         browser() {
             testTask {
                 testLogging.showStandardStreams = true
-                dependsOn(project.tasks.named("wasmCompile"))
+                dependsOn(wasmCrossCompile)
                 useKarma {
                     useChromeHeadless()
                 }
@@ -203,25 +248,32 @@ tasks.withType(JavaCompile::class.java).configureEach {
     this.getOptions().compilerArgs.addAll(listOf("-source", "11", "-target", "11"))
 }
 
-fun skiaPreprocessorFlags(skiaDir: String): Array<String> {
+fun skiaHeadersDirs(skiaDir: File): List<File> =
+    listOf(
+        skiaDir,
+        skiaDir.resolve("include"),
+        skiaDir.resolve("include/core"),
+        skiaDir.resolve("include/gpu"),
+        skiaDir.resolve("include/effects"),
+        skiaDir.resolve("include/pathops"),
+        skiaDir.resolve("include/utils"),
+        skiaDir.resolve("include/codec"),
+        skiaDir.resolve("include/svg"),
+        skiaDir.resolve("modules/skottie/include"),
+        skiaDir.resolve("modules/skparagraph/include"),
+        skiaDir.resolve("modules/skshaper/include"),
+        skiaDir.resolve("modules/sksg/include"),
+        skiaDir.resolve("modules/svg/include"),
+        skiaDir.resolve("third_party/externals/harfbuzz/src"),
+        skiaDir.resolve("third_party/icu"),
+        skiaDir.resolve("third_party/externals/icu/source/common"),
+    )
+
+fun includeHeadersFlags(headersDirs: List<File>) =
+    headersDirs.map { "-I${it.absolutePath}" }.toTypedArray()
+
+fun skiaPreprocessorFlags(): Array<String> {
     return listOf(
-        "-I$skiaDir",
-        "-I$skiaDir/include",
-        "-I$skiaDir/include/core",
-        "-I$skiaDir/include/gpu",
-        "-I$skiaDir/include/effects",
-        "-I$skiaDir/include/pathops",
-        "-I$skiaDir/include/utils",
-        "-I$skiaDir/include/codec",
-        "-I$skiaDir/include/svg",
-        "-I$skiaDir/modules/skottie/include",
-        "-I$skiaDir/modules/skparagraph/include",
-        "-I$skiaDir/modules/skshaper/include",
-        "-I$skiaDir/modules/sksg/include",
-        "-I$skiaDir/modules/svg/include",
-        "-I$skiaDir/third_party/externals/harfbuzz/src",
-        "-I$skiaDir/third_party/icu",
-        "-I$skiaDir/third_party/externals/icu/source/common",
         "-DSK_ALLOW_STATIC_GLOBAL_INITIALIZERS=1",
         "-DSK_FORCE_DISTANCE_FIELD_TEXT=0",
         "-DSK_GAMMA_APPLY_TO_A8",
@@ -249,10 +301,11 @@ tasks.withType(CppCompile::class.java).configureEach {
     }
     val jdkHome = System.getProperty("java.home") ?: error("'java.home' is null")
     dependsOn(skiaDir)
-    val skiaDir = skiaDir.get().absolutePath
     compilerArgs.addAll(
-        listOf("-I$jdkHome/include") + skiaPreprocessorFlags(skiaDir)
-    )
+        listOf("-I$jdkHome/include")
+                    + includeHeadersFlags(skiaHeadersDirs(skiaDir.get()))
+                    + skiaPreprocessorFlags()
+        )
     compilerArgs.add("-I$projectDir/src/jvmMain/cpp/include")
     when (targetOs) {
         OS.MacOS -> {
@@ -343,52 +396,6 @@ fun List<String>.findAllFiles(suffix: String): List<String> = this
     .map { it.absolutePath }
     .filter { it.endsWith(suffix) }
 
-
-val wasmCompile = project.tasks.register<Exec>("wasmCompile") {
-        dependsOn(skiaWasmDir)
-        val skiaDir = skiaWasmDir.get().absolutePath
-        val outDir = "$buildDir/wasm"
-        val srcs = listOf(
-            "$projectDir/src/jsMain/cpp",
-            "$projectDir/src/commonMain/cpp"
-        ).findAllFiles(".cc").toTypedArray()
-        val outJs = "$outDir/skiko.js"
-        val outWasm = "$outDir/skiko.wasm"
-        workingDir = File(outDir)
-        val skikoJsPrefix = "$projectDir/src/jsMain/resources/setup.js"
-        if (supportWasm) {
-            commandLine = listOf(
-                "emcc",
-                *buildType.clangFlags,
-                *Arch.Wasm.clangFlags,
-                "-I$projectDir/src/commonMain/cpp",
-                *skiaPreprocessorFlags(skiaDir),
-                "-l", "GL",
-                "-s", "USE_WEBGL2=1",
-                "-s", "OFFSCREEN_FRAMEBUFFER=1",
-                "-DSK_SUPPORT_GPU=1",
-                "-o", outJs,
-                "--extern-post-js", skikoJsPrefix,
-                *srcs
-            )
-            argumentProviders.add(CommandLineArgumentProvider {
-                val skiaBinDir = "$skiaDir/out/${buildType.id}-wasm-wasm"
-                // We must compute this list after Skia unpacking task has finished.
-                listOf(skiaBinDir).findAllFiles(".a")
-            })
-        } else {
-            // Create dummy files anyway.
-            commandLine = listOf(
-                "touch",
-                outJs,
-                outWasm
-            )
-        }
-        file(outDir).mkdirs()
-        inputs.files(srcs + listOf(skikoJsPrefix))
-        outputs.files(outJs, outWasm)
-}
-
 val generateVersion = project.tasks.register("generateVersion") {
     val outDir = generatedKotlin
     file(outDir).mkdirs()
@@ -432,12 +439,12 @@ project.tasks.register<Exec>("nativeBridgesCompile") {
         *targetOs.clangFlags,
         *buildType.clangFlags,
         "-c",
-        "-DPROVIDE_JNI_TYPES",
         "-DSK_SHAPER_CORETEXT_AVAILABLE",
         "-DSK_BUILD_FOR_MAC",
         "-DSK_METAL",
         "-I$projectDir/src/commonMain/cpp",
-        *skiaPreprocessorFlags(skiaDir.get().absolutePath),
+        *includeHeadersFlags(skiaHeadersDirs(skiaDir.get())),
+        *skiaPreprocessorFlags(),
         *srcs
     )
     file(outDir).mkdirs()
@@ -686,18 +693,22 @@ val skikoJvmRuntimeJar by project.tasks.registering(Jar::class) {
 }
 
 val skikoWasmJar by project.tasks.registering(Jar::class) {
+    dependsOn(wasmCrossCompile)
     // We produce jar that contains .js of wrapper/bindings and .wasm with Skia + bindings.
-    val wasmOutputs = wasmCompile.get().outputs
-    from(wasmOutputs.files.single { it.name.endsWith(".wasm")})
+    val wasmOutDir = wasmCrossCompile.map { it.outDir }
+
+    from(wasmOutDir) {
+        include("*.wasm")
+    }
 
-    from(wasmOutputs) {
-        include("skiko.js")
+    from(wasmOutDir) {
+        include("*.js")
         filter { line -> line.replace("_org_jetbrains_", "org_jetbrains_") }
     }
 
     archiveBaseName.set("skiko-wasm")
     doLast {
-        println("Wasm and JS at ${outputs.files.files.single()}")
+        println("Wasm and JS at: ${archiveFile.get().asFile.absolutePath}")
     }
 }
 

skiko/buildSrc/src/main/kotlin/WasmCrossCompileTask.kt

+import org.gradle.api.DefaultTask
+import org.gradle.api.file.*
+import org.gradle.api.provider.ListProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.*
+import org.gradle.process.ExecOperations
+
+import javax.inject.Inject
+
+import java.io.File
+import java.util.LinkedHashSet
+
+abstract class WasmCrossCompileTask : DefaultTask() {
+    @get:Inject
+    abstract val execOperations: ExecOperations
+
+    @get:Input
+    abstract val targetArch: Property<Arch>
+
+    @get:Input
+    abstract val buildVariant: Property<SkiaBuildType>
+
+    @get:Input
+    abstract val flags: ListProperty<String>
+
+    @get:InputFiles
+    lateinit var sourceFiles: FileCollection
+
+    @get:InputFiles
+    lateinit var libFiles: FileCollection
+
+    @get:Optional
+    @get:Input
+    abstract val wasmFileName: Property<String>
+
+    @get:Optional
+    @get:Input
+    abstract val jsFileName: Property<String>
+
+    @get:OutputDirectory
+    abstract val outDir: DirectoryProperty
+
+    @get:InputFile
+    @get:Optional
+    abstract val skikoJsPrefix: RegularFileProperty
+
+    @get:Input
+    var compiler: String = "emcc"
+
+    /**
+     * Used only for up-to-date checks of headers' content
+     *
+     * @see [headersDirs]
+     */
+    @Suppress("UNUSED")
+    @get:InputFiles
+    val headerFiles: FileCollection
+        get() {
+            fun File.isHeaderFile(): Boolean =
+                isFile && name.endsWith(".h", ignoreCase = true)
+
+            val headers = hashSetOf<File>()
+            for (dir in headersDirs) {
+                val canonicalDir = dir.canonicalFile
+                dir.listFiles()?.forEach { file ->
+                    if (file.isHeaderFile()) {
+                        headers.add(canonicalDir.resolve(file.name))
+                    }
+                }
+            }
+            return project.files(headers)
+        }
+
+    @get:Internal
+    internal val headersDirs = LinkedHashSet<File>()
+    fun includeHeadersNonRecursive(dirs: Collection<File>) {
+        headersDirs.addAll(dirs)
+    }
+    fun includeHeadersNonRecursive(dir: File) {
+        headersDirs.add(dir)
+    }
+
+    @TaskAction
+    fun exec() {
+        execOperations.exec {
+            executable = compiler
+            val args = argumentProviders
+
+            args.add { headersDirs.map { "-I${it.absolutePath}" } }
+
+            args.add {
+                // todo: ensure that flags do not start with '-I' (all headers should be added via [headersDirs])
+                val bt = buildVariant.get()
+                (flags.get() + targetArch.get().clangFlags + bt.flags + bt.clangFlags).asIterable()
+            }
+
+            args.add { sourceFiles.files.map { it.absolutePath } }
+            args.add { libFiles.files.map { it.absolutePath } }
+
+            args.add { listOf("-o", outDir.resolveToAbsolutePath(wasmFileName)) }
+            args.add { listOf("-o", outDir.resolveToAbsolutePath(jsFileName)) }
+        }
+    }
+}
\ No newline at end of file

skiko/buildSrc/src/main/kotlin/properties.kt

 import org.gradle.api.Project
 import java.io.File
 
-enum class OS(val id: String, val clangFlags: Array<String>) {
+enum class OS(
+    val id: String,
+    val clangFlags: Array<String>
+) {
     Linux("linux", arrayOf()),
     Windows("windows", arrayOf()),
     MacOS("macos", arrayOf("-mmacosx-version-min=10.13")),
@@ -12,7 +15,10 @@ enum class OS(val id: String, val clangFlags: Array<String>) {
         get() = this == Windows
 }
 
-enum class Arch(val id: String, val clangFlags: Array<String>) {
+enum class Arch(
+    val id: String,
+    val clangFlags: Array<String>
+) {
     X64("x64", arrayOf("-arch", "x86_64")),
     Arm64("arm64", arrayOf("-arch", "arm64")),
     Wasm("wasm", arrayOf("-std=c++17", "--bind", "-DSKIKO_WASM"))
@@ -90,7 +96,7 @@ class SkikoProperties(private val myProject: Project) {
     val buildType: SkiaBuildType
         get() = if (myProject.findProperty("skiko.debug") == "true") SkiaBuildType.DEBUG else SkiaBuildType.RELEASE
 
-    fun skiaReleaseFor(os: OS, arch: Arch): String {
+    fun skiaReleaseFor(os: OS, arch: Arch, buildType: SkiaBuildType): String {
         val target = "${os.id}-${arch.id}"
         val tag = myProject.property("dependencies.skia.$target") as String
         val suffix = if (os == OS.Linux && arch == Arch.X64) "-ubuntu14" else ""
@@ -106,6 +112,7 @@ class SkikoProperties(private val myProject: Project) {
     val visualStudioBuildToolsDir: File?
         get() = System.getenv()["SKIKO_VSBT_PATH"]?.let { File(it) }?.takeIf { it.isDirectory }
 
+    // todo: make compatible with the configuration cache
     val skiaDir: File?
         get() = (System.getenv()["SKIA_DIR"] ?: System.getProperty("skia.dir"))?.let { File(it) }
             ?.takeIf { it.isDirectory }

skiko/buildSrc/src/main/kotlin/utils.kt

+import org.gradle.api.file.FileSystemLocation
+import org.gradle.api.provider.Provider
+
+internal fun Provider<out FileSystemLocation>.resolveToAbsolutePath(path: Provider<String>): String =
+    get().asFile.absoluteFile.resolve(path.get()).absolutePath