Skip to content

S
skiko
Unverified Commit 7120885c authored by Alexey Tsvetkov's avatar Alexey Tsvetkov Committed by GitHub
Browse files
Options

Cross compile native bridges (#227)

parent 30185cb7
Hide whitespace changes
Inline Side-by-side
Showing with 313 additions and 74 deletions
skiko/build.gradle.kts
View file @ 7120885c
......@@ -62,6 +62,8 @@ val skiaCrossDownloadTasks: Map<Pair<OS, Arch>, Provider<File>> = crossTargets.k
val out = skiko.dependenciesDir.resolve("skia/skia-$suffix.zip")
tasks.register<Download>("skiaCrossDownload$suffix") {
onlyIf { skiko.skiaDir == null && !out.exists() }
val release = skiko.skiaReleaseFor(osArch.first, osArch.second, buildType)
src("https://github.com/JetBrains/skia-pack/releases/download/$release.zip")
dest(out.absolutePath)
......@@ -92,14 +94,15 @@ val skiaDirProviderForCrossTargets: Map<Pair<OS, Arch>, Provider<File>> = crossT
}
}
val wasmCrossCompile = tasks.register<WasmCrossCompileTask>("wasmCrossCompile") {
val wasmCrossCompile = tasks.register<CrossCompileTask>("wasmCrossCompile") {
val osArch = OS.Wasm to Arch.Wasm
val unzipper = skiaDirProviderForCrossTargets[osArch]!!
dependsOn(unzipper)
val unpackedSkia = unzipper.get()
targetArch.set(osArch.second)
compiler.set("emcc")
crossCompileTargetArch.set(osArch.second)
buildVariant.set(buildType)
sourceFiles =
......@@ -108,38 +111,52 @@ val wasmCrossCompile = tasks.register<WasmCrossCompileTask>("wasmCrossCompile")
if (skiko.includeTestHelpers) {
sourceFiles += project.fileTree("src/commonTest/cpp") { include("**/*.cc") }
}
outDir.set(project.layout.buildDirectory.dir("out/compile/${buildType.id}-${osArch.first.id}-${osArch.second.id}"))
includeHeadersNonRecursive(projectDir.resolve("src/commonMain/cpp"))
includeHeadersNonRecursive(skiaHeadersDirs(unpackedSkia))
flags.set(listOf(
*skiaPreprocessorFlags(),
"-DSK_SUPPORT_GPU=1"
))
}
val linkWasm = tasks.register<LinkWasmTask>("linkWasm") {
val osArch = OS.Wasm to Arch.Wasm
dependsOn(wasmCrossCompile)
val skiaAFilesDir = unpackedSkia.resolve("out/${buildType.id}-${osArch.first.id}-${osArch.second.id}")
libFiles = project.fileTree(skiaAFilesDir) { include("**/*.a") }
val unzipper = skiaDirProviderForCrossTargets[osArch]!!
dependsOn(unzipper)
val unpackedSkia = unzipper.get()
libFiles = project.fileTree(unpackedSkia) { include("**/*.a") }
objectFiles = project.fileTree(wasmCrossCompile.map { it.outDir.get() }) { include("**/*.o") }
outDir.set(project.layout.buildDirectory.dir("out/${buildType.id}-${osArch.first.id}-${osArch.second.id}"))
wasmFileName.set("skiko.wasm")
jsFileName.set("skiko.js")
skikoJsPrefix.set(project.layout.projectDirectory.file("src/jsMain/resources/setup.js"))
includeHeadersNonRecursive(projectDir.resolve("src/commonMain/cpp"))
includeHeadersNonRecursive(skiaHeadersDirs(unpackedSkia))
outDir.set(project.layout.buildDirectory.dir("out/link/${buildType.id}-${osArch.first.id}-${osArch.second.id}"))
flags.set(listOf(
*skiaPreprocessorFlags(),
"-s", "USE_WEBGL2=1",
"-l", "GL",
"-s", "USE_WEBGL2=1",
"-s", "OFFSCREEN_FRAMEBUFFER=1",
"-DSK_SUPPORT_GPU=1",
"--extern-post-js",
skikoJsPrefix.get().asFile.absolutePath,
))
finalizedBy(replaceSymbolsInSkikoJsOutput)
}
val replaceSymbolsInSkikoJsOutput by project.tasks.registering {
doLast {
val wasmTask = project.tasks.getByName("wasmCrossCompile") as WasmCrossCompileTask
val skikoJsFile: File = wasmTask.outDir.asFile.get().resolve("skiko.js")
val replacedContent = skikoJsFile.readText().replace("_org_jetbrains", "org_jetbrains")
skikoJsFile.writeText(replacedContent)
// skiko.js file is directly referenced in karma.config.d/wasm.js
// so symbols must be replaced right after linking
val jsFiles = outDir.asFile.get().walk()
.filter { it.isFile && it.name.endsWith(".js") }
for (jsFile in jsFiles) {
val originalContent = jsFile.readText()
val newContent = originalContent.replace("_org_jetbrains", "org_jetbrains")
jsFile.writeText(newContent)
}
}
}
......@@ -180,7 +197,7 @@ kotlin {
browser() {
testTask {
testLogging.showStandardStreams = true
dependsOn(wasmCrossCompile)
dependsOn(linkWasm)
useKarma {
useChromeHeadless()
}
......@@ -436,63 +453,56 @@ val generateVersion = project.tasks.register("generateVersion") {
// Very hacky way to compile native bridges and add the
// resulting object files into the final native klib.
project.tasks.register<Exec>("nativeBridgesCompile") {
val compileNativeBridges = project.tasks.register<CrossCompileTask>("compileNativeBridges") {
dependsOn(skiaDir)
val inputDirs = mutableListOf(
"$projectDir/src/nativeMain/cpp/common",
"$projectDir/src/commonMain/cpp"
).apply {
if (skiko.includeTestHelpers) add("$projectDir/src/commonTest/cpp/TestHelpers.cc")
}
val outDir = "$buildDir/nativeBridges/obj/$target"
val srcs = inputDirs
.findAllFiles(".cc")
.toTypedArray()
val outs = srcs
.map { it.substringAfterLast("/") }
.map { File(it).nameWithoutExtension }
.map { "$outDir/$it.o" }
.toTypedArray()
workingDir = File(outDir)
commandLine = listOf(
"clang++",
*targetArch.clangFlags,
*targetOs.clangFlags,
*buildType.clangFlags,
"-c",
compiler.set("clang++")
crossCompileTargetOS.set(targetOs)
crossCompileTargetArch.set(targetArch)
buildVariant.set(buildType)
sourceFiles =
project.fileTree("src/nativeMain/cpp") { include("**/*.cc") } +
project.fileTree("src/commonMain/cpp") { include("**/*.cc") }
if (skiko.includeTestHelpers) {
sourceFiles += project.fileTree("src/commonTest/cpp") { include("**/*.cc") }
}
outDir.set(project.layout.buildDirectory.dir("nativeBridges/obj/$target"))
includeHeadersNonRecursive(projectDir.resolve("src/commonMain/cpp"))
includeHeadersNonRecursive(skiaHeadersDirs(skiaDir.get()))
flags.set(listOf(
"-DSK_SHAPER_CORETEXT_AVAILABLE",
"-DSK_BUILD_FOR_MAC",
"-DSK_METAL",
"-I$projectDir/src/commonMain/cpp",
*includeHeadersFlags(skiaHeadersDirs(skiaDir.get())),
*skiaPreprocessorFlags(),
*srcs
)
file(outDir).mkdirs()
inputs.files(srcs)
outputs.files(outs)
))
}
project.tasks.register<Exec>("nativeBridgesLink") {
dependsOn(project.tasks.getByName("nativeBridgesCompile"))
inputs.files(project.tasks.getByName("nativeBridgesCompile").outputs)
val linkNativeBridges = project.tasks.register<Exec>("linkNativeBridges") {
dependsOn(compileNativeBridges)
val objectFilesDir = compileNativeBridges.map { it.outDir.get() }
val objectFiles = project.fileTree(objectFilesDir) {
include("**/*.o")
}
inputs.files(objectFiles)
val outDir = "$buildDir/nativeBridges/static/$target"
val srcs = inputs.files.files
.map { it.absolutePath }
.toTypedArray()
val staticLib = "$outDir/skiko-native-bridges-$target.a"
workingDir = File(outDir)
commandLine = listOf(
"libtool",
"-static",
"-o",
staticLib,
*srcs
)
executable = "libtool"
argumentProviders.add {
listOf(
"-static",
"-o",
staticLib
)
}
argumentProviders.add { objectFiles.files.map { it.absolutePath } }
file(outDir).mkdirs()
outputs.files(staticLib)
outputs.dir(outDir)
}
fun localSign(signer: String, lib: File): File {
......@@ -749,17 +759,13 @@ val skikoJvmRuntimeJar by project.tasks.registering(Jar::class) {
}
val skikoWasmJar by project.tasks.registering(Jar::class) {
dependsOn(wasmCrossCompile)
dependsOn(linkWasm)
// We produce jar that contains .js of wrapper/bindings and .wasm with Skia + bindings.
val wasmOutDir = wasmCrossCompile.map { it.outDir }
val wasmOutDir = linkWasm.map { it.outDir }
from(wasmOutDir) {
include("*.wasm")
}
from(wasmOutDir) {
include("*.js")
filter { line -> line.replace("_org_jetbrains_", "org_jetbrains_") }
}
archiveBaseName.set("skiko-wasm")
......@@ -930,7 +936,7 @@ afterEvaluate {
source(generatedKotlin)
if (supportNative) {
dependsOn(tasks.getByName("nativeBridgesLink"))
dependsOn(linkNativeBridges)
}
}
}
......
skiko/buildSrc/src/main/kotlin/WasmCrossCompileTask.kt → skiko/buildSrc/src/main/kotlin/CrossCompileTask.kt
View file @ 7120885c
import org.gradle.api.DefaultTask
import org.gradle.api.file.*
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.FileCollection
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.tasks.*
import org.gradle.process.ExecOperations
import javax.inject.Inject
import org.gradle.work.ChangeType
import org.gradle.work.FileChange
import org.gradle.work.Incremental
import org.gradle.work.InputChanges
import java.io.File
import java.util.LinkedHashSet
import java.util.concurrent.Callable
import javax.inject.Inject
abstract class WasmCrossCompileTask : DefaultTask() {
abstract class CrossCompileTask : DefaultTask() {
@get:Inject
abstract val execOperations: ExecOperations
@get:Input
abstract val targetArch: Property<Arch>
@get:Optional
abstract val crossCompileTargetOS: Property<OS>
@get:Input
abstract val crossCompileTargetArch: Property<Arch>
@get:Input
abstract val buildVariant: Property<SkiaBuildType>
......@@ -24,28 +32,14 @@ abstract class WasmCrossCompileTask : DefaultTask() {
abstract val flags: ListProperty<String>
@get:InputFiles
@get:Incremental
lateinit var sourceFiles: FileCollection
@get:InputFiles
lateinit var libFiles: FileCollection
@get:Optional
@get:Input
abstract val wasmFileName: Property<String>
@get:Optional
@get:Input
abstract val jsFileName: Property<String>
@get:OutputDirectory
abstract val outDir: DirectoryProperty
@get:InputFile
@get:Optional
abstract val skikoJsPrefix: RegularFileProperty
@get:Input
var compiler: String = "emcc"
abstract val compiler: Property<String>
/**
* Used only for up-to-date checks of headers' content
......@@ -54,8 +48,9 @@ abstract class WasmCrossCompileTask : DefaultTask() {
*/
@Suppress("UNUSED")
@get:InputFiles
val headerFiles: FileCollection
get() {
@get:Incremental
val headerFiles: FileCollection =
project.files(Callable {
fun File.isHeaderFile(): Boolean =
isFile && name.endsWith(".h", ignoreCase = true)
......@@ -68,8 +63,8 @@ abstract class WasmCrossCompileTask : DefaultTask() {
}
}
}
return project.files(headers)
}
headers
})
@get:Internal
internal val headersDirs = LinkedHashSet<File>()
......@@ -81,24 +76,76 @@ abstract class WasmCrossCompileTask : DefaultTask() {
}
@TaskAction
fun exec() {
open fun run(inputChanges: InputChanges) {
execOperations.exec {
executable = compiler
val args = argumentProviders
val sourcesToCompile = determineSourcesToCompile(inputChanges)
executable = compiler.get()
args = arrayListOf<String>().also { args ->
configureCompilerArgs(args)
args.addAll(sourcesToCompile.map { it.absolutePath })
}
args.add { headersDirs.map { "-I${it.absolutePath}" } }
workingDir = outDir.get().asFile
// todo: log args to file system
}
}
args.add {
// todo: ensure that flags do not start with '-I' (all headers should be added via [headersDirs])
val bt = buildVariant.get()
(flags.get() + targetArch.get().clangFlags + bt.flags + bt.clangFlags).asIterable()
}
private sealed class Mode {
class Incremental(val changes: Iterable<FileChange>) : Mode()
class NonIncremental(val reason: String) : Mode()
}
args.add { sourceFiles.files.map { it.absolutePath } }
args.add { libFiles.files.map { it.absolutePath } }
private fun determineSourcesToCompile(inputChanges: InputChanges): Collection<File> {
val compilationMode = analyzeIncrementalChanges(inputChanges)
val outDir = outDir.get().asFile
return when (compilationMode) {
is Mode.Incremental -> {
val sourcesToCompile = arrayListOf<File>()
for (change in compilationMode.changes) {
val outdatedOutputFile = outDir.resolve(change.file.nameWithoutExtension + ".o")
if (outdatedOutputFile.exists()) {
// `outdatedOutputFile` might not exist,
// when `change.file` is a new file, which was not compiled
// todo: log in verbose mode
outdatedOutputFile.delete()
}
if (change.changeType != ChangeType.REMOVED) {
sourcesToCompile.add(change.file)
}
}
logger.warn("Compiling ${sourcesToCompile.size} files incrementally")
sourcesToCompile
}
is Mode.NonIncremental -> {
outDir.deleteRecursively()
outDir.mkdirs()
logger.warn("Recompiling all files: ${compilationMode.reason}")
sourceFiles.files
}
}
}
private fun analyzeIncrementalChanges(inputChanges: InputChanges): Mode {
if (!inputChanges.isIncremental) {
return Mode.NonIncremental("input changes are not incremental")
}
args.add { listOf("-o", outDir.resolveToAbsolutePath(wasmFileName)) }
args.add { listOf("-o", outDir.resolveToAbsolutePath(jsFileName)) }
if (inputChanges.getFileChanges(headerFiles).any()) {
return Mode.NonIncremental("header files are modified or removed")
}
return Mode.Incremental(inputChanges.getFileChanges(sourceFiles))
}
open fun configureCompilerArgs(args: MutableList<String>) {
args.add("-c")
args.addAll(headersDirs.map { "-I${it.absolutePath}" })
// todo: ensure that flags do not start with '-I' (all headers should be added via [headersDirs])
args.addAll(flags.get())
args.addAll(crossCompileTargetOS.orNull?.clangFlags ?: arrayOf())
args.addAll(crossCompileTargetArch.get().clangFlags)
val bt = buildVariant.get()
args.addAll(bt.flags)
args.addAll(bt.clangFlags)
}
}
\ No newline at end of file
skiko/buildSrc/src/main/kotlin/LinkWasmTask.kt 0 → 100644
View file @ 7120885c
import org.gradle.api.DefaultTask
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.FileCollection
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.tasks.*
import org.gradle.process.ExecOperations
import javax.inject.Inject
abstract class LinkWasmTask : DefaultTask() {
@get:Inject
abstract val execOperations: ExecOperations
@get:InputFiles
lateinit var libFiles: FileCollection
@get:InputFiles
lateinit var objectFiles: FileCollection
@get:Input
abstract val wasmFileName: Property<String>
@get:Input
abstract val jsFileName: Property<String>
@get:InputFile
@get:Optional
abstract val skikoJsPrefix: RegularFileProperty
@get:OutputDirectory
abstract val outDir: DirectoryProperty
@get:Input
abstract val flags: ListProperty<String>
@TaskAction
fun run() {
outDir.get().asFile.apply {
deleteRecursively()
mkdirs()
}
execOperations.exec {
executable = "emcc"
args = arrayListOf<String>().also { configureCompilerArgs(it) }
workingDir = outDir.get().asFile
// todo: log args to file system
}
}
fun configureCompilerArgs(args: MutableList<String>) {
args.addAll(flags.get())
args.addAll(objectFiles.files.map { it.absolutePath })
args.addAll(libFiles.files.map { it.absolutePath })
args.add("--extern-post-js")
args.add(skikoJsPrefix.get().asFile.absolutePath)
args.add("-o")
args.add(outDir.resolveToAbsolutePath(wasmFileName))
args.add("-o")
args.add(outDir.resolveToAbsolutePath(jsFileName))
}
}
\ No newline at end of file
skiko/buildSrc/src/main/kotlin/utils.kt
View file @ 7120885c
import org.gradle.api.Task
import org.gradle.api.file.FileSystemLocation
import org.gradle.api.model.ObjectFactory
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
internal val Provider<out FileSystemLocation>.absolutePath: String
get() = get().asFile.absolutePath
internal fun Provider<out FileSystemLocation>.resolveToAbsolutePath(path: Provider<String>): String =
get().asFile.absoluteFile.resolve(path.get()).absolutePath
inline fun <reified T : Any> ObjectFactory.nullableProperty(): Property<T?> =
property(T::class.java)
inline fun <reified T : Any> ObjectFactory.notNullProperty(): Property<T> =
property(T::class.java)
inline fun <reified T : Any> ObjectFactory.notNullProperty(defaultValue: T): Property<T> =
property(T::class.java).value(defaultValue)
inline fun <reified T> Task.provider(noinline fn: () -> T): Provider<T> =
project.provider(fn)
skiko/karma.config.d/wasm.js
View file @ 7120885c
......@@ -6,7 +6,7 @@ config.browserConsoleLogOptions.level = "debug";
const basePath = config.basePath;
const projectPath = path.resolve(basePath, "..", "..", "..", "..", "..");
const wasmPath = path.resolve(projectPath, "build", "out", "Release-wasm-wasm")
const wasmPath = path.resolve(projectPath, "build", "out", "link", "Release-wasm-wasm")
const debug = message => console.log(`[karma-config] ${message}`);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment