From adaaae2129cabfd963a6f7bcffbafa977558f719 Mon Sep 17 00:00:00 2001 From: Gary Tierney Date: Sun, 18 Jun 2017 21:12:39 +0100 Subject: [PATCH] Refactor the plugin compile task for performance Integrates the plugin script compilation task with Gradle so we no longer need to start a new instance of the JVM for each set of plugin scripts that we want to compile. --- buildSrc/build.gradle | 26 ++++ .../build/compile/KotlinScriptCompiler.kt | 139 ++++++++++++++++++ .../build/tasks/KotlinScriptCompileTask.kt | 54 +++++++ game/plugins.gradle | 41 ++++-- 4 files changed, 244 insertions(+), 16 deletions(-) create mode 100644 buildSrc/build.gradle create mode 100644 buildSrc/src/main/kotlin/org/apollo/build/compile/KotlinScriptCompiler.kt create mode 100644 buildSrc/src/main/kotlin/org/apollo/build/tasks/KotlinScriptCompileTask.kt diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle new file mode 100644 index 00000000..b559f223 --- /dev/null +++ b/buildSrc/build.gradle @@ -0,0 +1,26 @@ +apply plugin: 'kotlin' + +buildscript { + ext { + kotlinVersion = '1.1.2-4' + } + + repositories { + mavenCentral() + } + + dependencies { + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion" + } +} + +repositories { + mavenLocal() + maven { url "https://repo.maven.apache.org/maven2" } +} + +dependencies { + compile gradleApi() + compile group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jre8', version: "$kotlinVersion" + compile group: 'org.jetbrains.kotlin', name: 'kotlin-compiler-embeddable', version: "$kotlinVersion" +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/org/apollo/build/compile/KotlinScriptCompiler.kt b/buildSrc/src/main/kotlin/org/apollo/build/compile/KotlinScriptCompiler.kt new file mode 100644 index 00000000..9384f16e --- /dev/null +++ b/buildSrc/src/main/kotlin/org/apollo/build/compile/KotlinScriptCompiler.kt @@ -0,0 +1,139 @@ +package org.apollo.build.compile + +import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys +import org.jetbrains.kotlin.cli.common.messages.* +import org.jetbrains.kotlin.cli.jvm.compiler.* +import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot +import org.jetbrains.kotlin.codegen.CompilationException +import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer +import org.jetbrains.kotlin.config.* +import org.jetbrains.kotlin.script.KotlinScriptDefinitionFromAnnotatedTemplate +import java.io.File +import java.lang.management.ManagementFactory +import java.net.URISyntaxException +import java.net.URLClassLoader +import java.nio.file.* +import java.util.* + + +class KotlinMessageCollector : MessageCollector { + + override fun clear() { + } + + override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation) { + if (severity.isError) { + println("${location.path}:${location.line}-${location.column}: $message") + println(">>> ${location.lineContent}") + } + } + + override fun hasErrors(): Boolean { + return false + } + +} + +data class KotlinCompilerResult(val fqName: String, val outputPath: Path) + +class KotlinScriptCompiler { + + val classpath: List + val messageCollector: MessageCollector + val compilerConfiguration: CompilerConfiguration + + constructor(scriptDefinitionClassName: String, classpath: Collection, messageCollector: MessageCollector) { + this.classpath = classpath + currentClasspath() + this.messageCollector = messageCollector + this.compilerConfiguration = createCompilerConfiguration(scriptDefinitionClassName) + } + + companion object { + + fun currentClasspath(): List { + val classLoader = Thread.currentThread().contextClassLoader as? URLClassLoader ?: + throw RuntimeException("Unable to resolve classpath for current ClassLoader") + + val classpathUrls = classLoader.urLs + val classpath = ArrayList() + + for (classpathUrl in classpathUrls) { + try { + classpath.add(File(classpathUrl.toURI())) + } catch (e: URISyntaxException) { + throw RuntimeException("URL returned by ClassLoader is invalid") + } + + } + + val runtimeBean = ManagementFactory.getRuntimeMXBean() + if (!runtimeBean.isBootClassPathSupported) { + println("Warning! Boot class path is not supported, must be supplied on the command line") + } else { + val bootClasspath = runtimeBean.bootClassPath + classpath.addAll(bootClasspath.split(File.pathSeparatorChar).map { File(it) }) + } + + return classpath + } + } + + private fun createCompilerConfiguration(scriptDefinitionClassName: String): CompilerConfiguration { + val classLoader = URLClassLoader(classpath.map { it.toURL() }.toTypedArray()) + val configuration = CompilerConfiguration() + val scriptDefinitionClass = classLoader.loadClass(scriptDefinitionClassName) + val scriptDefinition = KotlinScriptDefinitionFromAnnotatedTemplate(scriptDefinitionClass.kotlin) + + configuration.add(JVMConfigurationKeys.SCRIPT_DEFINITIONS, scriptDefinition) + configuration.put(JVMConfigurationKeys.CONTENT_ROOTS, classpath.map { JvmClasspathRoot(it) }) + configuration.put(JVMConfigurationKeys.RETAIN_OUTPUT_IN_MEMORY, true) + configuration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, KotlinMessageCollector()) + configuration.copy() + + return configuration + } + + @Throws(KotlinPluginCompilerException::class) + fun compile(inputPath: Path, outputPath: Path): KotlinCompilerResult { + val rootDisposable = Disposer.newDisposable() + val configuration = compilerConfiguration.copy() + + configuration.put(CommonConfigurationKeys.MODULE_NAME, inputPath.toString()) + configuration.addKotlinSourceRoot(inputPath.toAbsolutePath().toString()) + + val configFiles = EnvironmentConfigFiles.JVM_CONFIG_FILES + val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, configFiles) + + try { + val generationState = KotlinToJVMBytecodeCompiler.analyzeAndGenerate(environment) + if (generationState == null) { + throw KotlinPluginCompilerException("Failed to generate bytecode for kotlin script") + } + + val sourceFiles = environment.getSourceFiles() + val script = sourceFiles[0].script ?: throw KotlinPluginCompilerException("Main script file isnt a script") + + val scriptFilePath = script.fqName.asString().replace('.', '/') + ".class" + val scriptFileClass = generationState.factory.get(scriptFilePath) + + if (scriptFileClass == null) { + throw KotlinPluginCompilerException("Unable to find compiled plugin class file $scriptFilePath") + } + + generationState.factory.asList().forEach { + Files.write(outputPath.resolve(it.relativePath), it.asByteArray(), StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING) + } + + return KotlinCompilerResult(script.fqName.asString(), outputPath.resolve(scriptFileClass.relativePath)) + } catch (e: CompilationException) { + throw KotlinPluginCompilerException("Compilation failed", e) + } finally { + Disposer.dispose(rootDisposable) + } + } + +} + +class KotlinPluginCompilerException(message: String, cause: Throwable? = null) : Exception(message, cause) { + +} diff --git a/buildSrc/src/main/kotlin/org/apollo/build/tasks/KotlinScriptCompileTask.kt b/buildSrc/src/main/kotlin/org/apollo/build/tasks/KotlinScriptCompileTask.kt new file mode 100644 index 00000000..373e6478 --- /dev/null +++ b/buildSrc/src/main/kotlin/org/apollo/build/tasks/KotlinScriptCompileTask.kt @@ -0,0 +1,54 @@ +package org.apollo.build.tasks + +import org.apollo.build.compile.KotlinMessageCollector +import org.apollo.build.compile.KotlinScriptCompiler +import org.gradle.api.DefaultTask +import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.* +import org.gradle.api.tasks.incremental.IncrementalTaskInputs +import java.io.File + +open class KotlinScriptCompileTask : DefaultTask() { + @OutputDirectory + var outputsDir: File? = null + + @Input + var compileClasspath: FileCollection? = null + + @Input + var scriptDefinitionClass: String? = null + + @TaskAction + fun execute(inputs: IncrementalTaskInputs) { + if (scriptDefinitionClass == null) { + throw Exception("No script definition class given") + } + + if (compileClasspath == null) { + throw Exception("No compile classpath given") + } + + val classpath = compileClasspath!!.files + val compiler = KotlinScriptCompiler(scriptDefinitionClass!!, classpath, KotlinMessageCollector()) + + inputs.outOfDate { + removeBinariesFor(it.file) + compiler.compile(it.file.toPath(), outputsDir!!.toPath()) + } + + inputs.removed { + removeBinariesFor(it.file) + } + } + + private fun removeBinariesFor(file: File) { + val normalizedFilename = file.name.replace("[^A-Z_]", "_") + val normalizedPrefix = normalizedFilename.subSequence(0, normalizedFilename.lastIndexOf('.')) + + val binaries = outputsDir!!.listFiles { dir, name -> name.startsWith(normalizedPrefix) } + + binaries.forEach { + it.delete() + } + } +} diff --git a/game/plugins.gradle b/game/plugins.gradle index e16aa141..cacf150a 100644 --- a/game/plugins.gradle +++ b/game/plugins.gradle @@ -1,4 +1,5 @@ import com.moandjiezana.toml.Toml +import org.apollo.build.tasks.KotlinScriptCompileTask import java.nio.file.Paths @@ -15,7 +16,7 @@ buildscript { } } -task pluginTests { +task testPlugins { group = "plugin-verification" doLast { @@ -23,7 +24,7 @@ task pluginTests { } } -check.dependsOn pluginTests +check.dependsOn testPlugins class PluginBuildData { PluginBuildData(String normalizedName, SourceSet mainSources, SourceSet testSources, @@ -68,8 +69,9 @@ def configurePluginDependencies(SourceSet mainSources, SourceSet testSources, def configurePluginTasks(String name, SourceSet mainSources, SourceSet testSources, FileCollection scriptFiles, List pluginDependencies) { + def taskName = name.split("_").collect { it.capitalize() }.join("") - def testsTask = task("${name}Tests", type: Test) { + def testsTask = task("test${taskName}", type: Test) { group = "plugin-verification" testClassesDir = testSources.output.classesDir @@ -87,26 +89,27 @@ def configurePluginTasks(String name, SourceSet mainSources, SourceSet testSourc } } - pluginTests.dependsOn testsTask + testPlugins.dependsOn testsTask if (!scriptFiles.empty) { - def compileScriptsTask = task("compile${name}Scripts", type: JavaExec) { + def compileScriptsTask = task("compile${taskName}Scripts", type: KotlinScriptCompileTask) { group = "plugin-compile" - def outputDir = mainSources.output.classesDir.toString() + def outputDir = mainSources.output.classesDir inputs.files scriptFiles - outputs.dir outputDir + outputsDir = outputDir - classpath = sourceSets.main.compileClasspath + sourceSets.main.runtimeClasspath - main = 'org.apollo.game.plugin.kotlin.KotlinPluginCompiler' - args = [outputDir] + scriptFiles.collect { it.absoluteFile.toString() } + compileClasspath = sourceSets.main.compileClasspath + + sourceSets.main.runtimeClasspath + + mainSources.compileClasspath + + mainSources.runtimeClasspath + + scriptDefinitionClass = "org.apollo.game.plugin.kotlin.KotlinPluginScript" + mustRunAfter tasks[mainSources.classesTaskName] } - tasks[mainSources.classesTaskName].outputs.upToDateWhen { false } - tasks[mainSources.classesTaskName].doLast { - compileScriptsTask.execute() - } + testPlugins.dependsOn compileScriptsTask } } @@ -122,25 +125,31 @@ pluginDefinitions.each { file -> def pluginFolder = Paths.get(file.parentFile.absolutePath) def name = meta.getString("name", pluginFolder.getFileName().toString()) def normalizedName = name.replaceAll("[^a-zA-Z0-9_]", '_').toLowerCase() + def displayNameArray = normalizedName.split("_").collect { it.capitalize() }.join("").toCharArray() + displayNameArray[0] = displayNameArray[0].toLowerCase() + + def sourceSetName = new String(displayNameArray) + def packageName = meta.getString("package", "org.apollo.game.plugin") def authors = meta.getList("authors", new ArrayList()) def dependencies = meta.getList("dependencies", new ArrayList()) def scripts = fileTree(file.parentFile) { include '**/*.plugin.kts' + exclude '*.kt' } def srcsDir = meta.getString("config.src", "src/") def testDir = meta.getString("config.test", "test/") - def mainSources = sourceSets.create("${normalizedName}_main") { + def mainSources = sourceSets.create("${sourceSetName}Main") { kotlin { srcDir pluginFolder.resolve(srcsDir).toString() exclude '*.kts' } } - def testSources = sourceSets.create("${normalizedName}_test") { + def testSources = sourceSets.create("${sourceSetName}Test") { kotlin { srcDir pluginFolder.resolve(testDir).toString() }