From 7ffef28117fe50fbe173390e433d3a2e89ad1c8f Mon Sep 17 00:00:00 2001 From: Gary Tierney Date: Sun, 28 May 2017 01:38:58 +0100 Subject: [PATCH] Compile plugins at build-time instead of runtime Adds gradle tasks to build all plugin scripts under data/plugins with the KotlinPluginCompiler implementation previously used for runtime code generation. In addition to .plugin.kts files, scripts can also declare API code in .kt files which will also be included on the classpath and made available to other plugins. --- build.gradle | 2 +- game/build.gradle | 37 ++++- .../plugin/kotlin/KotlinPluginCompiler.kt | 148 +++++++++++++++--- .../game/plugin/KotlinPluginEnvironment.java | 81 +++++++--- .../apollo/game/plugin/PluginEnvironment.java | 9 +- .../org/apollo/game/plugin/PluginManager.java | 38 +---- .../game/plugin/RubyPluginEnvironment.java | 61 -------- 7 files changed, 230 insertions(+), 146 deletions(-) delete mode 100644 game/src/main/org/apollo/game/plugin/RubyPluginEnvironment.java diff --git a/build.gradle b/build.gradle index a9cb46d6..a4bdcb9a 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ subprojects { def gameSubproject = project(':game') -task(run, dependsOn: gameSubproject.tasks['classes'], type: JavaExec) { +task(run, dependsOn: gameSubproject.tasks['assemble'], type: JavaExec) { def gameClasspath = gameSubproject.sourceSets.main.runtimeClasspath main = 'org.apollo.Server' diff --git a/game/build.gradle b/game/build.gradle index c380c06a..92df95e8 100644 --- a/game/build.gradle +++ b/game/build.gradle @@ -14,9 +14,15 @@ apply plugin: 'kotlin' sourceSets { main.kotlin.srcDirs += 'src/kotlin' + main.kotlin.excludes += 'stub.kt' + main.kotlin.excludes += '**/*.kts' plugins { - kotlin.srcDirs += 'data/plugins' + kotlin { + srcDir 'data/plugins' + exclude 'stub.kt' + exclude '**/*.kts' + } } } @@ -28,6 +34,33 @@ dependencies { pluginsCompile(configurations.compile) pluginsCompile(sourceSets.main.output) - compile group: 'org.jetbrains.kotlin', name: 'kotlin-compiler', version: "$kotlinVersion" compile group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jre8', version: "$kotlinVersion" + compile group: 'org.jetbrains.kotlin', name: 'kotlin-compiler', version: "$kotlinVersion" + + runtime files("$buildDir/plugins") + runtime sourceSets.plugins.output } + + +task('compilePluginScripts', dependsOn: [classes, pluginsClasses], type: JavaExec) { + def compilerClasspath = [ + configurations.compile.asPath, + configurations.runtime.asPath, + sourceSets.main.compileClasspath.asPath, + sourceSets.main.runtimeClasspath.asPath + ] + + def inputDir = "$projectDir/data/plugins" + def outputDir = "$buildDir/plugins" + def manifestPath = "$buildDir/plugins/manifest.txt" + + inputs.source inputDir + outputs.dir outputDir + + println compilerClasspath.join(':') + classpath = sourceSets.main.compileClasspath + sourceSets.main.runtimeClasspath + main = 'org.apollo.game.plugin.kotlin.KotlinPluginCompiler' + args = [inputDir, outputDir, manifestPath, compilerClasspath.join(':')] +} + +assemble.dependsOn(compilePluginScripts) \ No newline at end of file diff --git a/game/src/kotlin/org/apollo/game/plugin/kotlin/KotlinPluginCompiler.kt b/game/src/kotlin/org/apollo/game/plugin/kotlin/KotlinPluginCompiler.kt index e87dbe5d..71bfb4cf 100644 --- a/game/src/kotlin/org/apollo/game/plugin/kotlin/KotlinPluginCompiler.kt +++ b/game/src/kotlin/org/apollo/game/plugin/kotlin/KotlinPluginCompiler.kt @@ -1,41 +1,132 @@ package org.apollo.game.plugin.kotlin +import com.google.common.base.CaseFormat import com.intellij.openapi.util.Disposer -import org.apollo.Server -import org.apollo.game.model.World -import org.apollo.game.plugin.PluginContext import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys -import org.jetbrains.kotlin.cli.common.messages.MessageCollector -import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles -import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment -import org.jetbrains.kotlin.cli.jvm.compiler.KotlinToJVMBytecodeCompiler +import org.jetbrains.kotlin.cli.common.messages.* +import org.jetbrains.kotlin.cli.jvm.compiler.* import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot import org.jetbrains.kotlin.codegen.CompilationException -import org.jetbrains.kotlin.config.CommonConfigurationKeys -import org.jetbrains.kotlin.config.CompilerConfiguration -import org.jetbrains.kotlin.config.JVMConfigurationKeys -import org.jetbrains.kotlin.config.addKotlinSourceRoot +import org.jetbrains.kotlin.config.* import org.jetbrains.kotlin.script.KotlinScriptDefinitionFromAnnotatedTemplate import java.io.File +import java.lang.management.ManagementFactory +import java.net.URISyntaxException +import java.net.URLClassLoader +import java.nio.file.* +import java.nio.file.StandardOpenOption.* +import java.nio.file.attribute.BasicFileAttributes +import java.util.* +import java.util.function.BiPredicate + +class KotlinMessageCollector : MessageCollector { + + override fun clear() { + } + + override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation) { + if (severity.isError) { + println("${location.path}:${location.line}-${location.column}: $message") + println(">>> ${location.lineContent}") + } + } + + override fun hasErrors(): Boolean { + return false + } + +} + +data class KotlinCompilerResult(val fqName: String, val outputPath: Path) class KotlinPluginCompiler(val classpath: List, val messageCollector: MessageCollector) { - private fun createCompilerConfiguration(inputPath: String): CompilerConfiguration { + companion object { + + private val maxSearchDepth = 1024; + + fun currentClasspath(): List { + val classLoader = Thread.currentThread().contextClassLoader as? URLClassLoader ?: + throw RuntimeException("Unable to resolve classpath for current ClassLoader") + + val classpathUrls = classLoader.urLs + val classpath = ArrayList() + + for (classpathUrl in classpathUrls) { + try { + classpath.add(File(classpathUrl.toURI())) + } catch (e: URISyntaxException) { + throw RuntimeException("URL returned by ClassLoader is invalid") + } + + } + + return classpath + } + + @JvmStatic + fun main(args: Array) { + if (args.size < 4) throw RuntimeException("Usage: ") + + val inputDir = Paths.get(args[0]) + val outputDir = Paths.get(args[1]) + val manifestPath = Paths.get(args[2]) + val classpathEntries = args[3].split(':') + val classpath = mutableListOf() + + val runtimeBean = ManagementFactory.getRuntimeMXBean() + if (!runtimeBean.isBootClassPathSupported) { + println("Warning! Boot class path is not supported, must be supplied on the command line") + } else { + val bootClasspath = runtimeBean.bootClassPath + classpath.addAll(bootClasspath.split(':').map { File(it) }) + } + + /** + * Classpath entries on the command line contain the kotlin runtime + * and plugin API code. We use our current classpath to provide + * Kotlin with access to the JRE and apollo modules. + */ + classpath.addAll(currentClasspath()) + classpath.addAll(classpathEntries.map { File(it) }) + + val inputScriptsMatcher = { path: Path, _: BasicFileAttributes -> path.toString().endsWith(".plugin.kts") } + val inputScripts = Files.find(inputDir, maxSearchDepth, BiPredicate(inputScriptsMatcher)) + + val compiler = KotlinPluginCompiler(classpath, MessageCollector.NONE) + val compiledScriptClasses = mutableListOf() + + try { + Files.createDirectory(outputDir) + + inputScripts.forEach { + compiledScriptClasses.add(compiler.compile(it, outputDir).fqName) + } + + Files.write(manifestPath, compiledScriptClasses, CREATE, TRUNCATE_EXISTING) + } catch (t: Throwable) { + t.printStackTrace() + System.exit(1) + } + } + } + + private fun createCompilerConfiguration(inputPath: Path): CompilerConfiguration { val configuration = CompilerConfiguration() val scriptDefinition = KotlinScriptDefinitionFromAnnotatedTemplate(KotlinPluginScript::class) configuration.add(JVMConfigurationKeys.SCRIPT_DEFINITIONS, scriptDefinition) configuration.put(JVMConfigurationKeys.CONTENT_ROOTS, classpath.map { JvmClasspathRoot(it) }) configuration.put(JVMConfigurationKeys.RETAIN_OUTPUT_IN_MEMORY, true) - configuration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, messageCollector) - configuration.put(CommonConfigurationKeys.MODULE_NAME, inputPath) - configuration.addKotlinSourceRoot(inputPath) + configuration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, KotlinMessageCollector()) + configuration.put(CommonConfigurationKeys.MODULE_NAME, inputPath.toString()) + configuration.addKotlinSourceRoot(inputPath.toAbsolutePath().toString()) return configuration } @Throws(KotlinPluginCompilerException::class) - fun compile(inputPath: String): Class { + fun compile(inputPath: Path, outputPath: Path): KotlinCompilerResult { val rootDisposable = Disposer.newDisposable() val configuration = createCompilerConfiguration(inputPath) @@ -43,13 +134,26 @@ class KotlinPluginCompiler(val classpath: List, val messageCollector: Mess val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, configFiles) try { - val clazz = KotlinToJVMBytecodeCompiler.compileScript(environment, Server::class.java.classLoader) - - if (clazz?.getConstructor(World::class.java, PluginContext::class.java) == null) { - throw KotlinPluginCompilerException("Unable to compile $inputPath, no plugin constructor found") + val generationState = KotlinToJVMBytecodeCompiler.analyzeAndGenerate(environment) + if (generationState == null) { + throw KotlinPluginCompilerException("Failed to generate bytecode for kotlin script") } - return clazz as Class + val sourceFiles = environment.getSourceFiles() + val script = sourceFiles[0].script ?: throw KotlinPluginCompilerException("Main script file isnt a script") + + val scriptFilePath = script.fqName.asString().replace('.', '/') + ".class" + val scriptFileClass = generationState.factory.get(scriptFilePath) + + if (scriptFileClass == null) { + throw KotlinPluginCompilerException("Unable to find compiled plugin class file $scriptFilePath") + } + + generationState.factory.asList().forEach { + Files.write(outputPath.resolve(it.relativePath), it.asByteArray(), CREATE, WRITE, TRUNCATE_EXISTING) + } + + return KotlinCompilerResult(script.fqName.asString(), outputPath.resolve(scriptFileClass.relativePath)) } catch (e: CompilationException) { throw KotlinPluginCompilerException("Compilation failed", e) } finally { @@ -61,4 +165,4 @@ class KotlinPluginCompiler(val classpath: List, val messageCollector: Mess class KotlinPluginCompilerException(message: String, cause: Throwable? = null) : Exception(message, cause) { -} \ No newline at end of file +} diff --git a/game/src/main/org/apollo/game/plugin/KotlinPluginEnvironment.java b/game/src/main/org/apollo/game/plugin/KotlinPluginEnvironment.java index 0dfe06cd..dce8bb60 100644 --- a/game/src/main/org/apollo/game/plugin/KotlinPluginEnvironment.java +++ b/game/src/main/org/apollo/game/plugin/KotlinPluginEnvironment.java @@ -1,5 +1,20 @@ package org.apollo.game.plugin; +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Constructor; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; import org.apollo.game.model.World; import org.apollo.game.plugin.kotlin.KotlinPluginCompiler; import org.apollo.game.plugin.kotlin.KotlinPluginScript; @@ -8,17 +23,6 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation; import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity; import org.jetbrains.kotlin.cli.common.messages.MessageCollector; -import java.io.File; -import java.io.InputStream; -import java.lang.reflect.Constructor; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - public class KotlinPluginEnvironment implements PluginEnvironment, MessageCollector { private static final Logger logger = Logger.getLogger(KotlinPluginEnvironment.class.getName()); @@ -61,17 +65,56 @@ public class KotlinPluginEnvironment implements PluginEnvironment, MessageCollec } @Override - public void parse(InputStream is, String name) { - //@todo - wait until all plugin classes are loading until running constructors? - try { - Class pluginClass = pluginCompiler.compile(name); - Constructor pluginConstructor = pluginClass.getConstructor(World.class, - PluginContext.class); + public void load(Collection plugins) { + try (InputStream resource = KotlinPluginEnvironment.class.getResourceAsStream("/manifest.txt")) { + BufferedReader reader = new BufferedReader(new InputStreamReader(resource)); + List pluginClassNames = reader.lines().collect(Collectors.toList()); - pluginConstructor.newInstance(world, context); + for (String pluginClassName : pluginClassNames) { + Class pluginClass = + (Class) Class.forName(pluginClassName); + + Constructor pluginConstructor = + pluginClass.getConstructor(World.class, PluginContext.class); + + KotlinPluginScript plugin = pluginConstructor.newInstance(world, context); + } } catch (Exception e) { throw new RuntimeException(e); } + + List> pluginClasses = new ArrayList<>(); + List sourceRoots = new ArrayList<>(); + + for (PluginMetaData plugin : plugins) { + List pluginSourceRoots = Arrays.stream(plugin.getScripts()) + .map(script -> plugin.getBase() + "/" + script) + .collect(Collectors.toList()); + + sourceRoots.addAll(pluginSourceRoots); + } + + for (String scriptSource : sourceRoots) { +// try { + List dependencySourceRoots = new ArrayList<>(sourceRoots); + dependencySourceRoots.remove(scriptSource); + +// pluginClasses.add(pluginCompiler.compile(scriptSource)); +// } catch (KotlinPluginCompilerException e) { +// throw new RuntimeException(e); +// } + } + + for (Class pluginClass : pluginClasses) { + try { + Constructor constructor = pluginClass + .getConstructor(World.class, PluginContext.class); + + KotlinPluginScript script = constructor.newInstance(world, context); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } @Override @@ -89,7 +132,7 @@ public class KotlinPluginEnvironment implements PluginEnvironment, MessageCollec @NotNull CompilerMessageLocation location) { if (severity.isError()) { logger.log(Level.SEVERE, String.format("%s:%s-%s: %s", location.getPath(), location.getLine(), - location.getColumn(), message)); + location.getColumn(), message)); } } diff --git a/game/src/main/org/apollo/game/plugin/PluginEnvironment.java b/game/src/main/org/apollo/game/plugin/PluginEnvironment.java index f049f974..505d8473 100644 --- a/game/src/main/org/apollo/game/plugin/PluginEnvironment.java +++ b/game/src/main/org/apollo/game/plugin/PluginEnvironment.java @@ -1,6 +1,8 @@ package org.apollo.game.plugin; import java.io.InputStream; +import java.util.Collection; +import java.util.Set; /** * Represents some sort of environment that plugins could be executed in, e.g. {@code javax.script} or Jython. @@ -10,12 +12,11 @@ import java.io.InputStream; public interface PluginEnvironment { /** - * Parses the input stream. + * Load all of the plugins defined in the given {@link Set} of {@link PluginMetaData}. * - * @param is The input stream. - * @param name The name of the file. + * @param plugins The plugins to be loaded. */ - public void parse(InputStream is, String name); + void load(Collection plugins); /** * Sets the context for this environment. diff --git a/game/src/main/org/apollo/game/plugin/PluginManager.java b/game/src/main/org/apollo/game/plugin/PluginManager.java index d816a733..db1b45dd 100644 --- a/game/src/main/org/apollo/game/plugin/PluginManager.java +++ b/game/src/main/org/apollo/game/plugin/PluginManager.java @@ -149,43 +149,7 @@ public final class PluginManager { PluginEnvironment env = new KotlinPluginEnvironment(world); // TODO isolate plugins if possible in the future! env.setContext(context); - for (PluginMetaData plugin : plugins.values()) { - start(env, plugin, plugins, started); - } - } - - /** - * Starts a specific plugin. - * - * @param env The environment. - * @param plugin The plugin. - * @param plugins The plugin map. - * @param started A set of started plugins. - * @throws DependencyException If a dependency error occurs. - * @throws IOException If an I/O error occurs. - */ - private void start(PluginEnvironment env, PluginMetaData plugin, Map plugins, Set started) throws DependencyException, IOException { - // TODO check for cyclic dependencies! this way just won't cut it, we need an exception - if (started.contains(plugin)) { - return; - } - started.add(plugin); - - for (String dependencyId : plugin.getDependencies()) { - PluginMetaData dependency = plugins.get(dependencyId); - if (dependency == null) { - throw new DependencyException("Unresolved dependency: " + dependencyId + "."); - } - start(env, dependency, plugins, started); - } - - String[] scripts = plugin.getScripts(); - - for (String script : scripts) { - File scriptFile = new File(plugin.getBase(), script); - InputStream is = new FileInputStream(scriptFile); - env.parse(is, scriptFile.getAbsolutePath()); - } + env.load(plugins.values()); } } \ No newline at end of file diff --git a/game/src/main/org/apollo/game/plugin/RubyPluginEnvironment.java b/game/src/main/org/apollo/game/plugin/RubyPluginEnvironment.java deleted file mode 100644 index 9ac48c67..00000000 --- a/game/src/main/org/apollo/game/plugin/RubyPluginEnvironment.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.apollo.game.plugin; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; - -import org.apollo.game.model.World; -import org.jruby.embed.ScriptingContainer; - -/** - * A {@link PluginEnvironment} which uses Ruby. - * - * @author Graham - */ -public final class RubyPluginEnvironment implements PluginEnvironment { - - /** - * The scripting container. - */ - private final ScriptingContainer container = new ScriptingContainer(); - - /** - * Creates and bootstraps the Ruby plugin environment. - * - * @param world The {@link World} this RubyPluginEnvironment is for. - * @throws IOException If an I/O error occurs during bootstrapping. - */ - public RubyPluginEnvironment(World world) throws IOException { - container.put("$world", world); - parseBootstrapper(); - } - - @Override - public void parse(InputStream is, String name) { - try { - container.runScriptlet(is, name); - } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException("Error parsing scriptlet " + name + ".", e); - } - } - - /** - * Parses the bootstrapper. - * - * @throws IOException If an I/O error occurs. - */ - private void parseBootstrapper() throws IOException { - File bootstrap = new File("./data/plugins/bootstrap.rb"); - try (InputStream is = new FileInputStream(bootstrap)) { - parse(is, bootstrap.getAbsolutePath()); - } - } - - @Override - public void setContext(PluginContext context) { - container.put("$ctx", context); - } - -} \ No newline at end of file