Compile plugins at build-time instead of runtime

Adds gradle tasks to build all plugin scripts under data/plugins with
the KotlinPluginCompiler implementation previously used for runtime code
generation.  In addition to .plugin.kts files, scripts can also declare
API code in .kt files which will also be included on the classpath and
made available to other plugins.

build.gradle

@@ -48,7 +48,7 @@ subprojects {
 
 def gameSubproject = project(':game')
 
-task(run, dependsOn: gameSubproject.tasks['classes'], type: JavaExec) {
+task(run, dependsOn: gameSubproject.tasks['assemble'], type: JavaExec) {
     def gameClasspath = gameSubproject.sourceSets.main.runtimeClasspath
 
     main = 'org.apollo.Server'

game/build.gradle

@@ -14,9 +14,15 @@ apply plugin: 'kotlin'
 
 sourceSets {
     main.kotlin.srcDirs += 'src/kotlin'
+    main.kotlin.excludes += 'stub.kt'
+    main.kotlin.excludes += '**/*.kts'
 
     plugins {
-        kotlin.srcDirs += 'data/plugins'
+        kotlin {
+            srcDir 'data/plugins'
+            exclude 'stub.kt'
+            exclude '**/*.kts'
+        }
     }
 }
 
@@ -28,6 +34,33 @@ dependencies {
     pluginsCompile(configurations.compile)
     pluginsCompile(sourceSets.main.output)
 
-    compile group: 'org.jetbrains.kotlin', name: 'kotlin-compiler', version: "$kotlinVersion"
     compile group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jre8', version: "$kotlinVersion"
+    compile group: 'org.jetbrains.kotlin', name: 'kotlin-compiler', version: "$kotlinVersion"
+
+    runtime files("$buildDir/plugins")
+    runtime sourceSets.plugins.output
 }
+
+
+task('compilePluginScripts', dependsOn: [classes, pluginsClasses], type: JavaExec) {
+    def compilerClasspath = [
+            configurations.compile.asPath,
+            configurations.runtime.asPath,
+            sourceSets.main.compileClasspath.asPath,
+            sourceSets.main.runtimeClasspath.asPath
+    ]
+
+    def inputDir = "$projectDir/data/plugins"
+    def outputDir = "$buildDir/plugins"
+    def manifestPath = "$buildDir/plugins/manifest.txt"
+
+    inputs.source inputDir
+    outputs.dir outputDir
+
+    println compilerClasspath.join(':')
+    classpath = sourceSets.main.compileClasspath + sourceSets.main.runtimeClasspath
+    main = 'org.apollo.game.plugin.kotlin.KotlinPluginCompiler'
+    args = [inputDir, outputDir, manifestPath, compilerClasspath.join(':')]
+}
+
+assemble.dependsOn(compilePluginScripts)

game/src/kotlin/org/apollo/game/plugin/kotlin/KotlinPluginCompiler.kt

@@ -1,41 +1,132 @@
 package org.apollo.game.plugin.kotlin
 
+import com.google.common.base.CaseFormat
 import com.intellij.openapi.util.Disposer
-import org.apollo.Server
-import org.apollo.game.model.World
-import org.apollo.game.plugin.PluginContext
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
-import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.messages.*
-import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
+import org.jetbrains.kotlin.cli.jvm.compiler.*
-import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
-import org.jetbrains.kotlin.cli.jvm.compiler.KotlinToJVMBytecodeCompiler
 import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot
 import org.jetbrains.kotlin.codegen.CompilationException
-import org.jetbrains.kotlin.config.CommonConfigurationKeys
+import org.jetbrains.kotlin.config.*
-import org.jetbrains.kotlin.config.CompilerConfiguration
-import org.jetbrains.kotlin.config.JVMConfigurationKeys
-import org.jetbrains.kotlin.config.addKotlinSourceRoot
 import org.jetbrains.kotlin.script.KotlinScriptDefinitionFromAnnotatedTemplate
 import java.io.File
+import java.lang.management.ManagementFactory
+import java.net.URISyntaxException
+import java.net.URLClassLoader
+import java.nio.file.*
+import java.nio.file.StandardOpenOption.*
+import java.nio.file.attribute.BasicFileAttributes
+import java.util.*
+import java.util.function.BiPredicate
+
+class KotlinMessageCollector : MessageCollector {
+
+    override fun clear() {
+    }
+
+    override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation) {
+        if (severity.isError) {
+            println("${location.path}:${location.line}-${location.column}: $message")
+            println(">>> ${location.lineContent}")
+        }
+    }
+
+    override fun hasErrors(): Boolean {
+        return false
+    }
+
+}
+
+data class KotlinCompilerResult(val fqName: String, val outputPath: Path)
 
 class KotlinPluginCompiler(val classpath: List<File>, val messageCollector: MessageCollector) {
 
-    private fun createCompilerConfiguration(inputPath: String): CompilerConfiguration {
+    companion object {
+
+        private val maxSearchDepth = 1024;
+
+        fun currentClasspath(): List<File> {
+            val classLoader = Thread.currentThread().contextClassLoader as? URLClassLoader ?:
+                    throw RuntimeException("Unable to resolve classpath for current ClassLoader")
+
+            val classpathUrls = classLoader.urLs
+            val classpath = ArrayList<File>()
+
+            for (classpathUrl in classpathUrls) {
+                try {
+                    classpath.add(File(classpathUrl.toURI()))
+                } catch (e: URISyntaxException) {
+                    throw RuntimeException("URL returned by ClassLoader is invalid")
+                }
+
+            }
+
+            return classpath
+        }
+
+        @JvmStatic
+        fun main(args: Array<String>) {
+            if (args.size < 4) throw RuntimeException("Usage: <inputDir> <outputDir> <manifestPath> <classpath>")
+
+            val inputDir = Paths.get(args[0])
+            val outputDir = Paths.get(args[1])
+            val manifestPath = Paths.get(args[2])
+            val classpathEntries = args[3].split(':')
+            val classpath = mutableListOf<File>()
+
+            val runtimeBean = ManagementFactory.getRuntimeMXBean()
+            if (!runtimeBean.isBootClassPathSupported) {
+                println("Warning!  Boot class path is not supported, must be supplied on the command line")
+            } else {
+                val bootClasspath = runtimeBean.bootClassPath
+                classpath.addAll(bootClasspath.split(':').map { File(it) })
+            }
+
+            /**
+             * Classpath entries on the command line contain the kotlin runtime
+             * and plugin API code.  We use our current classpath to provide
+             * Kotlin with access to the JRE and apollo modules.
+             */
+            classpath.addAll(currentClasspath())
+            classpath.addAll(classpathEntries.map { File(it) })
+
+            val inputScriptsMatcher = { path: Path, _: BasicFileAttributes -> path.toString().endsWith(".plugin.kts") }
+            val inputScripts = Files.find(inputDir, maxSearchDepth, BiPredicate(inputScriptsMatcher))
+
+            val compiler = KotlinPluginCompiler(classpath, MessageCollector.NONE)
+            val compiledScriptClasses = mutableListOf<String>()
+
+            try {
+                Files.createDirectory(outputDir)
+
+                inputScripts.forEach {
+                    compiledScriptClasses.add(compiler.compile(it, outputDir).fqName)
+                }
+
+                Files.write(manifestPath, compiledScriptClasses, CREATE, TRUNCATE_EXISTING)
+            } catch (t: Throwable) {
+                t.printStackTrace()
+                System.exit(1)
+            }
+        }
+    }
+
+    private fun createCompilerConfiguration(inputPath: Path): CompilerConfiguration {
         val configuration = CompilerConfiguration()
         val scriptDefinition = KotlinScriptDefinitionFromAnnotatedTemplate(KotlinPluginScript::class)
 
         configuration.add(JVMConfigurationKeys.SCRIPT_DEFINITIONS, scriptDefinition)
         configuration.put(JVMConfigurationKeys.CONTENT_ROOTS, classpath.map { JvmClasspathRoot(it) })
         configuration.put(JVMConfigurationKeys.RETAIN_OUTPUT_IN_MEMORY, true)
-        configuration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, messageCollector)
+        configuration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, KotlinMessageCollector())
-        configuration.put(CommonConfigurationKeys.MODULE_NAME, inputPath)
+        configuration.put(CommonConfigurationKeys.MODULE_NAME, inputPath.toString())
-        configuration.addKotlinSourceRoot(inputPath)
+        configuration.addKotlinSourceRoot(inputPath.toAbsolutePath().toString())
 
         return configuration
     }
 
     @Throws(KotlinPluginCompilerException::class)
-    fun compile(inputPath: String): Class<out KotlinPluginScript> {
+    fun compile(inputPath: Path, outputPath: Path): KotlinCompilerResult {
         val rootDisposable = Disposer.newDisposable()
         val configuration = createCompilerConfiguration(inputPath)
 
@@ -43,13 +134,26 @@ class KotlinPluginCompiler(val classpath: List<File>, val messageCollector: Mess
         val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, configFiles)
 
         try {
-            val clazz = KotlinToJVMBytecodeCompiler.compileScript(environment, Server::class.java.classLoader)
+            val generationState = KotlinToJVMBytecodeCompiler.analyzeAndGenerate(environment)
-
+            if (generationState == null) {
-            if (clazz?.getConstructor(World::class.java, PluginContext::class.java) == null) {
+                throw KotlinPluginCompilerException("Failed to generate bytecode for kotlin script")
-                throw KotlinPluginCompilerException("Unable to compile $inputPath, no plugin constructor found")
             }
 
-            return clazz as Class<out KotlinPluginScript>
+            val sourceFiles = environment.getSourceFiles()
+            val script = sourceFiles[0].script ?: throw KotlinPluginCompilerException("Main script file isnt a script")
+
+            val scriptFilePath = script.fqName.asString().replace('.', '/') + ".class"
+            val scriptFileClass = generationState.factory.get(scriptFilePath)
+
+            if (scriptFileClass == null) {
+                throw KotlinPluginCompilerException("Unable to find compiled plugin class file $scriptFilePath")
+            }
+
+            generationState.factory.asList().forEach {
+                Files.write(outputPath.resolve(it.relativePath), it.asByteArray(), CREATE, WRITE, TRUNCATE_EXISTING)
+            }
+
+            return KotlinCompilerResult(script.fqName.asString(), outputPath.resolve(scriptFileClass.relativePath))
         } catch (e: CompilationException) {
             throw KotlinPluginCompilerException("Compilation failed", e)
         } finally {

game/src/main/org/apollo/game/plugin/KotlinPluginEnvironment.java

@@ -1,5 +1,20 @@
 package org.apollo.game.plugin;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.reflect.Constructor;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
 import org.apollo.game.model.World;
 import org.apollo.game.plugin.kotlin.KotlinPluginCompiler;
 import org.apollo.game.plugin.kotlin.KotlinPluginScript;
@@ -8,17 +23,6 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation;
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity;
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector;
 
-import java.io.File;
-import java.io.InputStream;
-import java.lang.reflect.Constructor;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
 public class KotlinPluginEnvironment implements PluginEnvironment, MessageCollector {
 
 	private static final Logger logger = Logger.getLogger(KotlinPluginEnvironment.class.getName());
@@ -61,17 +65,56 @@ public class KotlinPluginEnvironment implements PluginEnvironment, MessageCollec
 	}
 
 	@Override
-	public void parse(InputStream is, String name) {
+	public void load(Collection<PluginMetaData> plugins) {
-		//@todo - wait until all plugin classes are loading until running constructors?
+		try (InputStream resource = KotlinPluginEnvironment.class.getResourceAsStream("/manifest.txt")) {
-		try {
+			BufferedReader reader = new BufferedReader(new InputStreamReader(resource));
-			Class<? extends KotlinPluginScript> pluginClass = pluginCompiler.compile(name);
+			List<String> pluginClassNames = reader.lines().collect(Collectors.toList());
-			Constructor<? extends KotlinPluginScript> pluginConstructor = pluginClass.getConstructor(World.class,
-				PluginContext.class);
 
-			pluginConstructor.newInstance(world, context);
+			for (String pluginClassName : pluginClassNames) {
+				Class<? extends KotlinPluginScript> pluginClass =
+					(Class<? extends KotlinPluginScript>) Class.forName(pluginClassName);
+
+				Constructor<? extends KotlinPluginScript> pluginConstructor =
+					pluginClass.getConstructor(World.class, PluginContext.class);
+
+				KotlinPluginScript plugin = pluginConstructor.newInstance(world, context);
+			}
 		} catch (Exception e) {
 			throw new RuntimeException(e);
 		}
+
+		List<Class<? extends KotlinPluginScript>> pluginClasses = new ArrayList<>();
+		List<String> sourceRoots = new ArrayList<>();
+
+		for (PluginMetaData plugin : plugins) {
+			List<String> pluginSourceRoots = Arrays.stream(plugin.getScripts())
+				.map(script -> plugin.getBase() + "/" + script)
+				.collect(Collectors.toList());
+
+			sourceRoots.addAll(pluginSourceRoots);
+		}
+
+		for (String scriptSource : sourceRoots) {
+//			try {
+			List<String> dependencySourceRoots = new ArrayList<>(sourceRoots);
+			dependencySourceRoots.remove(scriptSource);
+
+//				pluginClasses.add(pluginCompiler.compile(scriptSource));
+//			} catch (KotlinPluginCompilerException e) {
+//				throw new RuntimeException(e);
+//			}
+		}
+
+		for (Class<? extends KotlinPluginScript> pluginClass : pluginClasses) {
+			try {
+				Constructor<? extends KotlinPluginScript> constructor = pluginClass
+					.getConstructor(World.class, PluginContext.class);
+
+				KotlinPluginScript script = constructor.newInstance(world, context);
+			} catch (Exception e) {
+				throw new RuntimeException(e);
+			}
+		}
 	}
 
 	@Override
@@ -89,7 +132,7 @@ public class KotlinPluginEnvironment implements PluginEnvironment, MessageCollec
 					   @NotNull CompilerMessageLocation location) {
 		if (severity.isError()) {
 			logger.log(Level.SEVERE, String.format("%s:%s-%s: %s", location.getPath(), location.getLine(),
-				location.getColumn(), message));
+												   location.getColumn(), message));
 		}
 	}
 

game/src/main/org/apollo/game/plugin/PluginEnvironment.java

@@ -1,6 +1,8 @@
 package org.apollo.game.plugin;
 
 import java.io.InputStream;
+import java.util.Collection;
+import java.util.Set;
 
 /**
  * Represents some sort of environment that plugins could be executed in, e.g. {@code javax.script} or Jython.
@@ -10,12 +12,11 @@ import java.io.InputStream;
 public interface PluginEnvironment {
 
 	/**
-	 * Parses the input stream.
+	 * Load all of the plugins defined in the given {@link Set} of {@link PluginMetaData}.
 	 *
-	 * @param is The input stream.
+	 * @param plugins The plugins to be loaded.
-	 * @param name The name of the file.
 	 */
-	public void parse(InputStream is, String name);
+	void load(Collection<PluginMetaData> plugins);
 
 	/**
 	 * Sets the context for this environment.

game/src/main/org/apollo/game/plugin/PluginManager.java

@@ -149,43 +149,7 @@ public final class PluginManager {
 		PluginEnvironment env = new KotlinPluginEnvironment(world); // TODO isolate plugins if possible in the future!
 		env.setContext(context);
 
-		for (PluginMetaData plugin : plugins.values()) {
+		env.load(plugins.values());
-			start(env, plugin, plugins, started);
-		}
-	}
-
-	/**
-	 * Starts a specific plugin.
-	 *
-	 * @param env The environment.
-	 * @param plugin The plugin.
-	 * @param plugins The plugin map.
-	 * @param started A set of started plugins.
-	 * @throws DependencyException If a dependency error occurs.
-	 * @throws IOException If an I/O error occurs.
-	 */
-	private void start(PluginEnvironment env, PluginMetaData plugin, Map<String, PluginMetaData> plugins, Set<PluginMetaData> started) throws DependencyException, IOException {
-		// TODO check for cyclic dependencies! this way just won't cut it, we need an exception
-		if (started.contains(plugin)) {
-			return;
-		}
-		started.add(plugin);
-
-		for (String dependencyId : plugin.getDependencies()) {
-			PluginMetaData dependency = plugins.get(dependencyId);
-			if (dependency == null) {
-				throw new DependencyException("Unresolved dependency: " + dependencyId + ".");
-			}
-			start(env, dependency, plugins, started);
-		}
-
-		String[] scripts = plugin.getScripts();
-
-		for (String script : scripts) {
-			File scriptFile = new File(plugin.getBase(), script);
-			InputStream is = new FileInputStream(scriptFile);
-			env.parse(is, scriptFile.getAbsolutePath());
-		}
 	}
 
 }

game/src/main/org/apollo/game/plugin/RubyPluginEnvironment.java

@@ -1,61 +0,0 @@
-package org.apollo.game.plugin;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apollo.game.model.World;
-import org.jruby.embed.ScriptingContainer;
-
-/**
- * A {@link PluginEnvironment} which uses Ruby.
- *
- * @author Graham
- */
-public final class RubyPluginEnvironment implements PluginEnvironment {
-
-	/**
-	 * The scripting container.
-	 */
-	private final ScriptingContainer container = new ScriptingContainer();
-
-	/**
-	 * Creates and bootstraps the Ruby plugin environment.
-	 *
-	 * @param world The {@link World} this RubyPluginEnvironment is for.
-	 * @throws IOException If an I/O error occurs during bootstrapping.
-	 */
-	public RubyPluginEnvironment(World world) throws IOException {
-		container.put("$world", world);
-		parseBootstrapper();
-	}
-
-	@Override
-	public void parse(InputStream is, String name) {
-		try {
-			container.runScriptlet(is, name);
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw new RuntimeException("Error parsing scriptlet " + name + ".", e);
-		}
-	}
-
-	/**
-	 * Parses the bootstrapper.
-	 *
-	 * @throws IOException If an I/O error occurs.
-	 */
-	private void parseBootstrapper() throws IOException {
-		File bootstrap = new File("./data/plugins/bootstrap.rb");
-		try (InputStream is = new FileInputStream(bootstrap)) {
-			parse(is, bootstrap.getAbsolutePath());
-		}
-	}
-
-	@Override
-	public void setContext(PluginContext context) {
-		container.put("$ctx", context);
-	}
-
-}