incremental compilation of scripts

assets/Scripts/compiler/compiler.js

@@ -25,13 +25,35 @@ let COMPILER = {
      * The top level directory, "/"
      */
     topLevelDirectory: {
+        //as reqiored by our framework
         Scripts: {
             compiler: {
                 "host_access.js": {
                     content: "",
-                }
+                    version: 0,
-            }
+                },
-        }
+                version: 0,
+                isDir: true,
+            },
+            version: 0,
+            isDir: true,
+        },
+        //as required by language service
+        node_modules: {
+            "@types": {
+                "lib.d.ts": {
+                    content: "",
+                    version: 0,
+                    isDir: false,
+                },
+                version: 0,
+                isDir: true,
+            },
+            version: 0,
+            isDir: true,
+        },
+        version: 0,
+        isDir: true,
     },
 
     /**
@@ -41,44 +63,44 @@ let COMPILER = {
 
     /**
      * Registers a file with the compiler
-     * @param {*} fileName The file's name
+     * @param {string} fileName The file's name
-     * @param {*} content The content of the file
+     * @param {string} content The content of the file
      * @returns The list of all files that still need to be registered by the host
      */
     registerFile: (fileName, content) => [],
 
     /**
      * Creates a file object for a given path
-     * @param {*} fileName The name of the file
+     * @param string} fileName The name of the file
-     * @param {*} content The content of the file
+     * @param {string} content The content of the file
      * @returns The file object
      */
     createFile: (fileName, content) => null,
 
     /**
      * Gets the path for the file
-     * @param {*} fullyQualifiedFilePath The fully qualified file path
+     * @param {string} fullyQualifiedFilePath The fully qualified file path
-     * @returns The array of directories ending with the name of the file
+     * @returns {string[]} The array of directories ending with the name of the file
      */
     getPath: (fullyQualifiedFilePath) => null,
 
     /**
      * Gets the path for the file
-     * @param {*} fullyQualifiedFilePath The fully qualified file path
+     * @param {stringp[]} filePathArray The fully qualified file path
      * @returns The array of directories ending with the name of the file
      */
     getFileByPath: (filePathArray) => null,
 
     /**
      * Checks if a file exists
-     * @param {*} filePathArray The file path array
+     * @param {string[]} filePathArray The file path array
      * @returns true if it exists, false otherwise
      */
     fileExists: (filePathArray) => null,
 
     /**
      * The callback invoked when the compiler host tries to read a file
-     * @param {*} fileName The name of the file
+     * @param {string} fileName The name of the file
      * @param {*} languageVersion The language version
      * @returns The file if it exists, null otherwise
      */
@@ -96,6 +118,25 @@ let COMPILER = {
      */
     compilerOptions: { },
 
+    /**
+     * Tracks whether the compiler has run or not
+     */
+    compilerHasRun: false,
+
+    /**
+     * Emits a file
+     * @param {string} fileName The name of the file
+     * @returns {void}
+     */
+    emitFile: (fileName) => null,
+
+    /**
+     * Logs errors raised during emission of files
+     * @param {string} fileName The name of the file to log errors about
+     * @returns {void}
+     */
+    logEmitError: (fileName) => null,
+
     /**
      * Instructs Typescript to emit the final compiled value
      */
@@ -153,7 +194,7 @@ COMPILER.getFileByPath = (filePathArray) => {
         throw new Error("Trying to get a file with a path array of length 0!")
     }
 
-    while(mutableArray.length > 1){
+    while(mutableArray?.length > 1){
         let nextDirName = mutableArray.shift()
         currentFolder = currentFolder?.[nextDirName]
         if(!currentFolder){
@@ -162,7 +203,7 @@ COMPILER.getFileByPath = (filePathArray) => {
             throw new Error(errorMessage)
         }
     }
-    return currentFolder[mutableArray[0]]
+    return currentFolder[mutableArray?.[0]]
 }
 
 
@@ -222,6 +263,7 @@ COMPILER.createFile = (fileName, content) => {
         isDir: false,
         dir: currentFolder,
         content: content,
+        version: 0,
     }
 
     //return the file
@@ -305,6 +347,11 @@ COMPILER.registerFile = (fileName, content) => {
                 loggerScripts.INFO(" - " + normalizedDependentFilePath)
             }
         })
+
+        //If the compiler has already run once, run the language service against only this file
+        if(!!COMPILER.compilerHasRun){
+            COMPILER.emitFile(fileName)
+        }
     }
     return dependentFiles;
 }
@@ -352,33 +399,108 @@ COMPILER.customCompilerHost = {
             moduleContent: finalData, //to be eval'd from require()
         }
     },
-    getDefaultLibFileName: () => "lib.d.ts",
+    getDefaultLibFileName: ts.getDefaultLibFileName,
     useCaseSensitiveFileNames: () => false,
     getCanonicalFileName: filename => filename,
     getCurrentDirectory: () => "/",
     getNewLine: () => "\n",
     getDirectories: (path) => {
-        loggerScripts.DEBUG('[ts] get dirs ' + path)
+        loggerScripts.DEBUG('[ts] getDirectories ' + path)
         const dirs = Object.keys(COMPILER.getFileByPath(COMPILER.getPath(path)))
         loggerScripts.DEBUG('[ts] dirs: ' + dirs)
         return dirs
     },
     directoryExists: (path) => {
-        const exists = COMPILER.fileExists(COMPILER.getPath(path))
+        let exists = COMPILER.fileExists(COMPILER.getPath(path))
-        loggerScripts.DEBUG('[ts] directory exists? ' + path + " - " + exists)
+        if(exists){
-        return exists
+            exists = COMPILER.getFileByPath(COMPILER.getPath(path))?.isDir
+        }
+        loggerScripts.DEBUG('[ts] directoryExists ' + path + " - " + exists)
+        return false
     },
     fileExists: (path) => {
         const exists = COMPILER.fileExists(COMPILER.getPath(path))
-        loggerScripts.DEBUG('[ts] file exists? ' + path + " - " + exists)
+        loggerScripts.DEBUG('[ts] fileExists ' + path + " - " + exists)
         return exists
     },
     readFile: (path) => {
-        loggerScripts.DEBUG('[ts] read file ' + path)
+        loggerScripts.DEBUG('[ts] readFile ' + path)
         const file = COMPILER.getFileByPath(COMPILER.getPath(path))
-        loggerScripts.DEBUG('[ts] content: ' + file.content)
+        loggerScripts.DEBUG('[ts] readFile (content): ' + file.content)
         return file.content
     },
+    getScriptFileNames: () => {
+        loggerScripts.DEBUG('[ts] getScriptFileNames')
+        return COMPILER.sourceFiles
+    },
+    getScriptVersion: (fileName) => {
+        loggerScripts.DEBUG('[ts] getScriptVersion: ' + fileName)
+        const file = COMPILER.getFileByPath(COMPILER.getPath(fileName))
+        return file?.version
+    },
+    //https://github.com/microsoft/TypeScript/wiki/Using-the-Language-Service-API#scriptsnapshot
+    getScriptSnapshot: (fileName) => {
+        loggerScripts.DEBUG('[ts] getScriptSnapshot: ' + fileName)
+        const file = COMPILER.getFileByPath(COMPILER.getPath(fileName))
+        if(file){
+            return ts.ScriptSnapshot.fromString(file.content)
+        } else {
+            return undefined
+        }
+    },
+    getCompilationSettings: () => COMPILER.compilerOptions,
+
+}
+
+/**
+ * Emits a file
+ * @param {string} fileName The name of the file
+ * @returns {void}
+ */
+COMPILER.emitFile = (fileName) => {
+    loggerScripts.DEBUG('Compiler evaluating source path ' + fileName)
+    /**
+     * {
+     *   outputFiles: [ ],
+     *   emitSkipped: boolean,
+     *   diagnostics: { },
+     * }
+     */
+    const output = COMPILER.program.getEmitOutput(fileName)
+    if (!output.emitSkipped) {
+        output.outputFiles.forEach(outputFile => {
+            loggerScripts.DEBUG(`[ts] Emitting ${outputFile}`);
+            COMPILER.customCompilerHost.writeFile(outputFile.name, outputFile.text)
+        })
+    } else {
+        loggerScripts.DEBUG(`[ts] Emitting ${fileName} failed`);
+        COMPILER.logEmitError(fileName);
+    }
+}
+
+/**
+ * Logs errors raised during emission of files
+ * @param {string} fileName The name of the file to log errors about
+ * @returns {void}
+ */
+const logEmitError = (fileName) => {
+    loggerScripts.DEBUG('[ts] logErrors ' + fileName)
+    let allDiagnostics = services
+        .getCompilerOptionsDiagnostics()
+        .concat(services.getSyntacticDiagnostics(fileName))
+        .concat(services.getSemanticDiagnostics(fileName));
+
+    allDiagnostics.forEach(diagnostic => {
+        let message = ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n");
+        if (diagnostic.file) {
+            let { line, character } = diagnostic.file.getLineAndCharacterOfPosition(
+                diagnostic.start
+            );
+            loggerScripts.DEBUG(`[ts]  Error ${diagnostic.file.fileName} (${line + 1},${character +1}): ${message}`);
+        } else {
+            loggerScripts.DEBUG(`[ts]  Error: ${message}`);
+        }
+    });
 }
 
 /**
@@ -387,16 +509,17 @@ COMPILER.customCompilerHost = {
 COMPILER.run = () => {
     loggerScripts.INFO('COMPILE ALL REGISTERED FILES')
 
-    COMPILER.program = ts.createProgram(
+    if(!COMPILER.program){
-        COMPILER.sourceFiles, COMPILER.compilerOptions, COMPILER.customCompilerHost
+        COMPILER.program = ts.createLanguageService(COMPILER.customCompilerHost, ts.createDocumentRegistry());
-    )
+    }
-    console.log('program')
+
-    console.log(COMPILER.program)
+    //Emit all currently known files
-    console.log(Object.keys(COMPILER.program))
+    COMPILER.sourceFiles.forEach(sourcePath => {
-    const emitResult = COMPILER.program.emit()
+        COMPILER.emitFile(sourcePath)
-    console.log("Emit result")
+    })
-    console.log(emitResult)
+
-    console.log(Object.keys(emitResult))
+    //flag that the compiler has run (ie only incrementally compile when new files are added, now)
+    COMPILER.compilerHasRun = true
 }
 
 /**

src/main/java/electrosphere/script/ScriptEngine.java

@@ -11,7 +11,6 @@ import org.graalvm.polyglot.PolyglotException;
 import org.graalvm.polyglot.Source;
 import org.graalvm.polyglot.Source.Builder;
 import org.graalvm.polyglot.Value;
-import org.graalvm.polyglot.HostAccess.Export;
 
 import electrosphere.engine.Globals;
 import electrosphere.logger.LoggerInterface;
@@ -248,8 +247,6 @@ public class ScriptEngine {
     public void initScene(String scenePath){
         //add files to virtual filesystem in script engine
         registerFile(scenePath);
-        //compile with new files added
-        compile();
         //require the module
         requireModule(scenePath);
     }