From 6704454ac541890f33706afbc7019c272d53a4f7 Mon Sep 17 00:00:00 2001 From: austin Date: Sat, 5 Apr 2025 17:19:42 -0400 Subject: [PATCH] script cache busting --- docs/src/progress/renderertodo.md | 1 + .../electrosphere/script/ScriptContext.java | 14 +++- .../electrosphere/script/ScriptEngine.java | 68 ++++++++++++++++--- .../script/ScriptFileChecksumMap.java | 37 ++++++++++ .../java/electrosphere/util/FileUtils.java | 14 ++++ 5 files changed, 122 insertions(+), 12 deletions(-) create mode 100644 src/main/java/electrosphere/script/ScriptFileChecksumMap.java diff --git a/docs/src/progress/renderertodo.md b/docs/src/progress/renderertodo.md index e6b86beb..d479753d 100644 --- a/docs/src/progress/renderertodo.md +++ b/docs/src/progress/renderertodo.md @@ -1452,6 +1452,7 @@ Stone Axe item Fix human data for RH sword slash attack moves Cursor only for specific items Added shovel (works inversely of how you would expect currently) +Script cache busting on file modification diff --git a/src/main/java/electrosphere/script/ScriptContext.java b/src/main/java/electrosphere/script/ScriptContext.java index 72cce355..1bef3109 100644 --- a/src/main/java/electrosphere/script/ScriptContext.java +++ b/src/main/java/electrosphere/script/ScriptContext.java @@ -228,10 +228,11 @@ public class ScriptContext { * Compiles the project */ protected void compile(){ + ScriptFileChecksumMap checksumMap = this.parent.getChecksumMap(); //actually compile this.invokeMemberFunction("COMPILER", "run"); - //TODO:update local cache Value fileMap = this.topLevelValue.getMember("COMPILER").getMember("fileMap"); + //register new files, update cache where appropriate for(String key : fileMap.getMemberKeys()){ Value fileData = fileMap.getMember(key); String content = fileData.getMember("content").asString(); @@ -245,6 +246,15 @@ public class ScriptContext { LoggerInterface.loggerFileIO.ERROR(e); } + //update cached timestamp + { + String pathRaw = toWriteFile.toPath() + ""; + pathRaw = pathRaw.replace(".\\.cache\\tscache\\src\\", "./assets/"); + File correspondingFile = new File(pathRaw.replace(".\\.cache\\tscache\\src\\", "./assets/")); + String cacheKey = pathRaw.replace("./assets", "").replace("\\", "/"); + checksumMap.getFileLastModifyMap().put(cacheKey, correspondingFile.lastModified() + ""); + } + //write the actual file try { Files.writeString(toWriteFile.toPath(), content); @@ -252,6 +262,8 @@ public class ScriptContext { LoggerInterface.loggerFileIO.ERROR(e); } } + //write out cache map file + this.parent.writeChecksumMap(); } /** diff --git a/src/main/java/electrosphere/script/ScriptEngine.java b/src/main/java/electrosphere/script/ScriptEngine.java index 4e5903db..428910d4 100644 --- a/src/main/java/electrosphere/script/ScriptEngine.java +++ b/src/main/java/electrosphere/script/ScriptEngine.java @@ -65,9 +65,9 @@ public class ScriptEngine extends SignalServiceImpl { Map sourceMap; /** - * Stores all loaded files' md5 checksums + * The storage for the file->checksum map */ - Map fileChecksumMap = new HashMap(); + ScriptFileChecksumMap checksumMap; /** * The script context @@ -225,44 +225,70 @@ public class ScriptEngine extends SignalServiceImpl { * @return true if files were read from cache, false otherwise */ private boolean initCache(){ + boolean rVal = false; File tsCache = new File(ScriptEngine.TS_SOURCE_CACHE_DIR); + this.checksumMap = new ScriptFileChecksumMap(); if(!tsCache.exists()){ try { Files.createDirectories(tsCache.toPath()); } catch (IOException e) { LoggerInterface.loggerFileIO.ERROR(e); } - return false; } else { + //read file checksum map from disk if it exists + File hashMapFile = new File(ScriptEngine.TS_SOURCE_CACHE_DIR + "/hashmap.json"); + if(hashMapFile.exists()){ + this.checksumMap = FileUtils.loadObjectFromFile(hashMapFile, ScriptFileChecksumMap.class); + } + + //recursively read all files Value fileMap = this.scriptContext.getTopLevelValue("COMPILER").getMember("fileMap"); - this.recursivelyRegisterCachedFiles(tsCache,fileMap,tsCache); - return true; + rVal = this.recursivelyRegisterCachedFiles(tsCache,".",fileMap,tsCache,this.checksumMap.getFileChecksumMap(),this.checksumMap.getFileLastModifyMap()); } + return rVal; } /** * Register files recursively from the current file * @param tsCache The ts cache file + * @param compoundedPath The compounded path * @param fileMap The file map on script side * @param currentDirectory The current directory + * @param fileChecksumMap The file checksum map + * @param fileLastModifyMap The file last modified map + * @return true if all files were found in cache, false if some files were skipped */ - private void recursivelyRegisterCachedFiles(File tsCache, Value fileMap, File currentDirectory){ + private boolean recursivelyRegisterCachedFiles(File tsCache, String compoundedPath, Value fileMap, File currentDirectory, Map fileChecksumMap, Map fileLastModifyMap){ + boolean rVal = true; for(File file : currentDirectory.listFiles()){ if(file.isDirectory()){ - this.recursivelyRegisterCachedFiles(tsCache, fileMap, file); + String newPath = compoundedPath + "/" + file.getName(); + boolean childSuccess = this.recursivelyRegisterCachedFiles(tsCache, newPath, fileMap, file, fileChecksumMap, fileLastModifyMap); + if(!childSuccess){ + rVal = false; + } } else if(file.getPath().endsWith(".ts") || file.getPath().endsWith(".js")){ try { String relativePath = FileUtils.relativize(file, tsCache); String normalizedPath = "/" + relativePath; + boolean shouldLoad = true; + File correspondingSourceFile = FileUtils.getAssetFile(compoundedPath + "/" + file.getName()); - if(!this.fileChecksumMap.containsKey(normalizedPath)){ - - //read file + //determine if we should load the file + if(!fileLastModifyMap.containsKey(normalizedPath)){ + //cache does not contain this file + shouldLoad = false; + } else if(!fileLastModifyMap.get(normalizedPath).contains(correspondingSourceFile.lastModified() + "")) { + //cache is up to date + shouldLoad = false; + } + //actually load the file + if(shouldLoad){ String fileContent = Files.readString(file.toPath()); //store checksum try { - this.fileChecksumMap.put(normalizedPath,FileUtils.getChecksum(fileContent)); + fileChecksumMap.put(normalizedPath,FileUtils.getChecksum(fileContent)); } catch (NoSuchAlgorithmException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -271,12 +297,32 @@ public class ScriptEngine extends SignalServiceImpl { //store on script side LoggerInterface.loggerScripts.DEBUG("Preload: " + normalizedPath); this.scriptContext.getTopLevelValue("COMPILER").invokeMember("preloadFile", normalizedPath, fileContent); + } else { + rVal = false; } } catch (IOException e) { LoggerInterface.loggerFileIO.ERROR(e); } } } + return rVal; + } + + /** + * Gets the checksum map + * @return The checksum map + */ + protected ScriptFileChecksumMap getChecksumMap(){ + return this.checksumMap; + } + + /** + * Writes the checksum map to disk + */ + protected void writeChecksumMap(){ + File hashMapFile = new File(ScriptEngine.TS_SOURCE_CACHE_DIR + "/hashmap.json"); + //write cache map out + FileUtils.serializeObjectToFilePath(hashMapFile, this.checksumMap); } /** diff --git a/src/main/java/electrosphere/script/ScriptFileChecksumMap.java b/src/main/java/electrosphere/script/ScriptFileChecksumMap.java new file mode 100644 index 00000000..8d8ca9bf --- /dev/null +++ b/src/main/java/electrosphere/script/ScriptFileChecksumMap.java @@ -0,0 +1,37 @@ +package electrosphere.script; + +import java.util.HashMap; +import java.util.Map; + +/** + * Storage for the script file -> checksum map + */ +public class ScriptFileChecksumMap { + + /** + * Stores all loaded files' md5 checksums + */ + Map fileChecksumMap = new HashMap(); + + /** + * Stores all loaded files' last modified time + */ + Map fileModifyTimeMap = new HashMap(); + + /** + * Gets the file checksum map + * @return The map + */ + public Map getFileChecksumMap() { + return fileChecksumMap; + } + + /** + * Gets the file last modify map + * @return The map + */ + public Map getFileLastModifyMap() { + return fileModifyTimeMap; + } + +} diff --git a/src/main/java/electrosphere/util/FileUtils.java b/src/main/java/electrosphere/util/FileUtils.java index eee6622d..694ededd 100644 --- a/src/main/java/electrosphere/util/FileUtils.java +++ b/src/main/java/electrosphere/util/FileUtils.java @@ -178,6 +178,20 @@ public class FileUtils { LoggerInterface.loggerFileIO.ERROR(filePath, ex); } } + + /** + * Serializes an object to a filepath + * @param file The file to save to + * @param object The object + */ + public static void serializeObjectToFilePath(File file, Object object){ + Path path = file.toPath(); + try { + Files.write(path, gson.toJson(object).getBytes()); + } catch (IOException ex) { + LoggerInterface.loggerFileIO.ERROR(file.getAbsolutePath(), ex); + } + } /** * Gets an assets file