From bf8ce59b739b5de2df5cb14fbf10a6fbe8ab4571 Mon Sep 17 00:00:00 2001 From: Sky Johnson Date: Wed, 4 Jun 2025 21:50:56 -0500 Subject: [PATCH] optimize moduleLoader, re-add to runner --- moonshark.go | 2 +- runner/moduleLoader.go | 384 +++++++++++++---------------------------- runner/runner.go | 10 +- 3 files changed, 131 insertions(+), 265 deletions(-) diff --git a/moonshark.go b/moonshark.go index c1d0261..d7682e0 100644 --- a/moonshark.go +++ b/moonshark.go @@ -148,7 +148,7 @@ func (s *Moonshark) initRunner(poolSize int) error { sessions.GlobalSessionManager.SetCookieOptions("MoonsharkSID", "/", "", false, true, 86400) var err error - s.LuaRunner, err = runner.NewRunner(poolSize, s.Config.Dirs.Data, s.Config.Dirs.FS) + s.LuaRunner, err = runner.NewRunner(poolSize, s.Config.Dirs.Data, s.Config.Dirs.FS, s.Config.Dirs.Libs) if err != nil { return fmt.Errorf("lua runner init failed: %v", err) } diff --git a/runner/moduleLoader.go b/runner/moduleLoader.go index 1880d81..c228003 100644 --- a/runner/moduleLoader.go +++ b/runner/moduleLoader.go @@ -12,403 +12,267 @@ import ( luajit "git.sharkk.net/Sky/LuaJIT-to-Go" ) -// ModuleConfig holds configuration for Lua's module loading system type ModuleConfig struct { - ScriptDir string // Base directory for script being executed - LibDirs []string // Additional library directories + ScriptDir string + LibDirs []string } -// ModuleLoader manages module loading and caching type ModuleLoader struct { - config *ModuleConfig - pathCache map[string]string // Cache module paths for fast lookups - bytecodeCache map[string][]byte // Cache of compiled bytecode - debug bool - mu sync.RWMutex + config *ModuleConfig + pathCache map[string]string // For reverse lookups (path -> module name) + debug bool + mu sync.RWMutex } -// NewModuleLoader creates a new module loader func NewModuleLoader(config *ModuleConfig) *ModuleLoader { if config == nil { - config = &ModuleConfig{ - ScriptDir: "", - LibDirs: []string{}, - } + config = &ModuleConfig{} } return &ModuleLoader{ - config: config, - pathCache: make(map[string]string), - bytecodeCache: make(map[string][]byte), - debug: false, + config: config, + pathCache: make(map[string]string), } } -// EnableDebug turns on debug logging func (l *ModuleLoader) EnableDebug() { l.debug = true } -// SetScriptDir sets the script directory func (l *ModuleLoader) SetScriptDir(dir string) { l.mu.Lock() defer l.mu.Unlock() l.config.ScriptDir = dir } -// debugLog logs a message if debug mode is enabled -func (l *ModuleLoader) debugLog(format string, args ...interface{}) { +func (l *ModuleLoader) debugLog(format string, args ...any) { if l.debug { logger.Debugf("ModuleLoader "+format, args...) } } -// SetupRequire configures the require system in a Lua state func (l *ModuleLoader) SetupRequire(state *luajit.State) error { - l.mu.RLock() - defer l.mu.RUnlock() - - // Initialize our module registry in Lua - err := state.DoString(` - -- Initialize global module registry - __module_paths = {} - __module_bytecode = {} - __ready_modules = {} - - -- Create module preload table - package.preload = package.preload or {} - `) - - if err != nil { - return err - } - - // Set up package.path based on search paths + // Set package.path paths := l.getSearchPaths() pathStr := strings.Join(paths, ";") - escapedPathStr := escapeLuaString(pathStr) - return state.DoString(`package.path = "` + escapedPathStr + `"`) + return state.DoString(`package.path = "` + escapeLuaString(pathStr) + `"`) } -// getSearchPaths returns a list of Lua search paths func (l *ModuleLoader) getSearchPaths() []string { - absPaths := []string{} - seen := map[string]bool{} + var paths []string + seen := make(map[string]bool) - // Add script directory (highest priority) + // Script directory first if l.config.ScriptDir != "" { - absPath, err := filepath.Abs(l.config.ScriptDir) - if err == nil && !seen[absPath] { - absPaths = append(absPaths, filepath.Join(absPath, "?.lua")) + if absPath, err := filepath.Abs(l.config.ScriptDir); err == nil && !seen[absPath] { + paths = append(paths, filepath.Join(absPath, "?.lua")) seen[absPath] = true } } - // Add lib directories + // Library directories for _, dir := range l.config.LibDirs { if dir == "" { continue } - - absPath, err := filepath.Abs(dir) - if err == nil && !seen[absPath] { - absPaths = append(absPaths, filepath.Join(absPath, "?.lua")) + if absPath, err := filepath.Abs(dir); err == nil && !seen[absPath] { + paths = append(paths, filepath.Join(absPath, "?.lua")) seen[absPath] = true } } - return absPaths + return paths } -// PreloadModules preloads modules from library directories func (l *ModuleLoader) PreloadModules(state *luajit.State) error { l.mu.Lock() defer l.mu.Unlock() // Reset caches l.pathCache = make(map[string]string) - l.bytecodeCache = make(map[string][]byte) - - // Reset module registry in Lua - if err := state.DoString(` - -- Reset module registry - __module_paths = {} - __module_bytecode = {} - __ready_modules = {} - - -- Clear non-core modules from package.loaded - local core_modules = { - string = true, table = true, math = true, os = true, - package = true, io = true, coroutine = true, debug = true, _G = true - } + // Clear non-core modules + err := state.DoString(` + local core = {string=1, table=1, math=1, os=1, package=1, io=1, coroutine=1, debug=1, _G=1} for name in pairs(package.loaded) do - if not core_modules[name] then - package.loaded[name] = nil - end + if not core[name] then package.loaded[name] = nil end end - - -- Reset preload table package.preload = {} - `); err != nil { + `) + if err != nil { return err } - // Scan and preload modules from all library directories + // Scan and preload modules for _, dir := range l.config.LibDirs { - if dir == "" { - continue - } - - absDir, err := filepath.Abs(dir) - if err != nil { - continue - } - - l.debugLog("Scanning directory: %s", absDir) - - // Find all Lua files - err = filepath.Walk(absDir, func(path string, info os.FileInfo, err error) error { - if err != nil || info.IsDir() || !strings.HasSuffix(path, ".lua") { - return nil - } - - // Get module name from path - relPath, err := filepath.Rel(absDir, path) - if err != nil || strings.HasPrefix(relPath, "..") { - return nil - } - - // Convert path to module name - modName := strings.TrimSuffix(relPath, ".lua") - modName = strings.ReplaceAll(modName, string(filepath.Separator), ".") - - l.debugLog("Found module: %s at %s", modName, path) - - // Register in our caches - l.pathCache[modName] = path - - // Load file content - content, err := os.ReadFile(path) - if err != nil { - l.debugLog("Failed to read module file: %v", err) - return nil - } - - // Compile to bytecode - bytecode, err := state.CompileBytecode(string(content), path) - if err != nil { - l.debugLog("Failed to compile module: %v", err) - return nil - } - - // Cache bytecode - l.bytecodeCache[modName] = bytecode - - // Register in Lua - store path info - escapedPath := escapeLuaString(path) - escapedName := escapeLuaString(modName) - - if err := state.DoString(`__module_paths["` + escapedName + `"] = "` + escapedPath + `"`); err != nil { - return nil - } - - // Load bytecode and register in package.preload properly - if err := state.LoadBytecode(bytecode, path); err != nil { - return nil - } - - // Store the function in package.preload - the function is on the stack - state.GetGlobal("package") - state.GetField(-1, "preload") - state.PushString(modName) - state.PushCopy(-4) // Copy the compiled function - state.SetTable(-3) // preload[modName] = function - state.Pop(2) // Pop package and preload tables - - // Mark as ready - if err := state.DoString(`__ready_modules["` + escapedName + `"] = true`); err != nil { - state.Pop(1) // Remove the function from stack - return nil - } - - state.Pop(1) // Remove the function from stack - return nil - }) - - if err != nil { + if err := l.scanDirectory(state, dir); err != nil { return err } } - // Install optimized require implementation + // Install simplified require return state.DoString(` - -- Setup environment-aware require function function __setup_require(env) - -- Create require function specific to this environment env.require = function(modname) - -- Check if already loaded if package.loaded[modname] then return package.loaded[modname] end - - -- Check preloaded modules - if __ready_modules[modname] then - local loader = package.preload[modname] - if loader then - -- Set environment for loader - setfenv(loader, env) - - -- Execute and store result - local result = loader() - if result == nil then - result = true - end - - package.loaded[modname] = result - return result - end + + local loader = package.preload[modname] + if loader then + setfenv(loader, env) + local result = loader() or true + package.loaded[modname] = result + return result end - - -- Direct file load as fallback - if __module_paths[modname] then - local path = __module_paths[modname] - local chunk, err = loadfile(path) - if chunk then - setfenv(chunk, env) - local result = chunk() - if result == nil then - result = true - end - package.loaded[modname] = result - return result - end - end - - -- Full path search as last resort - local errors = {} + + -- Standard path search for path in package.path:gmatch("[^;]+") do - local file_path = path:gsub("?", modname:gsub("%.", "/")) - local chunk, err = loadfile(file_path) + local file = path:gsub("?", modname:gsub("%.", "/")) + local chunk = loadfile(file) if chunk then setfenv(chunk, env) - local result = chunk() - if result == nil then - result = true - end + local result = chunk() or true package.loaded[modname] = result return result end - table.insert(errors, "\tno file '" .. file_path .. "'") end - - error("module '" .. modname .. "' not found:\n" .. table.concat(errors, "\n"), 2) + + error("module '" .. modname .. "' not found", 2) end - return env end `) } -// GetModuleByPath finds the module name for a file path +func (l *ModuleLoader) scanDirectory(state *luajit.State, dir string) error { + if dir == "" { + return nil + } + + absDir, err := filepath.Abs(dir) + if err != nil { + return nil + } + + l.debugLog("Scanning directory: %s", absDir) + + return filepath.Walk(absDir, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() || !strings.HasSuffix(path, ".lua") { + return nil + } + + relPath, err := filepath.Rel(absDir, path) + if err != nil || strings.HasPrefix(relPath, "..") { + return nil + } + + // Convert to module name + modName := strings.TrimSuffix(relPath, ".lua") + modName = strings.ReplaceAll(modName, string(filepath.Separator), ".") + + l.debugLog("Found module: %s at %s", modName, path) + l.pathCache[modName] = path + + // Load and compile module + content, err := os.ReadFile(path) + if err != nil { + l.debugLog("Failed to read %s: %v", path, err) + return nil + } + + if err := state.LoadString(string(content)); err != nil { + l.debugLog("Failed to compile %s: %v", path, err) + return nil + } + + // Store in package.preload + state.GetGlobal("package") + state.GetField(-1, "preload") + state.PushString(modName) + state.PushCopy(-4) // Copy compiled function + state.SetTable(-3) + state.Pop(2) // Pop package and preload + state.Pop(1) // Pop function + + return nil + }) +} + func (l *ModuleLoader) GetModuleByPath(path string) (string, bool) { l.mu.RLock() defer l.mu.RUnlock() - // Convert to absolute path for consistent comparison absPath, err := filepath.Abs(path) if err != nil { absPath = filepath.Clean(path) } - // Try direct lookup from cache with absolute path + // Direct lookup for modName, modPath := range l.pathCache { if modPath == absPath { return modName, true } } - // Try to construct module name from lib dirs + // Construct from lib dirs for _, dir := range l.config.LibDirs { absDir, err := filepath.Abs(dir) if err != nil { continue } - // Check if the file is under this lib directory relPath, err := filepath.Rel(absDir, absPath) - if err != nil || strings.HasPrefix(relPath, "..") { + if err != nil || strings.HasPrefix(relPath, "..") || !strings.HasSuffix(relPath, ".lua") { continue } - if strings.HasSuffix(relPath, ".lua") { - modName := strings.TrimSuffix(relPath, ".lua") - modName = strings.ReplaceAll(modName, string(filepath.Separator), ".") - - l.debugLog("Found module %s for path %s", modName, path) - return modName, true - } + modName := strings.TrimSuffix(relPath, ".lua") + modName = strings.ReplaceAll(modName, string(filepath.Separator), ".") + return modName, true } - l.debugLog("No module found for path %s", path) return "", false } -// RefreshModule recompiles and updates a specific module func (l *ModuleLoader) RefreshModule(state *luajit.State, moduleName string) error { l.mu.Lock() defer l.mu.Unlock() - // Get module path path, exists := l.pathCache[moduleName] if !exists { - l.debugLog("Module not found in cache: %s", moduleName) return fmt.Errorf("module %s not found", moduleName) } - l.debugLog("Refreshing module: %s at %s", moduleName, path) + l.debugLog("Refreshing module: %s", moduleName) - // Read updated file content content, err := os.ReadFile(path) if err != nil { - return fmt.Errorf("failed to read module file: %w", err) + return fmt.Errorf("failed to read module: %w", err) } - // Recompile to bytecode - bytecode, err := state.CompileBytecode(string(content), path) - if err != nil { + // Compile new version + if err := state.LoadString(string(content)); err != nil { return fmt.Errorf("failed to compile module: %w", err) } - // Update bytecode cache - l.bytecodeCache[moduleName] = bytecode - - // Load new bytecode - if err := state.LoadBytecode(bytecode, path); err != nil { - return fmt.Errorf("failed to load bytecode: %w", err) - } - - // Update package.preload with new function (function is on stack) + // Update package.preload state.GetGlobal("package") state.GetField(-1, "preload") state.PushString(moduleName) - state.PushCopy(-4) // Copy the new compiled function - state.SetTable(-3) // preload[moduleName] = new_function - state.Pop(2) // Pop package and preload tables - state.Pop(1) // Pop the function + state.PushCopy(-4) // Copy function + state.SetTable(-3) + state.Pop(2) // Pop package and preload + state.Pop(1) // Pop function - // Clear from package.loaded so it gets reloaded - escapedName := escapeLuaString(moduleName) - if err := state.DoString(`package.loaded["` + escapedName + `"] = nil`); err != nil { - return fmt.Errorf("failed to clear loaded module: %w", err) - } + // Clear from loaded + state.DoString(`package.loaded["` + escapeLuaString(moduleName) + `"] = nil`) - l.debugLog("Successfully refreshed module: %s", moduleName) + l.debugLog("Successfully refreshed: %s", moduleName) return nil } -// RefreshModuleByPath refreshes a module by its file path func (l *ModuleLoader) RefreshModuleByPath(state *luajit.State, filePath string) error { moduleName, exists := l.GetModuleByPath(filePath) if !exists { @@ -417,14 +281,12 @@ func (l *ModuleLoader) RefreshModuleByPath(state *luajit.State, filePath string) return l.RefreshModule(state, moduleName) } -// escapeLuaString escapes special characters in a string for Lua func escapeLuaString(s string) string { - replacer := strings.NewReplacer( - "\\", "\\\\", - "\"", "\\\"", - "\n", "\\n", - "\r", "\\r", - "\t", "\\t", - ) - return replacer.Replace(s) + return strings.NewReplacer( + `\`, `\\`, + `"`, `\"`, + "\n", `\n`, + "\r", `\r`, + "\t", `\t`, + ).Replace(s) } diff --git a/runner/runner.go b/runner/runner.go index 867e17d..2f9f77d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1,4 +1,3 @@ -// runner.go - Simplified interface package runner import ( @@ -52,14 +51,19 @@ type Runner struct { paramsPool sync.Pool } -func NewRunner(poolSize int, dataDir, fsDir string) (*Runner, error) { +func NewRunner(poolSize int, dataDir, fsDir string, libDirs []string) (*Runner, error) { if poolSize <= 0 { poolSize = runtime.GOMAXPROCS(0) } + // Configure module loader with lib directories + moduleConfig := &ModuleConfig{ + LibDirs: libDirs, + } + r := &Runner{ poolSize: poolSize, - moduleLoader: NewModuleLoader(&ModuleConfig{}), + moduleLoader: NewModuleLoader(moduleConfig), ctxPool: sync.Pool{ New: func() any { return make(map[string]any, 8) }, },