diff --git a/core/runner/luarunner.go b/core/runner/luarunner.go index e7305cc..2a17902 100644 --- a/core/runner/luarunner.go +++ b/core/runner/luarunner.go @@ -3,6 +3,9 @@ package runner import ( "context" "errors" + "fmt" + "path/filepath" + "strings" "sync" "sync/atomic" @@ -139,9 +142,9 @@ func setupRequireFunction(state *luajit.State) error { function __setup_secure_require(env) -- Replace env.require with our secure version env.require = function(modname) - -- Check if already loaded in package.loaded - if package.loaded[modname] then - return package.loaded[modname] + -- Check if already loaded in this environment's package.loaded + if env.package.loaded[modname] then + return env.package.loaded[modname] end -- Try to load the module using our Go loader @@ -164,8 +167,8 @@ func setupRequireFunction(state *luajit.State) error { result = true end - -- Cache the result - package.loaded[modname] = result + -- Cache the result in this environment only + env.package.loaded[modname] = result return result end @@ -325,12 +328,87 @@ func (r *LuaRunner) Close() error { return nil } +// RequireCache returns the require cache for external access +func (r *LuaRunner) RequireCache() *RequireCache { + return r.requireCache +} + // ClearRequireCache clears the cache of loaded modules func (r *LuaRunner) ClearRequireCache() { - r.requireCache = NewRequireCache() + r.requireCache.Clear() } // AddModule adds a module to the sandbox environment func (r *LuaRunner) AddModule(name string, module any) { r.sandbox.AddModule(name, module) } + +// RefreshRequireCache refreshes the module cache if needed +func (r *LuaRunner) RefreshRequireCache() int { + count := r.requireCache.RefreshAll() + return count +} + +// ResetPackageLoaded resets the Lua package.loaded table +func (r *LuaRunner) ResetPackageLoaded() error { + return r.state.DoString(` + -- Create list of modules to unload (excluding core modules) + local to_unload = {} + for name, _ in pairs(package.loaded) do + -- Skip core modules + if name ~= "string" and + name ~= "table" and + name ~= "math" and + name ~= "os" and + name ~= "package" and + name ~= "io" and + name ~= "coroutine" and + name ~= "debug" and + name ~= "_G" then + table.insert(to_unload, name) + end + end + + -- Unload each module + for _, name in ipairs(to_unload) do + package.loaded[name] = nil + end + `) +} + +// RefreshModuleByName clears a specific module from Lua's package.loaded table +func (r *LuaRunner) RefreshModuleByName(modName string) error { + if r.state == nil { + return nil + } + + return r.state.DoString(fmt.Sprintf(` + package.loaded["%s"] = nil + `, modName)) +} + +// ProcessModuleChange handles a file change notification from a watcher +func (r *LuaRunner) ProcessModuleChange(filePath string) { + // Mark cache as needing refresh + r.requireCache.MarkNeedsRefresh() + + // Extract module name from file path for package.loaded clearing + ext := filepath.Ext(filePath) + if ext == ".lua" { + // Get relative path from lib directories + var modName string + for _, libDir := range r.requireCfg.LibDirs { + if rel, err := filepath.Rel(libDir, filePath); err == nil && !strings.HasPrefix(rel, "..") { + // Convert path to module name format + modName = strings.TrimSuffix(rel, ext) + modName = strings.ReplaceAll(modName, string(filepath.Separator), ".") + break + } + } + + if modName != "" { + // Clear from Lua's package.loaded (non-blocking) + go r.RefreshModuleByName(modName) + } + } +} diff --git a/core/runner/require.go b/core/runner/require.go index f9599d3..041873d 100644 --- a/core/runner/require.go +++ b/core/runner/require.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" luajit "git.sharkk.net/Sky/LuaJIT-to-Go" @@ -31,17 +32,21 @@ type RequireConfig struct { // RequireCache is a thread-safe cache for loaded Lua modules type RequireCache struct { - modules sync.Map // Maps full file paths to ModuleEntry - mu sync.Mutex - maxItems int // Maximum number of modules to cache + modules sync.Map // Maps full file paths to ModuleEntry + mu sync.Mutex + maxItems int // Maximum number of modules to cache + lastRefresh time.Time // When we last did a full refresh check + needsRefresh atomic.Bool // Flag for watchers to signal refresh needed } // NewRequireCache creates a new, empty require cache func NewRequireCache() *RequireCache { - return &RequireCache{ - modules: sync.Map{}, - maxItems: 100, // Default cache size - can be adjusted based on expected module load + cache := &RequireCache{ + modules: sync.Map{}, + maxItems: 100, // Default cache size + lastRefresh: time.Now(), } + return cache } // SetCacheSize adjusts the maximum cache size @@ -56,13 +61,18 @@ func (c *RequireCache) SetCacheSize(size int) { // Size returns the approximate number of items in the cache func (c *RequireCache) Size() int { size := 0 - c.modules.Range(func(_, _ interface{}) bool { + c.modules.Range(func(_, _ any) bool { size++ return true }) return size } +// MarkNeedsRefresh signals that modules have changed and need refresh +func (c *RequireCache) MarkNeedsRefresh() { + c.needsRefresh.Store(true) +} + // Get retrieves a module from the cache, updating its last used time func (c *RequireCache) Get(path string) ([]byte, bool) { value, ok := c.modules.Load(path) @@ -118,7 +128,7 @@ func (c *RequireCache) evictOldest() { first := true // Find oldest entry - c.modules.Range(func(key, value interface{}) bool { + c.modules.Range(func(key, value any) bool { // Handle different value types var lastUsed time.Time @@ -149,6 +159,82 @@ func (c *RequireCache) evictOldest() { } } +// Clear empties the entire cache +func (c *RequireCache) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + + // Create a new sync.Map to replace the existing one + c.modules = sync.Map{} +} + +// RefreshModule checks if a specific module needs to be refreshed +func (c *RequireCache) RefreshModule(path string) bool { + // Get the cached module + val, ok := c.modules.Load(path) + if !ok { + // Not in cache, nothing to refresh + return false + } + + // Get file info + fileInfo, err := os.Stat(path) + if err != nil { + // File no longer exists or can't be accessed, remove from cache + c.modules.Delete(path) + return true + } + + // Check if the cached module is up-to-date + entry, ok := val.(ModuleEntry) + if !ok { + // Invalid entry, remove it + c.modules.Delete(path) + return true + } + + // Check if the file has been modified since it was cached + if fileInfo.ModTime().After(entry.LastUsed) { + // File is newer than the cached version, remove from cache + c.modules.Delete(path) + return true + } + + return false +} + +// RefreshAll checks all cached modules and refreshes those that have changed +func (c *RequireCache) RefreshAll() int { + refreshed := 0 + + // No need to refresh if flag isn't set + if !c.needsRefresh.Load() { + return 0 + } + + // Collect paths to check + var paths []string + c.modules.Range(func(key, _ any) bool { + if path, ok := key.(string); ok { + paths = append(paths, path) + } + return true + }) + + // Check each path + for _, path := range paths { + if c.RefreshModule(path) { + refreshed++ + } + } + + // Reset the needsRefresh flag + c.needsRefresh.Store(false) + c.lastRefresh = time.Now() + + return refreshed +} + // UpdateRequirePaths updates the require paths in the config without further allocations or re-registering the loader. func UpdateRequirePaths(config *RequireConfig, scriptPath string) { if scriptPath != "" { @@ -201,9 +287,28 @@ func findAndCompileModule( } } - // Check if already in cache - using our Get method to update LRU info - if bytecode, ok := cache.Get(cleanPath); ok { - return bytecode, nil + // Check if already in cache + if value, ok := cache.modules.Load(cleanPath); ok { + entry, ok := value.(ModuleEntry) + if !ok { + // Legacy format, use it anyway + return value.([]byte), nil + } + + // Only do refresh check if marked as needed (by watcher) + if cache.needsRefresh.Load() { + fileInfo, err := os.Stat(cleanPath) + // Remove from cache if file changed or doesn't exist + if err != nil || (entry.LastUsed.Before(fileInfo.ModTime())) { + cache.modules.Delete(cleanPath) + // Continue to recompile + } else { + return entry.Bytecode, nil + } + } else { + // No refresh needed, use cached bytecode + return entry.Bytecode, nil + } } // Check if file exists @@ -224,8 +329,11 @@ func findAndCompileModule( return nil, err } - // Store in cache - using our Store method with LRU eviction - cache.Store(cleanPath, bytecode) + // Store in cache with current time + cache.modules.Store(cleanPath, ModuleEntry{ + Bytecode: bytecode, + LastUsed: time.Now(), + }) return bytecode, nil } diff --git a/core/runner/sandbox.go b/core/runner/sandbox.go index 931776e..43c2505 100644 --- a/core/runner/sandbox.go +++ b/core/runner/sandbox.go @@ -137,7 +137,7 @@ func (s *Sandbox) Setup(state *luajit.State) error { -- Function to run code in sandbox function __run_sandboxed(bytecode, ctx) - -- Create environment for this request + -- Create fresh environment for this request local env = __create_sandbox_env(ctx) -- Set environment and execute @@ -176,7 +176,7 @@ func (s *Sandbox) registerModules(state *luajit.State) error { // Execute runs bytecode in the sandbox func (s *Sandbox) Execute(state *luajit.State, bytecode []byte, ctx map[string]any) (any, error) { - // Update modules if needed + // Update custom modules if needed if !s.initialized { if err := s.registerModules(state); err != nil { return nil, err diff --git a/core/watchers/modulewatcher.go b/core/watchers/modulewatcher.go new file mode 100644 index 0000000..9ca17b9 --- /dev/null +++ b/core/watchers/modulewatcher.go @@ -0,0 +1,48 @@ +package watchers + +import ( + "git.sharkk.net/Sky/Moonshark/core/logger" + "git.sharkk.net/Sky/Moonshark/core/runner" +) + +// WatchLuaModules sets up an optimized watcher for Lua module directories +func WatchLuaModules(luaRunner *runner.LuaRunner, libDirs []string, log *logger.Logger) ([]*Watcher, error) { + watchers := make([]*Watcher, 0, len(libDirs)) + + for _, dir := range libDirs { + // Create a directory-specific callback that only signals changes + // without doing heavy processing in the callback itself + dirCopy := dir // Capture for closure + + callback := func() error { + log.Debug("Detected changes in Lua module directory: %s", dirCopy) + // Only mark that refresh is needed instead of doing refresh now + luaRunner.RequireCache().MarkNeedsRefresh() + return nil + } + + config := WatcherConfig{ + Dir: dir, + Callback: callback, + Log: log, + Recursive: true, + Adaptive: true, + // Use a longer debounce time to avoid too frequent updates + DebounceTime: defaultDebounceTime * 4, + } + + watcher, err := WatchDirectory(config) + if err != nil { + // Close any watchers we've already created + for _, w := range watchers { + w.Close() + } + return nil, err + } + + watchers = append(watchers, watcher) + log.Info("Started watching Lua modules directory: %s", dir) + } + + return watchers, nil +} diff --git a/core/watchers/watcher.go b/core/watchers/watcher.go index 2890745..5033f8b 100644 --- a/core/watchers/watcher.go +++ b/core/watchers/watcher.go @@ -267,6 +267,7 @@ func (w *Watcher) logWarning(format string, args ...any) { func (w *Watcher) logError(format string, args ...any) { w.log.Error("[Watcher] [%s] %s", w.dir, fmt.Sprintf(format, args...)) } + func (w *Watcher) checkForChanges() (bool, error) { // Get current state currentFiles := make(map[string]FileInfo) @@ -307,36 +308,43 @@ func (w *Watcher) checkForChanges() (bool, error) { return true, nil } - // Check for modified, added, or removed files + // Check for modified files + var changed bool for path, currentInfo := range currentFiles { prevInfo, exists := previousFiles[path] if !exists { // New file w.logDebug("New file detected: %s", path) - w.updateFiles(currentFiles) - return true, nil + changed = true + break } if currentInfo.ModTime != prevInfo.ModTime || currentInfo.Size != prevInfo.Size { // File modified w.logDebug("File modified: %s", path) - w.updateFiles(currentFiles) - return true, nil + changed = true + break } } // Check for deleted files - for path := range previousFiles { - if _, exists := currentFiles[path]; !exists { - // File deleted - w.logDebug("File deleted: %s", path) - w.updateFiles(currentFiles) - return true, nil + if !changed { + for path := range previousFiles { + if _, exists := currentFiles[path]; !exists { + // File deleted + w.logDebug("File deleted: %s", path) + changed = true + break + } } } - // No changes detected - return false, nil + // Update internal files state if there were changes + if changed { + w.updateFiles(currentFiles) + } + + return changed, nil } // updateFiles updates the internal file list diff --git a/moonshark.go b/moonshark.go index bf00de4..9fc178a 100644 --- a/moonshark.go +++ b/moonshark.go @@ -45,6 +45,48 @@ func initRouters(routesDir, staticDir string, log *logger.Logger) (*routers.LuaR return luaRouter, staticRouter, nil } +// setupWatchers initializes and starts all file watchers +func setupWatchers(luaRouter *routers.LuaRouter, staticRouter *routers.StaticRouter, + luaRunner *runner.LuaRunner, routesDir string, staticDir string, + libDirs []string, log *logger.Logger) ([]func() error, error) { + + var cleanupFuncs []func() error + + // Set up watcher for Lua routes + luaRouterWatcher, err := watchers.WatchLuaRouter(luaRouter, routesDir, log) + if err != nil { + log.Warning("Failed to watch routes directory: %v", err) + } else { + cleanupFuncs = append(cleanupFuncs, luaRouterWatcher.Close) + log.Info("File watcher active for Lua routes") + } + + // Set up watcher for static files + staticWatcher, err := watchers.WatchStaticRouter(staticRouter, staticDir, log) + if err != nil { + log.Warning("Failed to watch static directory: %v", err) + } else { + cleanupFuncs = append(cleanupFuncs, staticWatcher.Close) + log.Info("File watcher active for static files") + } + + // Set up watchers for Lua modules libraries + if len(libDirs) > 0 { + moduleWatchers, err := watchers.WatchLuaModules(luaRunner, libDirs, log) + if err != nil { + log.Warning("Failed to watch Lua module directories: %v", err) + } else { + for _, watcher := range moduleWatchers { + w := watcher // Capture variable for closure + cleanupFuncs = append(cleanupFuncs, w.Close) + } + log.Info("File watchers active for %d Lua module directories", len(moduleWatchers)) + } + } + + return cleanupFuncs, nil +} + func main() { // Initialize logger log := logger.New(logger.LevelDebug, true) @@ -59,6 +101,7 @@ func main() { cfg = config.New() } + // Set log level from config switch cfg.GetString("log_level", "info") { case "debug": log.SetLevel(logger.LevelDebug) @@ -74,52 +117,66 @@ func main() { log.SetLevel(logger.LevelInfo) } - // Get port from config or use default + // Get configuration values port := cfg.GetInt("port", 3117) - - // Initialize routers routesDir := cfg.GetString("routes_dir", "./routes") staticDir := cfg.GetString("static_dir", "./static") + bufferSize := cfg.GetInt("buffer_size", 20) + + // Get library directories + var libDirs []string + libDirsConfig := cfg.GetStringArray("lib_dirs") + if libDirsConfig != nil && len(libDirsConfig) > 0 { + libDirs = libDirsConfig + } else { + // Default lib directory + libDirs = []string{"./libs"} + } + + // Ensure lib directories exist + for _, dir := range libDirs { + if err := utils.EnsureDir(dir); err != nil { + log.Warning("Lib directory doesn't exist, and could not create it: %v", err) + } + } + + // Initialize routers luaRouter, staticRouter, err := initRouters(routesDir, staticDir, log) if err != nil { log.Fatal("Router initialization failed: %v", err) } - if cfg.GetBool("watchers", true) { - // Set up file watchers for automatic reloading - luaWatcher, err := watchers.WatchLuaRouter(luaRouter, routesDir, log) - if err != nil { - log.Warning("Failed to watch routes directory: %v", err) - } else { - defer luaWatcher.Close() - log.Info("File watcher active for Lua routes") - } - - staticWatcher, err := watchers.WatchStaticRouter(staticRouter, staticDir, log) - if err != nil { - log.Warning("Failed to watch static directory: %v", err) - } else { - defer staticWatcher.Close() - log.Info("File watcher active for static files") - } - } - - // Get buffer size from config or use default - bufferSize := cfg.GetInt("buffer_size", 20) - // Initialize Lua runner - runner, err := runner.NewRunner( + luaRunner, err := runner.NewRunner( runner.WithBufferSize(bufferSize), - runner.WithLibDirs("./libs"), + runner.WithLibDirs(libDirs...), ) if err != nil { log.Fatal("Failed to initialize Lua runner: %v", err) } log.Server("Lua runner initialized with buffer size %d", bufferSize) - defer runner.Close() + defer luaRunner.Close() + + // Set up file watchers if enabled + var cleanupFuncs []func() error + if cfg.GetBool("watchers", true) { + cleanupFuncs, err = setupWatchers(luaRouter, staticRouter, luaRunner, routesDir, staticDir, libDirs, log) + if err != nil { + log.Warning("Error setting up watchers: %v", err) + } + } + + // Register cleanup functions + defer func() { + for _, cleanup := range cleanupFuncs { + if err := cleanup(); err != nil { + log.Warning("Cleanup error: %v", err) + } + } + }() // Create HTTP server - server := http.New(luaRouter, staticRouter, runner, log) + server := http.New(luaRouter, staticRouter, luaRunner, log) // Handle graceful shutdown stop := make(chan os.Signal, 1)