require watcher

core/runner/luarunner.go

@@ -3,6 +3,9 @@ package runner
 import (
 	"context"
 	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
 	"sync"
 	"sync/atomic"
 
@@ -139,9 +142,9 @@ func setupRequireFunction(state *luajit.State) error {
 		function __setup_secure_require(env)
 			-- Replace env.require with our secure version
 			env.require = function(modname)
-				-- Check if already loaded in package.loaded
-				if package.loaded[modname] then
-					return package.loaded[modname]
+				-- Check if already loaded in this environment's package.loaded
+				if env.package.loaded[modname] then
+					return env.package.loaded[modname]
 				end
 
 				-- Try to load the module using our Go loader
@@ -164,8 +167,8 @@ func setupRequireFunction(state *luajit.State) error {
 					result = true
 				end
 
-				-- Cache the result
-				package.loaded[modname] = result
+				-- Cache the result in this environment only
+				env.package.loaded[modname] = result
 
 				return result
 			end
@@ -325,12 +328,87 @@ func (r *LuaRunner) Close() error {
 	return nil
 }
 
+// RequireCache returns the require cache for external access
+func (r *LuaRunner) RequireCache() *RequireCache {
+	return r.requireCache
+}
+
 // ClearRequireCache clears the cache of loaded modules
 func (r *LuaRunner) ClearRequireCache() {
-	r.requireCache = NewRequireCache()
+	r.requireCache.Clear()
 }
 
 // AddModule adds a module to the sandbox environment
 func (r *LuaRunner) AddModule(name string, module any) {
 	r.sandbox.AddModule(name, module)
 }
+
+// RefreshRequireCache refreshes the module cache if needed
+func (r *LuaRunner) RefreshRequireCache() int {
+	count := r.requireCache.RefreshAll()
+	return count
+}
+
+// ResetPackageLoaded resets the Lua package.loaded table
+func (r *LuaRunner) ResetPackageLoaded() error {
+	return r.state.DoString(`
+		-- Create list of modules to unload (excluding core modules)
+		local to_unload = {}
+		for name, _ in pairs(package.loaded) do
+			-- Skip core modules
+			if name ~= "string" and
+			   name ~= "table" and
+			   name ~= "math" and
+			   name ~= "os" and
+			   name ~= "package" and
+			   name ~= "io" and
+			   name ~= "coroutine" and
+			   name ~= "debug" and
+			   name ~= "_G" then
+				table.insert(to_unload, name)
+			end
+		end
+
+		-- Unload each module
+		for _, name in ipairs(to_unload) do
+			package.loaded[name] = nil
+		end
+	`)
+}
+
+// RefreshModuleByName clears a specific module from Lua's package.loaded table
+func (r *LuaRunner) RefreshModuleByName(modName string) error {
+	if r.state == nil {
+		return nil
+	}
+
+	return r.state.DoString(fmt.Sprintf(`
+		package.loaded["%s"] = nil
+	`, modName))
+}
+
+// ProcessModuleChange handles a file change notification from a watcher
+func (r *LuaRunner) ProcessModuleChange(filePath string) {
+	// Mark cache as needing refresh
+	r.requireCache.MarkNeedsRefresh()
+
+	// Extract module name from file path for package.loaded clearing
+	ext := filepath.Ext(filePath)
+	if ext == ".lua" {
+		// Get relative path from lib directories
+		var modName string
+		for _, libDir := range r.requireCfg.LibDirs {
+			if rel, err := filepath.Rel(libDir, filePath); err == nil && !strings.HasPrefix(rel, "..") {
+				// Convert path to module name format
+				modName = strings.TrimSuffix(rel, ext)
+				modName = strings.ReplaceAll(modName, string(filepath.Separator), ".")
+				break
+			}
+		}
+
+		if modName != "" {
+			// Clear from Lua's package.loaded (non-blocking)
+			go r.RefreshModuleByName(modName)
+		}
+	}
+}

core/runner/require.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	luajit "git.sharkk.net/Sky/LuaJIT-to-Go"
@@ -34,14 +35,18 @@ type RequireCache struct {
 	modules      sync.Map // Maps full file paths to ModuleEntry
 	mu           sync.Mutex
 	maxItems     int         // Maximum number of modules to cache
+	lastRefresh  time.Time   // When we last did a full refresh check
+	needsRefresh atomic.Bool // Flag for watchers to signal refresh needed
 }
 
 // NewRequireCache creates a new, empty require cache
 func NewRequireCache() *RequireCache {
-	return &RequireCache{
+	cache := &RequireCache{
 		modules:     sync.Map{},
-		maxItems: 100, // Default cache size - can be adjusted based on expected module load
+		maxItems:    100, // Default cache size
+		lastRefresh: time.Now(),
 	}
+	return cache
 }
 
 // SetCacheSize adjusts the maximum cache size
@@ -56,13 +61,18 @@ func (c *RequireCache) SetCacheSize(size int) {
 // Size returns the approximate number of items in the cache
 func (c *RequireCache) Size() int {
 	size := 0
-	c.modules.Range(func(_, _ interface{}) bool {
+	c.modules.Range(func(_, _ any) bool {
 		size++
 		return true
 	})
 	return size
 }
 
+// MarkNeedsRefresh signals that modules have changed and need refresh
+func (c *RequireCache) MarkNeedsRefresh() {
+	c.needsRefresh.Store(true)
+}
+
 // Get retrieves a module from the cache, updating its last used time
 func (c *RequireCache) Get(path string) ([]byte, bool) {
 	value, ok := c.modules.Load(path)
@@ -118,7 +128,7 @@ func (c *RequireCache) evictOldest() {
 	first := true
 
 	// Find oldest entry
-	c.modules.Range(func(key, value interface{}) bool {
+	c.modules.Range(func(key, value any) bool {
 		// Handle different value types
 		var lastUsed time.Time
 
@@ -149,6 +159,82 @@ func (c *RequireCache) evictOldest() {
 	}
 }
 
+// Clear empties the entire cache
+func (c *RequireCache) Clear() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Create a new sync.Map to replace the existing one
+	c.modules = sync.Map{}
+}
+
+// RefreshModule checks if a specific module needs to be refreshed
+func (c *RequireCache) RefreshModule(path string) bool {
+	// Get the cached module
+	val, ok := c.modules.Load(path)
+	if !ok {
+		// Not in cache, nothing to refresh
+		return false
+	}
+
+	// Get file info
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		// File no longer exists or can't be accessed, remove from cache
+		c.modules.Delete(path)
+		return true
+	}
+
+	// Check if the cached module is up-to-date
+	entry, ok := val.(ModuleEntry)
+	if !ok {
+		// Invalid entry, remove it
+		c.modules.Delete(path)
+		return true
+	}
+
+	// Check if the file has been modified since it was cached
+	if fileInfo.ModTime().After(entry.LastUsed) {
+		// File is newer than the cached version, remove from cache
+		c.modules.Delete(path)
+		return true
+	}
+
+	return false
+}
+
+// RefreshAll checks all cached modules and refreshes those that have changed
+func (c *RequireCache) RefreshAll() int {
+	refreshed := 0
+
+	// No need to refresh if flag isn't set
+	if !c.needsRefresh.Load() {
+		return 0
+	}
+
+	// Collect paths to check
+	var paths []string
+	c.modules.Range(func(key, _ any) bool {
+		if path, ok := key.(string); ok {
+			paths = append(paths, path)
+		}
+		return true
+	})
+
+	// Check each path
+	for _, path := range paths {
+		if c.RefreshModule(path) {
+			refreshed++
+		}
+	}
+
+	// Reset the needsRefresh flag
+	c.needsRefresh.Store(false)
+	c.lastRefresh = time.Now()
+
+	return refreshed
+}
+
 // UpdateRequirePaths updates the require paths in the config without further allocations or re-registering the loader.
 func UpdateRequirePaths(config *RequireConfig, scriptPath string) {
 	if scriptPath != "" {
@@ -201,9 +287,28 @@ func findAndCompileModule(
 			}
 		}
 
-		// Check if already in cache - using our Get method to update LRU info
-		if bytecode, ok := cache.Get(cleanPath); ok {
-			return bytecode, nil
+		// Check if already in cache
+		if value, ok := cache.modules.Load(cleanPath); ok {
+			entry, ok := value.(ModuleEntry)
+			if !ok {
+				// Legacy format, use it anyway
+				return value.([]byte), nil
+			}
+
+			// Only do refresh check if marked as needed (by watcher)
+			if cache.needsRefresh.Load() {
+				fileInfo, err := os.Stat(cleanPath)
+				// Remove from cache if file changed or doesn't exist
+				if err != nil || (entry.LastUsed.Before(fileInfo.ModTime())) {
+					cache.modules.Delete(cleanPath)
+					// Continue to recompile
+				} else {
+					return entry.Bytecode, nil
+				}
+			} else {
+				// No refresh needed, use cached bytecode
+				return entry.Bytecode, nil
+			}
 		}
 
 		// Check if file exists
@@ -224,8 +329,11 @@ func findAndCompileModule(
 			return nil, err
 		}
 
-		// Store in cache - using our Store method with LRU eviction
-		cache.Store(cleanPath, bytecode)
+		// Store in cache with current time
+		cache.modules.Store(cleanPath, ModuleEntry{
+			Bytecode: bytecode,
+			LastUsed: time.Now(),
+		})
 
 		return bytecode, nil
 	}

core/runner/sandbox.go

@@ -137,7 +137,7 @@ func (s *Sandbox) Setup(state *luajit.State) error {
 
 		-- Function to run code in sandbox
 		function __run_sandboxed(bytecode, ctx)
-			-- Create environment for this request
+			-- Create fresh environment for this request
 			local env = __create_sandbox_env(ctx)
 
 			-- Set environment and execute
@@ -176,7 +176,7 @@ func (s *Sandbox) registerModules(state *luajit.State) error {
 
 // Execute runs bytecode in the sandbox
 func (s *Sandbox) Execute(state *luajit.State, bytecode []byte, ctx map[string]any) (any, error) {
-	// Update modules if needed
+	// Update custom modules if needed
 	if !s.initialized {
 		if err := s.registerModules(state); err != nil {
 			return nil, err

core/watchers/modulewatcher.go

@@ -0,0 +1,48 @@
+package watchers
+
+import (
+	"git.sharkk.net/Sky/Moonshark/core/logger"
+	"git.sharkk.net/Sky/Moonshark/core/runner"
+)
+
+// WatchLuaModules sets up an optimized watcher for Lua module directories
+func WatchLuaModules(luaRunner *runner.LuaRunner, libDirs []string, log *logger.Logger) ([]*Watcher, error) {
+	watchers := make([]*Watcher, 0, len(libDirs))
+
+	for _, dir := range libDirs {
+		// Create a directory-specific callback that only signals changes
+		// without doing heavy processing in the callback itself
+		dirCopy := dir // Capture for closure
+
+		callback := func() error {
+			log.Debug("Detected changes in Lua module directory: %s", dirCopy)
+			// Only mark that refresh is needed instead of doing refresh now
+			luaRunner.RequireCache().MarkNeedsRefresh()
+			return nil
+		}
+
+		config := WatcherConfig{
+			Dir:       dir,
+			Callback:  callback,
+			Log:       log,
+			Recursive: true,
+			Adaptive:  true,
+			// Use a longer debounce time to avoid too frequent updates
+			DebounceTime: defaultDebounceTime * 4,
+		}
+
+		watcher, err := WatchDirectory(config)
+		if err != nil {
+			// Close any watchers we've already created
+			for _, w := range watchers {
+				w.Close()
+			}
+			return nil, err
+		}
+
+		watchers = append(watchers, watcher)
+		log.Info("Started watching Lua modules directory: %s", dir)
+	}
+
+	return watchers, nil
+}

core/watchers/watcher.go

@@ -267,6 +267,7 @@ func (w *Watcher) logWarning(format string, args ...any) {
 func (w *Watcher) logError(format string, args ...any) {
 	w.log.Error("[Watcher] [%s] %s", w.dir, fmt.Sprintf(format, args...))
 }
+
 func (w *Watcher) checkForChanges() (bool, error) {
 	// Get current state
 	currentFiles := make(map[string]FileInfo)
@@ -307,36 +308,43 @@ func (w *Watcher) checkForChanges() (bool, error) {
 		return true, nil
 	}
 
-	// Check for modified, added, or removed files
+	// Check for modified files
+	var changed bool
 	for path, currentInfo := range currentFiles {
 		prevInfo, exists := previousFiles[path]
 		if !exists {
 			// New file
 			w.logDebug("New file detected: %s", path)
-			w.updateFiles(currentFiles)
-			return true, nil
+			changed = true
+			break
 		}
 
 		if currentInfo.ModTime != prevInfo.ModTime || currentInfo.Size != prevInfo.Size {
 			// File modified
 			w.logDebug("File modified: %s", path)
-			w.updateFiles(currentFiles)
-			return true, nil
+			changed = true
+			break
 		}
 	}
 
 	// Check for deleted files
+	if !changed {
 		for path := range previousFiles {
 			if _, exists := currentFiles[path]; !exists {
 				// File deleted
 				w.logDebug("File deleted: %s", path)
-			w.updateFiles(currentFiles)
-			return true, nil
+				changed = true
+				break
+			}
 		}
 	}
 
-	// No changes detected
-	return false, nil
+	// Update internal files state if there were changes
+	if changed {
+		w.updateFiles(currentFiles)
+	}
+
+	return changed, nil
 }
 
 // updateFiles updates the internal file list

moonshark.go

@@ -45,6 +45,48 @@ func initRouters(routesDir, staticDir string, log *logger.Logger) (*routers.LuaR
 	return luaRouter, staticRouter, nil
 }
 
+// setupWatchers initializes and starts all file watchers
+func setupWatchers(luaRouter *routers.LuaRouter, staticRouter *routers.StaticRouter,
+	luaRunner *runner.LuaRunner, routesDir string, staticDir string,
+	libDirs []string, log *logger.Logger) ([]func() error, error) {
+
+	var cleanupFuncs []func() error
+
+	// Set up watcher for Lua routes
+	luaRouterWatcher, err := watchers.WatchLuaRouter(luaRouter, routesDir, log)
+	if err != nil {
+		log.Warning("Failed to watch routes directory: %v", err)
+	} else {
+		cleanupFuncs = append(cleanupFuncs, luaRouterWatcher.Close)
+		log.Info("File watcher active for Lua routes")
+	}
+
+	// Set up watcher for static files
+	staticWatcher, err := watchers.WatchStaticRouter(staticRouter, staticDir, log)
+	if err != nil {
+		log.Warning("Failed to watch static directory: %v", err)
+	} else {
+		cleanupFuncs = append(cleanupFuncs, staticWatcher.Close)
+		log.Info("File watcher active for static files")
+	}
+
+	// Set up watchers for Lua modules libraries
+	if len(libDirs) > 0 {
+		moduleWatchers, err := watchers.WatchLuaModules(luaRunner, libDirs, log)
+		if err != nil {
+			log.Warning("Failed to watch Lua module directories: %v", err)
+		} else {
+			for _, watcher := range moduleWatchers {
+				w := watcher // Capture variable for closure
+				cleanupFuncs = append(cleanupFuncs, w.Close)
+			}
+			log.Info("File watchers active for %d Lua module directories", len(moduleWatchers))
+		}
+	}
+
+	return cleanupFuncs, nil
+}
+
 func main() {
 	// Initialize logger
 	log := logger.New(logger.LevelDebug, true)
@@ -59,6 +101,7 @@ func main() {
 		cfg = config.New()
 	}
 
+	// Set log level from config
 	switch cfg.GetString("log_level", "info") {
 	case "debug":
 		log.SetLevel(logger.LevelDebug)
@@ -74,52 +117,66 @@ func main() {
 		log.SetLevel(logger.LevelInfo)
 	}
 
-	// Get port from config or use default
+	// Get configuration values
 	port := cfg.GetInt("port", 3117)
-
-	// Initialize routers
 	routesDir := cfg.GetString("routes_dir", "./routes")
 	staticDir := cfg.GetString("static_dir", "./static")
+	bufferSize := cfg.GetInt("buffer_size", 20)
+
+	// Get library directories
+	var libDirs []string
+	libDirsConfig := cfg.GetStringArray("lib_dirs")
+	if libDirsConfig != nil && len(libDirsConfig) > 0 {
+		libDirs = libDirsConfig
+	} else {
+		// Default lib directory
+		libDirs = []string{"./libs"}
+	}
+
+	// Ensure lib directories exist
+	for _, dir := range libDirs {
+		if err := utils.EnsureDir(dir); err != nil {
+			log.Warning("Lib directory doesn't exist, and could not create it: %v", err)
+		}
+	}
+
+	// Initialize routers
 	luaRouter, staticRouter, err := initRouters(routesDir, staticDir, log)
 	if err != nil {
 		log.Fatal("Router initialization failed: %v", err)
 	}
 
-	if cfg.GetBool("watchers", true) {
-		// Set up file watchers for automatic reloading
-		luaWatcher, err := watchers.WatchLuaRouter(luaRouter, routesDir, log)
-		if err != nil {
-			log.Warning("Failed to watch routes directory: %v", err)
-		} else {
-			defer luaWatcher.Close()
-			log.Info("File watcher active for Lua routes")
-		}
-
-		staticWatcher, err := watchers.WatchStaticRouter(staticRouter, staticDir, log)
-		if err != nil {
-			log.Warning("Failed to watch static directory: %v", err)
-		} else {
-			defer staticWatcher.Close()
-			log.Info("File watcher active for static files")
-		}
-	}
-
-	// Get buffer size from config or use default
-	bufferSize := cfg.GetInt("buffer_size", 20)
-
 	// Initialize Lua runner
-	runner, err := runner.NewRunner(
+	luaRunner, err := runner.NewRunner(
 		runner.WithBufferSize(bufferSize),
-		runner.WithLibDirs("./libs"),
+		runner.WithLibDirs(libDirs...),
 	)
 	if err != nil {
 		log.Fatal("Failed to initialize Lua runner: %v", err)
 	}
 	log.Server("Lua runner initialized with buffer size %d", bufferSize)
-	defer runner.Close()
+	defer luaRunner.Close()
+
+	// Set up file watchers if enabled
+	var cleanupFuncs []func() error
+	if cfg.GetBool("watchers", true) {
+		cleanupFuncs, err = setupWatchers(luaRouter, staticRouter, luaRunner, routesDir, staticDir, libDirs, log)
+		if err != nil {
+			log.Warning("Error setting up watchers: %v", err)
+		}
+	}
+
+	// Register cleanup functions
+	defer func() {
+		for _, cleanup := range cleanupFuncs {
+			if err := cleanup(); err != nil {
+				log.Warning("Cleanup error: %v", err)
+			}
+		}
+	}()
 
 	// Create HTTP server
-	server := http.New(luaRouter, staticRouter, runner, log)
+	server := http.New(luaRouter, staticRouter, luaRunner, log)
 
 	// Handle graceful shutdown
 	stop := make(chan os.Signal, 1)