From d1ec8ade9c2dd4cfa81840a2d13514fe7a7dc559 Mon Sep 17 00:00:00 2001 From: Sky Johnson Date: Sat, 24 May 2025 09:45:38 -0500 Subject: [PATCH] add LRU cache to fs retrieval --- go.mod | 3 ++- go.sum | 6 ++--- runner/fs.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0c99a60..8a873e1 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,13 @@ module Moonshark go 1.24.1 require ( + git.sharkk.net/Go/LRU v1.0.0 git.sharkk.net/Sky/LuaJIT-to-Go v0.4.1 github.com/VictoriaMetrics/fastcache v1.12.4 github.com/alexedwards/argon2id v1.0.0 github.com/deneonet/benc v1.1.8 github.com/goccy/go-json v0.10.5 + github.com/golang/snappy v1.0.0 github.com/matoous/go-nanoid/v2 v2.1.0 github.com/valyala/bytebufferpool v1.0.0 github.com/valyala/fasthttp v1.62.0 @@ -18,7 +20,6 @@ require ( github.com/andybalholm/brotli v1.1.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/golang/snappy v1.0.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index bb6fb14..2cfd634 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +git.sharkk.net/Go/LRU v1.0.0 h1:/KqdRVhHldi23aVfQZ4ss6vhCWZqA3vFiQyf1MJPpQc= +git.sharkk.net/Go/LRU v1.0.0/go.mod h1:8tdTyl85mss9a+KKwo+Wj9gKHOizhfLfpJhz1ltYz50= git.sharkk.net/Sky/LuaJIT-to-Go v0.4.1 h1:CAYt+C6Vgo4JxK876j0ApQ2GDFFvy9FKO0OoZBVD18k= git.sharkk.net/Sky/LuaJIT-to-Go v0.4.1/go.mod h1:HQz+D7AFxOfNbTIogjxP+shEBtz1KKrLlLucU+w07c8= github.com/VictoriaMetrics/fastcache v1.12.4 h1:2xvmwZBW+9QtHsXggfzAZRs1FZWCsBs8QDg22bMidf0= @@ -107,8 +109,6 @@ modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8= modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/libc v1.65.7 h1:Ia9Z4yzZtWNtUIuiPuQ7Qf7kxYrxP1/jeHZzG8bFu00= -modernc.org/libc v1.65.7/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU= modernc.org/libc v1.65.8 h1:7PXRJai0TXZ8uNA3srsmYzmTyrLoHImV5QxHeni108Q= modernc.org/libc v1.65.8/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= @@ -125,7 +125,5 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -zombiezen.com/go/sqlite v1.4.0 h1:N1s3RIljwtp4541Y8rM880qgGIgq3fTD2yks1xftnKU= -zombiezen.com/go/sqlite v1.4.0/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik= zombiezen.com/go/sqlite v1.4.2 h1:KZXLrBuJ7tKNEm+VJcApLMeQbhmAUOKA5VWS93DfFRo= zombiezen.com/go/sqlite v1.4.2/go.mod h1:5Kd4taTAD4MkBzT25mQ9uaAlLjyR0rFhsR6iINO70jc= diff --git a/runner/fs.go b/runner/fs.go index 906f425..220fa0d 100644 --- a/runner/fs.go +++ b/runner/fs.go @@ -6,15 +6,29 @@ import ( "os" "path/filepath" "strings" + "time" "Moonshark/utils/logger" + lru "git.sharkk.net/Go/LRU" luajit "git.sharkk.net/Sky/LuaJIT-to-Go" + "github.com/golang/snappy" ) // Global filesystem path (set during initialization) var fsBasePath string +// Global file cache with compressed data +var fileCache *lru.LRUCache + +// Cache entry info for statistics/debugging +type cacheStats struct { + hits int64 + misses int64 +} + +var stats cacheStats + // InitFS initializes the filesystem with the given base path func InitFS(basePath string) error { if basePath == "" { @@ -33,13 +47,20 @@ func InitFS(basePath string) error { } fsBasePath = absPath + + // Initialize file cache with 2000 entries (reasonable for most use cases) + fileCache = lru.NewLRUCache(2000) + logger.Server("Virtual filesystem initialized at: %s", fsBasePath) return nil } // CleanupFS performs any necessary cleanup func CleanupFS() { - // Nothing to clean up currently + if fileCache != nil { + fileCache.Clear() + logger.Server("File cache cleared - Stats: hits=%d, misses=%d", stats.hits, stats.misses) + } } // ResolvePath resolves a given path relative to the filesystem base @@ -79,6 +100,11 @@ func ResolvePath(path string) (string, error) { return fullPath, nil } +// getCacheKey creates a cache key from path and modification time +func getCacheKey(fullPath string, modTime time.Time) string { + return fmt.Sprintf("%s:%d", fullPath, modTime.Unix()) +} + // fsReadFile reads a file and returns its contents func fsReadFile(state *luajit.State) int { if !state.IsString(1) { @@ -93,12 +119,46 @@ func fsReadFile(state *luajit.State) int { return -1 } + // Get file info for cache key and validation + info, err := os.Stat(fullPath) + if err != nil { + state.PushString("fs.read_file: " + err.Error()) + return -1 + } + + // Create cache key with path and modification time + cacheKey := getCacheKey(fullPath, info.ModTime()) + + // Try to get from cache first + if fileCache != nil { + if cachedData, exists := fileCache.Get(cacheKey); exists { + if compressedData, ok := cachedData.([]byte); ok { + // Decompress cached data + data, err := snappy.Decode(nil, compressedData) + if err == nil { + stats.hits++ + state.PushString(string(data)) + return 1 + } + // Cache corruption - continue to disk read + } + } + } + + // Cache miss or error - read from disk + stats.misses++ data, err := os.ReadFile(fullPath) if err != nil { state.PushString("fs.read_file: " + err.Error()) return -1 } + // Compress and cache the data + if fileCache != nil { + compressedData := snappy.Encode(nil, data) + fileCache.Put(cacheKey, compressedData) + } + state.PushString(string(data)) return 1 } @@ -136,6 +196,12 @@ func fsWriteFile(state *luajit.State) int { return -1 } + // Invalidate cache entries for this file path + if fileCache != nil { + // We can't easily iterate through cache keys, so we'll let the cache + // naturally expire old entries when the file is read again + } + state.PushBoolean(true) return 1 }