Sky/Moonshark
1
0
Fork 0
Code Issues Releases Activity

add LRU cache to fs retrieval

Browse Source
This commit is contained in:
Sky Johnson 2025-05-24 09:45:38 -05:00
parent a055638e4b
commit d1ec8ade9c
3 changed files with 71 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -3,11 +3,13 @@ module Moonshark
go 1.24.1
require (
git.sharkk.net/Go/LRU v1.0.0
git.sharkk.net/Sky/LuaJIT-to-Go v0.4.1
github.com/VictoriaMetrics/fastcache v1.12.4
github.com/alexedwards/argon2id v1.0.0
github.com/deneonet/benc v1.1.8
github.com/goccy/go-json v0.10.5
github.com/golang/snappy v1.0.0
github.com/matoous/go-nanoid/v2 v2.1.0
github.com/valyala/bytebufferpool v1.0.0
github.com/valyala/fasthttp v1.62.0
@ -18,7 +20,6 @@ require (
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect

6
go.sum
View File

@ -1,3 +1,5 @@
git.sharkk.net/Go/LRU v1.0.0 h1:/KqdRVhHldi23aVfQZ4ss6vhCWZqA3vFiQyf1MJPpQc=
git.sharkk.net/Go/LRU v1.0.0/go.mod h1:8tdTyl85mss9a+KKwo+Wj9gKHOizhfLfpJhz1ltYz50=
git.sharkk.net/Sky/LuaJIT-to-Go v0.4.1 h1:CAYt+C6Vgo4JxK876j0ApQ2GDFFvy9FKO0OoZBVD18k=
git.sharkk.net/Sky/LuaJIT-to-Go v0.4.1/go.mod h1:HQz+D7AFxOfNbTIogjxP+shEBtz1KKrLlLucU+w07c8=
github.com/VictoriaMetrics/fastcache v1.12.4 h1:2xvmwZBW+9QtHsXggfzAZRs1FZWCsBs8QDg22bMidf0=
@ -107,8 +109,6 @@ modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8=
modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
modernc.org/libc v1.65.7 h1:Ia9Z4yzZtWNtUIuiPuQ7Qf7kxYrxP1/jeHZzG8bFu00=
modernc.org/libc v1.65.7/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU=
modernc.org/libc v1.65.8 h1:7PXRJai0TXZ8uNA3srsmYzmTyrLoHImV5QxHeni108Q=
modernc.org/libc v1.65.8/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
@ -125,7 +125,5 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
zombiezen.com/go/sqlite v1.4.0 h1:N1s3RIljwtp4541Y8rM880qgGIgq3fTD2yks1xftnKU=
zombiezen.com/go/sqlite v1.4.0/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik=
zombiezen.com/go/sqlite v1.4.2 h1:KZXLrBuJ7tKNEm+VJcApLMeQbhmAUOKA5VWS93DfFRo=
zombiezen.com/go/sqlite v1.4.2/go.mod h1:5Kd4taTAD4MkBzT25mQ9uaAlLjyR0rFhsR6iINO70jc=

68
runner/fs.go
View File

@ -6,15 +6,29 @@ import (
"os"
"path/filepath"
"strings"
"time"
"Moonshark/utils/logger"
lru "git.sharkk.net/Go/LRU"
luajit "git.sharkk.net/Sky/LuaJIT-to-Go"
"github.com/golang/snappy"
)
// Global filesystem path (set during initialization)
var fsBasePath string
// Global file cache with compressed data
var fileCache *lru.LRUCache
// Cache entry info for statistics/debugging
type cacheStats struct {
hits int64
misses int64
}
var stats cacheStats
// InitFS initializes the filesystem with the given base path
func InitFS(basePath string) error {
if basePath == "" {
@ -33,13 +47,20 @@ func InitFS(basePath string) error {
}
fsBasePath = absPath
// Initialize file cache with 2000 entries (reasonable for most use cases)
fileCache = lru.NewLRUCache(2000)
logger.Server("Virtual filesystem initialized at: %s", fsBasePath)
return nil
}
// CleanupFS performs any necessary cleanup
func CleanupFS() {
// Nothing to clean up currently
if fileCache != nil {
fileCache.Clear()
logger.Server("File cache cleared - Stats: hits=%d, misses=%d", stats.hits, stats.misses)
}
}
// ResolvePath resolves a given path relative to the filesystem base
@ -79,6 +100,11 @@ func ResolvePath(path string) (string, error) {
return fullPath, nil
}
// getCacheKey creates a cache key from path and modification time
func getCacheKey(fullPath string, modTime time.Time) string {
return fmt.Sprintf("%s:%d", fullPath, modTime.Unix())
}
// fsReadFile reads a file and returns its contents
func fsReadFile(state *luajit.State) int {
if !state.IsString(1) {
@ -93,12 +119,46 @@ func fsReadFile(state *luajit.State) int {
return -1
}
// Get file info for cache key and validation
info, err := os.Stat(fullPath)
if err != nil {
state.PushString("fs.read_file: " + err.Error())
return -1
}
// Create cache key with path and modification time
cacheKey := getCacheKey(fullPath, info.ModTime())
// Try to get from cache first
if fileCache != nil {
if cachedData, exists := fileCache.Get(cacheKey); exists {
if compressedData, ok := cachedData.([]byte); ok {
// Decompress cached data
data, err := snappy.Decode(nil, compressedData)
if err == nil {
stats.hits++
state.PushString(string(data))
return 1
}
// Cache corruption - continue to disk read
}
}
}
// Cache miss or error - read from disk
stats.misses++
data, err := os.ReadFile(fullPath)
if err != nil {
state.PushString("fs.read_file: " + err.Error())
return -1
}
// Compress and cache the data
if fileCache != nil {
compressedData := snappy.Encode(nil, data)
fileCache.Put(cacheKey, compressedData)
}
state.PushString(string(data))
return 1
}
@ -136,6 +196,12 @@ func fsWriteFile(state *luajit.State) int {
return -1
}
// Invalidate cache entries for this file path
if fileCache != nil {
// We can't easily iterate through cache keys, so we'll let the cache
// naturally expire old entries when the file is read again
}
state.PushBoolean(true)
return 1
}