Compare commits

...

4 Commits

Author SHA1 Message Date
da5f8b14b1 tiering: fix inverted test 2025-08-21 18:25:18 +12:00
d6d768980b tiering: skip directories 2025-08-21 18:25:18 +12:00
7d82cd8d57 tiering: add log messages 2025-08-21 18:25:12 +12:00
02bc633f1b tiering: shuffle file order 2025-08-21 18:15:56 +12:00

View File

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"io" "io"
"log" "log"
"math/rand"
"os" "os"
"path/filepath" "path/filepath"
"time" "time"
@ -155,20 +156,39 @@ func (ts *tieredStorage) migrateNow() error {
return fmt.Errorf("Reading hot storage files: %w", err) return fmt.Errorf("Reading hot storage files: %w", err)
} }
if len(dirents) == 0 {
return nil // Directory empty, nothing to do
}
cutOff := time.Now().Add(-TierMigrationAfter) cutOff := time.Now().Add(-TierMigrationAfter)
// Shuffle files to avoid getting stuck
rand.Shuffle(len(dirents), func(i, j int) {
dirents[i], dirents[j] = dirents[j], dirents[i]
})
log.Printf("tier-migration: Scanning %d items...", len(dirents))
var countMigrated int64 = 0
for _, dirent := range dirents { for _, dirent := range dirents {
fi, err := dirent.Info() fi, err := dirent.Info()
if err != nil { if err != nil {
return fmt.Errorf("Reading hot storage files: %w", err) // local files can't be stat'd = important error return fmt.Errorf("Reading hot storage files: %w", err) // local files can't be stat'd = important error
} }
if !fi.ModTime().After(cutOff) { if fi.IsDir() {
continue // probably . or ..
}
if fi.ModTime().After(cutOff) {
continue // not eligible continue // not eligible
} }
fileHash := dirent.Name() fileHash := dirent.Name()
log.Printf("tier-migration: Migrating %q...", fileHash)
// Copy to cold storage // Copy to cold storage
// Any concurrent reads will be serviced from the hot storage, so this // Any concurrent reads will be serviced from the hot storage, so this
// is a safe operation // is a safe operation
@ -188,9 +208,13 @@ func (ts *tieredStorage) migrateNow() error {
if err != nil { if err != nil {
return fmt.Errorf("Remove %q from hot storage: %w", err) // can't rm local file return fmt.Errorf("Remove %q from hot storage: %w", err) // can't rm local file
} }
countMigrated++
} }
// Migrated everything we can for now // Migrated everything we can for now
log.Printf("tier-migration: Sleeping (migrated %d/%d items)", countMigrated, len(dirents))
return nil return nil
} }