Compare commits
4 Commits
Author | SHA1 | Date | |
---|---|---|---|
da5f8b14b1 | |||
d6d768980b | |||
7d82cd8d57 | |||
02bc633f1b |
26
storage.go
26
storage.go
@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
@ -155,20 +156,39 @@ func (ts *tieredStorage) migrateNow() error {
|
|||||||
return fmt.Errorf("Reading hot storage files: %w", err)
|
return fmt.Errorf("Reading hot storage files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(dirents) == 0 {
|
||||||
|
return nil // Directory empty, nothing to do
|
||||||
|
}
|
||||||
|
|
||||||
cutOff := time.Now().Add(-TierMigrationAfter)
|
cutOff := time.Now().Add(-TierMigrationAfter)
|
||||||
|
|
||||||
|
// Shuffle files to avoid getting stuck
|
||||||
|
rand.Shuffle(len(dirents), func(i, j int) {
|
||||||
|
dirents[i], dirents[j] = dirents[j], dirents[i]
|
||||||
|
})
|
||||||
|
|
||||||
|
log.Printf("tier-migration: Scanning %d items...", len(dirents))
|
||||||
|
|
||||||
|
var countMigrated int64 = 0
|
||||||
|
|
||||||
for _, dirent := range dirents {
|
for _, dirent := range dirents {
|
||||||
fi, err := dirent.Info()
|
fi, err := dirent.Info()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Reading hot storage files: %w", err) // local files can't be stat'd = important error
|
return fmt.Errorf("Reading hot storage files: %w", err) // local files can't be stat'd = important error
|
||||||
}
|
}
|
||||||
|
|
||||||
if !fi.ModTime().After(cutOff) {
|
if fi.IsDir() {
|
||||||
|
continue // probably . or ..
|
||||||
|
}
|
||||||
|
|
||||||
|
if fi.ModTime().After(cutOff) {
|
||||||
continue // not eligible
|
continue // not eligible
|
||||||
}
|
}
|
||||||
|
|
||||||
fileHash := dirent.Name()
|
fileHash := dirent.Name()
|
||||||
|
|
||||||
|
log.Printf("tier-migration: Migrating %q...", fileHash)
|
||||||
|
|
||||||
// Copy to cold storage
|
// Copy to cold storage
|
||||||
// Any concurrent reads will be serviced from the hot storage, so this
|
// Any concurrent reads will be serviced from the hot storage, so this
|
||||||
// is a safe operation
|
// is a safe operation
|
||||||
@ -188,9 +208,13 @@ func (ts *tieredStorage) migrateNow() error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Remove %q from hot storage: %w", err) // can't rm local file
|
return fmt.Errorf("Remove %q from hot storage: %w", err) // can't rm local file
|
||||||
}
|
}
|
||||||
|
|
||||||
|
countMigrated++
|
||||||
}
|
}
|
||||||
|
|
||||||
// Migrated everything we can for now
|
// Migrated everything we can for now
|
||||||
|
log.Printf("tier-migration: Sleeping (migrated %d/%d items)", countMigrated, len(dirents))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user