Compare commits

..

No commits in common. "master" and "v1.6.0" have entirely different histories.

2 changed files with 7 additions and 34 deletions

View File

@ -87,9 +87,6 @@ You can optionally supply additional ordered parameters to `contented.init`:
## Changelog
2025-08-20: v1.6.1
- Expanded error logging for tiered storage migrations
2025-08-20: v1.6.0
- Support hot/cold tiered storage to move files between local path and S3 bucket
- Upgrade all dependencies

View File

@ -5,7 +5,6 @@ import (
"fmt"
"io"
"log"
"math/rand"
"os"
"path/filepath"
"time"
@ -153,68 +152,45 @@ func (ts *tieredStorage) migrateNow() error {
// List local files
dirents, err := os.ReadDir(ts.hot.dataDir)
if err != nil {
return fmt.Errorf("Reading hot storage files: %w", err)
}
if len(dirents) == 0 {
return nil // Directory empty, nothing to do
return err
}
cutOff := time.Now().Add(-TierMigrationAfter)
// Shuffle files to avoid getting stuck
rand.Shuffle(len(dirents), func(i, j int) {
dirents[i], dirents[j] = dirents[j], dirents[i]
})
log.Printf("tier-migration: Scanning %d items...", len(dirents))
var countMigrated int64 = 0
for _, dirent := range dirents {
fi, err := dirent.Info()
if err != nil {
return fmt.Errorf("Reading hot storage files: %w", err) // local files can't be stat'd = important error
return err // local files can't be stat'd = important error
}
if fi.IsDir() {
continue // probably . or ..
}
if fi.ModTime().After(cutOff) {
if !fi.ModTime().After(cutOff) {
continue // not eligible
}
fileHash := dirent.Name()
log.Printf("tier-migration: Migrating %q...", fileHash)
// Copy to cold storage
// Any concurrent reads will be serviced from the hot storage, so this
// is a safe operation
rc, err := ts.hot.ReadFile(context.Background(), fileHash)
rc, err := ts.cold.ReadFile(context.Background(), fileHash)
if err != nil {
return fmt.Errorf("Read %q from hot storage: %w", fileHash, err) // can't cat local file
return err // can't cat local file
}
err = ts.cold.SaveFile(context.Background(), fileHash, fi.Size(), rc)
_ = rc.Close()
if err != nil {
return fmt.Errorf("Write %q to cold storage: %w", fileHash, err) // can't save local file
return err // can't save local file
}
// Copy was successful. Delete local file
err = os.Remove(filepath.Join(ts.hot.dataDir, fileHash))
if err != nil {
return fmt.Errorf("Remove %q from hot storage: %w", err) // can't rm local file
return err // can't rm local file
}
countMigrated++
}
// Migrated everything we can for now
log.Printf("tier-migration: Sleeping (migrated %d/%d items)", countMigrated, len(dirents))
return nil
}