Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2434cccf59 | |||
| 426d5738a0 | |||
| 76c2e552ab |
@@ -87,6 +87,9 @@ You can optionally supply additional ordered parameters to `contented.init`:
|
|||||||
|
|
||||||
## Changelog
|
## Changelog
|
||||||
|
|
||||||
|
2025-08-20: v1.6.1
|
||||||
|
- Expanded error logging for tiered storage migrations
|
||||||
|
|
||||||
2025-08-20: v1.6.0
|
2025-08-20: v1.6.0
|
||||||
- Support hot/cold tiered storage to move files between local path and S3 bucket
|
- Support hot/cold tiered storage to move files between local path and S3 bucket
|
||||||
- Upgrade all dependencies
|
- Upgrade all dependencies
|
||||||
|
|||||||
12
storage.go
12
storage.go
@@ -152,7 +152,7 @@ func (ts *tieredStorage) migrateNow() error {
|
|||||||
// List local files
|
// List local files
|
||||||
dirents, err := os.ReadDir(ts.hot.dataDir)
|
dirents, err := os.ReadDir(ts.hot.dataDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("Reading hot storage files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cutOff := time.Now().Add(-TierMigrationAfter)
|
cutOff := time.Now().Add(-TierMigrationAfter)
|
||||||
@@ -160,7 +160,7 @@ func (ts *tieredStorage) migrateNow() error {
|
|||||||
for _, dirent := range dirents {
|
for _, dirent := range dirents {
|
||||||
fi, err := dirent.Info()
|
fi, err := dirent.Info()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err // local files can't be stat'd = important error
|
return fmt.Errorf("Reading hot storage files: %w", err) // local files can't be stat'd = important error
|
||||||
}
|
}
|
||||||
|
|
||||||
if !fi.ModTime().After(cutOff) {
|
if !fi.ModTime().After(cutOff) {
|
||||||
@@ -172,21 +172,21 @@ func (ts *tieredStorage) migrateNow() error {
|
|||||||
// Copy to cold storage
|
// Copy to cold storage
|
||||||
// Any concurrent reads will be serviced from the hot storage, so this
|
// Any concurrent reads will be serviced from the hot storage, so this
|
||||||
// is a safe operation
|
// is a safe operation
|
||||||
rc, err := ts.cold.ReadFile(context.Background(), fileHash)
|
rc, err := ts.hot.ReadFile(context.Background(), fileHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err // can't cat local file
|
return fmt.Errorf("Read %q from hot storage: %w", fileHash, err) // can't cat local file
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ts.cold.SaveFile(context.Background(), fileHash, fi.Size(), rc)
|
err = ts.cold.SaveFile(context.Background(), fileHash, fi.Size(), rc)
|
||||||
_ = rc.Close()
|
_ = rc.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err // can't save local file
|
return fmt.Errorf("Write %q to cold storage: %w", fileHash, err) // can't save local file
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy was successful. Delete local file
|
// Copy was successful. Delete local file
|
||||||
err = os.Remove(filepath.Join(ts.hot.dataDir, fileHash))
|
err = os.Remove(filepath.Join(ts.hot.dataDir, fileHash))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err // can't rm local file
|
return fmt.Errorf("Remove %q from hot storage: %w", err) // can't rm local file
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user