thumbnails: need to check fileHash, not fileId

tiering: add more verbose error logging
tiering: fix missing printf parameter in error message
2025-11-01 14:09:23 +13:00 · 2025-11-01 14:03:02 +13:00 · 2025-11-01 14:02:51 +13:00 · 2025-11-01 13:51:37 +13:00 · 2025-11-01 13:38:34 +13:00 · 2025-08-21 18:25:18 +12:00
11 changed files with 419 additions and 181 deletions
--- a/Metadata.go
+++ b/Metadata.go
@@ -9,7 +9,7 @@ import (
 	"os"
 	"time"

-	"github.com/speps/go-hashids"
+	"github.com/speps/go-hashids/v2"
 	bolt "go.etcd.io/bbolt"
 )

@@ -48,7 +48,11 @@ func idToString(v uint64) string {
 	hd := hashids.NewData()
 	hd.Salt = hashIdSalt
 	hd.MinLength = hashIdMinLength
-	h := hashids.NewWithData(hd)
+	h, err := hashids.NewWithData(hd)
+	if err != nil {
+		panic(err) // developer error
+	}
+
 	s, _ := h.EncodeInt64([]int64{int64(v)})
 	return s
 }
--- a/README.md
+++ b/README.md
@@ -11,6 +11,7 @@ The name is a pun on "content" and the -d suffix for server daemons.
 ## Features

 - Use local disk or S3-backed storage
+- Optional hot/cold storage tiering
 - Drag and drop upload
 - Multiple files upload
 - Pastebin upload
@@ -86,6 +87,13 @@ You can optionally supply additional ordered parameters to `contented.init`:

 ## Changelog

+2025-08-20: v1.6.1
+- Expanded error logging for tiered storage migrations
+
+2025-08-20: v1.6.0
+- Support hot/cold tiered storage to move files between local path and S3 bucket
+- Upgrade all dependencies
+
 2023-05-20: 1.5.1
 - Improve support for albums with no images, and for albums with missing interior images

--- a/Server.go
+++ b/Server.go
@@ -7,13 +7,10 @@ import (
 	"io/fs"
 	"log"
 	"net/http"
-	"os"
 	"regexp"
 	"strings"
 	"time"

-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/mxk/go-flowrate/flowrate"
 	bolt "go.etcd.io/bbolt"
 )
@@ -31,6 +28,7 @@ const (

 	STORAGE_LOCAL  int = 0
 	STORAGE_S3     int = 1
+	STORAGE_TIERED int = 2
 )

 type ServerPublicProperties struct {
@@ -39,10 +37,7 @@ type ServerPublicProperties struct {
 	CanonicalBaseURL string
 }

-type ServerOptions struct {
-	StorageType   int // STORAGE_xx
-	DataDirectory string
-	DataS3Options struct {
+type ServerS3StorageOptions struct {
 	Hostname  string
 	AccessKey string
 	SecretKey string
@@ -50,6 +45,10 @@ type ServerOptions struct {
 	Prefix    string
 }

+type ServerOptions struct {
+	StorageType            int // STORAGE_xx
+	DataDirectory          string
+	DataS3Options          ServerS3StorageOptions
 	DBPath                 string
 	DiskFilesWorldReadable bool
 	BandwidthLimit         int64
@@ -61,22 +60,14 @@ type ServerOptions struct {
 	ServerPublicProperties
 }

-func (this *ServerOptions) FileMode() os.FileMode {
-	if this.DiskFilesWorldReadable {
-		return 0644
-	} else {
-		return 0600
-	}
-}
-
 type Server struct {
 	opts           ServerOptions
-	s3client       *minio.Client
 	db             *bolt.DB
 	startTime      time.Time
 	thumbnailSem   chan struct{}
 	metadataBucket []byte
 	staticDir      fs.FS // interface
+	store          Storage
 }

 func NewServer(opts *ServerOptions) (*Server, error) {
@@ -99,16 +90,30 @@ func NewServer(opts *ServerOptions) (*Server, error) {
 	s.staticDir, _ = fs.Sub(staticAssets, `static`) // can't fail

 	// Maybe open s3 connection
-	if s.opts.StorageType == STORAGE_S3 {
-		cl, err := minio.New(opts.DataS3Options.Hostname, &minio.Options{
-			Creds:  credentials.NewStaticV4(opts.DataS3Options.AccessKey, opts.DataS3Options.SecretKey, ""),
-			Secure: true,
-		})
+	var err error = nil
+	switch s.opts.StorageType {
+	case STORAGE_S3:
+		s.store, err = NewS3Storage(s.opts.DataS3Options)
 		if err != nil {
-			return nil, fmt.Errorf("Connecting to S3 host: %w", err)
+			return nil, err
 		}

-		s.s3client = cl
+	case STORAGE_LOCAL:
+		s.store = NewLocalStorage(s.opts.DataDirectory, s.opts.DiskFilesWorldReadable)
+
+	case STORAGE_TIERED:
+		coldStore, err := NewS3Storage(s.opts.DataS3Options)
+		if err != nil {
+			return nil, err
+		}
+
+		s.store = NewTieredStorage(
+			NewLocalStorage(s.opts.DataDirectory, s.opts.DiskFilesWorldReadable),
+			coldStore,
+		)
+
+	default:
+		return nil, fmt.Errorf("Invalid storage type %d", s.opts.StorageType)
 	}

 	// "fill" the thumbnailer semaphore
--- a/cmd/contented/main.go
+++ b/cmd/contented/main.go
@@ -47,17 +47,24 @@ func main() {
 		},
 	}

-	if len(*s3AccessKey) > 0 {
-		opts.StorageType = contented.STORAGE_S3
+	// s3 or tiered storage
 	opts.DataS3Options.Hostname = *s3Host
 	opts.DataS3Options.AccessKey = *s3AccessKey
 	opts.DataS3Options.SecretKey = *s3SecretKey
 	opts.DataS3Options.Bucket = *s3Bucket
 	opts.DataS3Options.Prefix = *s3Prefix

+	// local or tiered storage
+	opts.DataDirectory = *dataDir
+
+	if len(*dataDir) > 0 && len(*s3AccessKey) > 0 {
+		opts.StorageType = contented.STORAGE_TIERED
+
+	} else if len(*s3AccessKey) > 0 {
+		opts.StorageType = contented.STORAGE_S3
+
 	} else if len(*dataDir) > 0 {
 		opts.StorageType = contented.STORAGE_LOCAL
-		opts.DataDirectory = *dataDir

 	} else {
 		log.Println("Please specify either the -data or -s3__ options.")
--- a/download.go
+++ b/download.go
@@ -27,7 +27,7 @@ func (this *Server) handleViewInternal(w http.ResponseWriter, r *http.Request, f
 	}

 	// Load file
-	f, err := this.ReadFile(r.Context(), m.FileHash)
+	f, err := this.store.ReadFile(r.Context(), m.FileHash)
 	if err != nil {
 		return err
 	}
--- a/go.mod
+++ b/go.mod
@@ -2,32 +2,33 @@ module code.ivysaur.me/contented

 require (
 	code.ivysaur.me/thumbnail v1.0.2
+	github.com/minio/minio-go/v7 v7.0.95
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f
-	github.com/speps/go-hashids v1.0.0
-	go.etcd.io/bbolt v1.3.7
+	github.com/speps/go-hashids/v2 v2.0.1
+	go.etcd.io/bbolt v1.4.3
 )

 require (
 	code.ivysaur.me/imagequant/v2 v2.12.6 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.16.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.52 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/rs/xid v1.4.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
-	golang.org/x/crypto v0.6.0 // indirect
-	golang.org/x/image v0.0.0-20200618115811-c13761719519 // indirect
-	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
+	github.com/philhofer/fwd v1.2.0 // indirect
+	github.com/rs/xid v1.6.0 // indirect
+	github.com/tinylib/msgp v1.3.0 // indirect
+	golang.org/x/crypto v0.41.0 // indirect
+	golang.org/x/image v0.30.0 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
 )

-go 1.19
+go 1.23.0
+
+toolchain go1.24.4
--- a/go.sum
+++ b/go.sum
@@ -2,68 +2,59 @@ code.ivysaur.me/imagequant/v2 v2.12.6 h1:xYrGj6GOdAcutmzqBxG7bDZ70r4jYHADOCZ+kty
 code.ivysaur.me/imagequant/v2 v2.12.6/go.mod h1:seCAm0sP2IBsb1YNBj4D+EZovIuGe16+6Xo0aiGyhDU=
 code.ivysaur.me/thumbnail v1.0.2 h1:vQaRPbBZOUGpr4b5rrUOHiZv08XSRJ83uu64WXFx7mo=
 code.ivysaur.me/thumbnail v1.0.2/go.mod h1:sXeHBfmPfiSe5ZBKsbGSES13C9OSZq0WmT4yZ/XBeeE=
-github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
-github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
+github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
-github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
+github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps=
-github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
-github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
-github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=
+github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
+github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
-github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/speps/go-hashids v1.0.0 h1:jdFC07PrExRM4Og5Ev4411Tox75aFpkC77NlmutadNI=
-github.com/speps/go-hashids v1.0.0/go.mod h1:P7hqPzMdnZOfyIk+xrlG1QaSMw+gCBdHKsBDnhpaZvc=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
-go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
-golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/image v0.0.0-20200618115811-c13761719519 h1:1e2ufUJNM3lCHEY5jIgac/7UTjd6cgJNdatjPdFWf34=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/speps/go-hashids/v2 v2.0.1 h1:ViWOEqWES/pdOSq+C1SLVa8/Tnsd52XC34RY7lt7m4g=
+github.com/speps/go-hashids/v2 v2.0.1/go.mod h1:47LKunwvDZki/uRVD6NImtyk712yFzIs3UF3KlHohGw=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
+github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
+go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
+go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
+golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
+golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
 golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/sys v0.0.0-20180606202747-9527bec2660b h1:5rOiLYVqtE+JehJPVJTXQJaP8aT3cpJC1Iy22+5WLFU=
-golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
-golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/image v0.30.0 h1:jD5RhkmVAnjqaCUXfbGBrn3lpxbknfN9w2UhHHU+5B4=
+golang.org/x/image v0.30.0/go.mod h1:SAEUTxCCMWSrJcCy/4HwavEsfZZJlYxeHLc6tTiAe/c=
+golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
+golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
-gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/preview.go
+++ b/preview.go
@@ -139,7 +139,7 @@ html, body {

 		if m.MimeType == ALBUM_MIMETYPE {
 			// Special handling for albums
-			f, err := this.ReadFile(ctx, m.FileHash)
+			f, err := this.store.ReadFile(ctx, m.FileHash)
 			if err != nil {
 				log.Printf("Opening file '%s' for preview of album '%s': %s", m.FileHash, fileID, err.Error())
 				http.Error(w, "Internal error", 500)
--- a/storage.go
+++ b/storage.go
@@ -2,36 +2,59 @@ package contented

 import (
 	"context"
+	"fmt"
 	"io"
+	"log"
+	"math/rand"
 	"os"
 	"path/filepath"
+	"time"

 	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 )

-func (this *Server) ReadFile(ctx context.Context, fileHash string) (io.ReadSeekCloser, error) {
-	if this.opts.StorageType == STORAGE_LOCAL {
-		fh, err := os.Open(filepath.Join(this.opts.DataDirectory, fileHash))
+type cleanupFunc func() error
+
+type Storage interface {
+	ReadFile(ctx context.Context, fileHash string) (io.ReadSeekCloser, error)
+	SaveFile(ctx context.Context, fileHash string, srcLen int64, src io.Reader) error
+
+	// TempLocalFilepath returns a local file path and a cleanup function that
+	// can be used to access the file through the filesystem.
+	// It's an alternative to ReadFile.
+	TempLocalFilepath(ctx context.Context, fileHash string) (string, cleanupFunc, error)
+}
+
+//
+
+type localStorage struct {
+	dataDir  string
+	fileMode os.FileMode
+}
+
+func NewLocalStorage(dataDir string, worldReadable bool) *localStorage {
+	ls := &localStorage{
+		dataDir:  dataDir,
+		fileMode: 0600,
+	}
+
+	if worldReadable {
+		ls.fileMode = 0644
+	}
+
+	return ls
+}
+
+func (ls *localStorage) ReadFile(ctx context.Context, fileHash string) (io.ReadSeekCloser, error) {
+	fh, err := os.Open(filepath.Join(ls.dataDir, fileHash))
 	return fh, err
-
-	} else if this.opts.StorageType == STORAGE_S3 {
-		obj, err := this.s3client.GetObject(ctx, this.opts.DataS3Options.Bucket, this.opts.DataS3Options.Prefix+fileHash, minio.GetObjectOptions{})
-		if err != nil {
-			return nil, err
 }

-		return obj, nil
-
-	} else {
-		panic("bad StorageType")
-	}
-}
-
-func (this *Server) SaveFile(ctx context.Context, fileHash string, srcLen int64, src io.Reader) error {
-	if this.opts.StorageType == STORAGE_LOCAL {
+func (ls *localStorage) SaveFile(ctx context.Context, fileHash string, srcLen int64, src io.Reader) error {

 	// Save file to disk
-		dest, err := os.OpenFile(filepath.Join(this.opts.DataDirectory, fileHash), os.O_CREATE|os.O_WRONLY, this.opts.FileMode())
+	dest, err := os.OpenFile(filepath.Join(ls.dataDir, fileHash), os.O_CREATE|os.O_WRONLY, ls.fileMode)
 	if err != nil {
 		if os.IsExist(err) {
 			return nil // hash matches existing upload
@@ -46,12 +69,229 @@ func (this *Server) SaveFile(ctx context.Context, fileHash string, srcLen int64,
 		return err
 	}
 	return nil
+}

-	} else if this.opts.StorageType == STORAGE_S3 {
-		_, err := this.s3client.PutObject(ctx, this.opts.DataS3Options.Bucket, this.opts.DataS3Options.Prefix+fileHash, src, srcLen, minio.PutObjectOptions{})
+func (ls *localStorage) TempLocalFilepath(ctx context.Context, fileHash string) (string, cleanupFunc, error) {
+	path := filepath.Join(ls.dataDir, fileHash)
+	cleanup := func() error { return nil }
+
+	if _, err := os.Stat(path); err != nil {
+		return "", nil, err // e.g. not exists
+	}
+
+	// Exists (but may be TOCTTOU)
+
+	return path, cleanup, nil
+}
+
+var _ Storage = &localStorage{} // interface assertion
+
+//
+
+type s3Storage struct {
+	s3client *minio.Client
+	ServerS3StorageOptions
+}
+
+func NewS3Storage(opts ServerS3StorageOptions) (*s3Storage, error) {
+
+	cl, err := minio.New(opts.Hostname, &minio.Options{
+		Creds:  credentials.NewStaticV4(opts.AccessKey, opts.SecretKey, ""),
+		Secure: true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Connecting to S3 host: %w", err)
+	}
+
+	return &s3Storage{
+		s3client:               cl,
+		ServerS3StorageOptions: opts,
+	}, nil
+}
+
+func (ss *s3Storage) ReadFile(ctx context.Context, fileHash string) (io.ReadSeekCloser, error) {
+	obj, err := ss.s3client.GetObject(ctx, ss.Bucket, ss.Prefix+fileHash, minio.GetObjectOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return obj, nil
+}
+
+func (ss *s3Storage) SaveFile(ctx context.Context, fileHash string, srcLen int64, src io.Reader) error {
+	_, err := ss.s3client.PutObject(ctx, ss.Bucket, ss.Prefix+fileHash, src, srcLen, minio.PutObjectOptions{})
 	return err
+}

-	} else {
-		panic("bad StorageType")
+func (ss *s3Storage) TempLocalFilepath(ctx context.Context, fileHash string) (string, cleanupFunc, error) {
+
+	// Download to temporary file
+	fh, err := os.CreateTemp("", "contented-temp-*")
+	if err != nil {
+		return "", nil, err
+	}
+
+	name := fh.Name() // n.b. This is the absolute path to the file
+
+	r, err := ss.ReadFile(ctx, fileHash)
+	if err != nil {
+		return "", nil, err
+	}
+
+	_, err = io.Copy(fh, r)
+	if err != nil {
+		return "", nil, err
+	}
+
+	_ = r.Close()
+	_ = fh.Close()
+
+	cleanup := func() error {
+		return os.Remove(name)
+	}
+
+	return name, cleanup, nil
+}
+
+var _ Storage = &s3Storage{} // interface assertion
+
+//
+
+const (
+	TierMigrationAfter        = 14 * 24 * time.Hour // 14 days
+	TierMigrationEvery        = 4 * time.Hour       // 4 hours
+	TierMigrationDelayStartup = 60 * time.Second
+)
+
+type tieredStorage struct {
+	hot  *localStorage
+	cold *s3Storage
+}
+
+func NewTieredStorage(hot *localStorage, cold *s3Storage) *tieredStorage {
+	ts := &tieredStorage{
+		hot:  hot,
+		cold: cold,
+	}
+
+	go ts.migrationWorker()
+
+	return ts
+}
+
+// migrationWorker is a background goroutine to trigger tier migrations.
+func (ts *tieredStorage) migrationWorker() {
+
+	// Startup delay
+	time.Sleep(TierMigrationDelayStartup)
+
+	// Worker loop
+	for {
+		err := ts.migrateNow()
+		if err != nil {
+			log.Printf("tier-migration: %v", err)
+		}
+
+		time.Sleep(TierMigrationEvery)
 	}
 }
+
+// migrateNow performs a tier migration for old files.
+func (ts *tieredStorage) migrateNow() error {
+
+	// List local files
+	dirents, err := os.ReadDir(ts.hot.dataDir)
+	if err != nil {
+		return fmt.Errorf("Reading hot storage files: %w", err)
+	}
+
+	if len(dirents) == 0 {
+		return nil // Directory empty, nothing to do
+	}
+
+	cutOff := time.Now().Add(-TierMigrationAfter)
+
+	// Shuffle files to avoid getting stuck
+	rand.Shuffle(len(dirents), func(i, j int) {
+		dirents[i], dirents[j] = dirents[j], dirents[i]
+	})
+
+	log.Printf("tier-migration: Scanning %d items...", len(dirents))
+
+	var countMigrated int64 = 0
+
+	for _, dirent := range dirents {
+		fi, err := dirent.Info()
+		if err != nil {
+			return fmt.Errorf("Reading hot storage files: %w", err) // local files can't be stat'd = important error
+		}
+
+		if fi.IsDir() {
+			continue // probably . or ..
+		}
+
+		if fi.ModTime().After(cutOff) {
+			continue // not eligible
+		}
+
+		fileHash := dirent.Name()
+
+		log.Printf("tier-migration: Migrating %q...", fileHash)
+
+		// Copy to cold storage
+		// Any concurrent reads will be serviced from the hot storage, so this
+		// is a safe operation
+		rc, err := ts.hot.ReadFile(context.Background(), fileHash)
+		if err != nil {
+			return fmt.Errorf("Read %q from hot storage: %w", fileHash, err) // can't cat local file
+		}
+
+		err = ts.cold.SaveFile(context.Background(), fileHash, fi.Size(), rc)
+		_ = rc.Close()
+		if err != nil {
+			return fmt.Errorf("Write %q to cold storage: %w", fileHash, err) // can't save local file
+		}
+
+		// Copy was successful. Delete local file
+		err = os.Remove(filepath.Join(ts.hot.dataDir, fileHash))
+		if err != nil {
+			return fmt.Errorf("Remove %q from hot storage: %w", fileHash, err) // can't rm local file
+		}
+
+		countMigrated++
+	}
+
+	// Migrated everything we can for now
+	log.Printf("tier-migration: Sleeping (migrated %d/%d items)", countMigrated, len(dirents))
+
+	return nil
+}
+
+func (ts *tieredStorage) ReadFile(ctx context.Context, fileHash string) (io.ReadSeekCloser, error) {
+	if rc, err := ts.hot.ReadFile(ctx, fileHash); err == nil {
+		return rc, nil
+	}
+
+	return ts.cold.ReadFile(ctx, fileHash)
+}
+
+func (ts *tieredStorage) SaveFile(ctx context.Context, fileHash string, srcLen int64, src io.Reader) error {
+	return ts.hot.SaveFile(ctx, fileHash, srcLen, src)
+}
+
+func (ts *tieredStorage) TempLocalFilepath(ctx context.Context, fileHash string) (string, cleanupFunc, error) {
+	path, cleanup, err_hot := ts.hot.TempLocalFilepath(ctx, fileHash)
+	if err_hot == nil {
+		return path, cleanup, nil
+	}
+
+	path, cleanup, err_cold := ts.cold.TempLocalFilepath(ctx, fileHash)
+	if err_cold == nil {
+		return path, cleanup, nil
+	}
+
+	// Neither err_hot nor err_cold worked
+	return "", nil, fmt.Errorf("TempLocalFilepath: failed to allocate local path for %q (hot error: %q, cold error: %q)", fileHash, err_hot.Error(), err_cold.Error())
+}
+
+var _ Storage = &tieredStorage{} // interface assertion
--- a/thumb.go
+++ b/thumb.go
@@ -4,11 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io"
 	"log"
 	"net/http"
-	"os"
-	"path/filepath"

 	"code.ivysaur.me/thumbnail"
 )
@@ -93,45 +90,30 @@ func (this *Server) handleThumbInternal(ctx context.Context, w http.ResponseWrit
 	// Load metadata
 	m, err := this.Metadata(fileId)
 	if err != nil {
-		return err
+		return fmt.Errorf("Metadata: %w", err)
 	}

 	if m.FileSize > this.opts.MaxThumbSizeBytes {
 		return errors.New("Don't want to thumbnail very large files, sorry")
 	}

-	var filePath string
-
-	if this.opts.StorageType == STORAGE_LOCAL {
-		filePath = filepath.Join(this.opts.DataDirectory, m.FileHash)
-
-	} else if this.opts.StorageType == STORAGE_S3 {
-		// Need to temporarily download it for thumbnailing (slow and costs money)
-
-		destFh, err := os.CreateTemp("", "contented-thumbcache-*")
-		defer os.Remove(destFh.Name())
-
-		srcFh, err := this.ReadFile(ctx, m.FileHash)
+	// Some storage backends can supply a local filepath immediately for thumbnailing
+	// Others (s3, tiered) need to download the file temporarily first
+	filePath, cleanup, err := this.store.TempLocalFilepath(ctx, m.FileHash)
 	if err != nil {
-			return err
+		return fmt.Errorf("TempLocalFilepath: %w", err)
 	}

-		_, err = io.CopyN(destFh, srcFh, m.FileSize)
-		srcFh.Close()
-		if err != nil {
-			return err
-		}
-
-		destFh.Seek(0, io.SeekStart)
-		filePath = destFh.Name()
-
-	} else {
-		panic("bad StorageType")
+	defer func() {
+		// delete temporary file, if necessary
+		if cleanupErr := cleanup(); cleanupErr != nil {
+			log.Printf("cleaning up temporary thumbnail file: %v", cleanupErr)
 		}
+	}()

 	thumb, err := t.RenderFileAs(filePath, m.MimeType)
 	if err != nil {
-		return err
+		return fmt.Errorf("RenderFileAs: %w", err)
 	}

 	w.Header().Set(`Cache-Control`, `max-age=31536000, immutable`)
--- a/upload.go
+++ b/upload.go
@@ -96,7 +96,7 @@ func (this *Server) handleUploadFile(src multipart.File, hdr *multipart.FileHead
 	fileHash := hex.EncodeToString(hasher.Sum(nil))

 	// Save file to disk/s3
-	err = this.SaveFile(context.Background(), fileHash, srcLen, src)
+	err = this.store.SaveFile(context.Background(), fileHash, srcLen, src)
 	if err != nil {
 		return "", err
 	}
Author	SHA1	Message	Date
mappu	a83a34a047	thumbnails: need to check fileHash, not fileId	2025-11-01 14:09:23 +13:00
mappu	3cf3a5ae06	tiering: add more verbose error logging	2025-11-01 14:03:02 +13:00
mappu	7bf72f2848	tiering: fix missing printf parameter in error message	2025-11-01 14:02:51 +13:00
mappu	a9031c8a2b	tiering: localfilepath should return an error if file is inaccessible	2025-11-01 13:51:37 +13:00
mappu	e891c77bce	tiering: support thumbnails	2025-11-01 13:38:34 +13:00
mappu	da5f8b14b1	tiering: fix inverted test	2025-08-21 18:25:18 +12:00
mappu	d6d768980b	tiering: skip directories	2025-08-21 18:25:18 +12:00
mappu	7d82cd8d57	tiering: add log messages	2025-08-21 18:25:12 +12:00
mappu	02bc633f1b	tiering: shuffle file order	2025-08-21 18:15:56 +12:00
mappu	2434cccf59	doc/README: changelog for v1.6.1	2025-08-20 19:27:25 +12:00
mappu	426d5738a0	storage/tiering: add more error messages	2025-08-20 19:26:46 +12:00
mappu	76c2e552ab	storage/tiering: fix read path	2025-08-20 19:26:39 +12:00
mappu	ea9ef2e466	doc/README: changelog for v1.6.0	2025-08-20 16:03:05 +12:00
mappu	6b5b11fb5f	deps update	2025-08-20 16:02:55 +12:00
mappu	156115d5f9	cmd/contented: allow both local+s3 on cli to use tiered storage	2025-08-20 15:56:21 +12:00
mappu	44af6efc87	storage: support new STORAGE_TIERED	2025-08-20 15:56:10 +12:00
mappu	9bd6e881b2	storage: refactor local/s3 to use an interface	2025-08-18 19:18:24 +12:00