s3 backend support

Server.go

@@ -3,6 +3,7 @@ package contented
 import (
 	"embed"
 	"encoding/json"
+	"fmt"
 	"io/fs"
 	"log"
 	"net/http"
@@ -11,6 +12,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/mxk/go-flowrate/flowrate"
 	bolt "go.etcd.io/bbolt"
 )
@@ -26,6 +29,8 @@ const (
 
 	ALBUM_MIMETYPE = `contented/album`
 
+	STORAGE_LOCAL int = 0
+	STORAGE_S3    int = 1
 )
 
 type ServerPublicProperties struct {
@@ -35,7 +40,16 @@ type ServerPublicProperties struct {
 }
 
 type ServerOptions struct {
-	DataDirectory          string
+	StorageType   int // STORAGE_xx
+	DataDirectory string
+	DataS3Options struct {
+		Hostname  string
+		AccessKey string
+		SecretKey string
+		Bucket    string
+		Prefix    string
+	}
+
 	DBPath                 string
 	DiskFilesWorldReadable bool
 	BandwidthLimit         int64
@@ -57,6 +71,7 @@ func (this *ServerOptions) FileMode() os.FileMode {
 
 type Server struct {
 	opts           ServerOptions
+	s3client       *minio.Client
 	db             *bolt.DB
 	startTime      time.Time
 	thumbnailSem   chan struct{}
@@ -83,6 +98,19 @@ func NewServer(opts *ServerOptions) (*Server, error) {
 
 	s.staticDir, _ = fs.Sub(staticAssets, `static`) // can't fail
 
+	// Maybe open s3 connection
+	if s.opts.StorageType == STORAGE_S3 {
+		cl, err := minio.New(opts.DataS3Options.Hostname, &minio.Options{
+			Creds:  credentials.NewStaticV4(opts.DataS3Options.AccessKey, opts.DataS3Options.SecretKey, ""),
+			Secure: true,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("Connecting to S3 host: %w", err)
+		}
+
+		s.s3client = cl
+	}
+
 	// "fill" the thumbnailer semaphore
 	s.thumbnailSem = make(chan struct{}, s.opts.MaxConcurrentThumbs)
 	for i := 0; i < s.opts.MaxConcurrentThumbs; i += 1 {

cmd/contented/main.go

@@ -24,12 +24,16 @@ func main() {
 		enableUpload           = flag.Bool("enableUpload", true, "Enable uploads (disable for read-only mode)")
 		diskFilesWorldReadable = flag.Bool("diskFilesWorldReadable", false, "Save files as 0644 instead of 0600")
 		maxConcurrentThumbs    = flag.Int("concurrentthumbs", contented.DEFAULT_MAX_CONCURRENT_THUMBS, "Simultaneous thumbnail generation")
+		s3Host                 = flag.String("s3hostname", "", "S3 Server hostname")
+		s3AccessKey            = flag.String("s3access", "", "S3 Access key")
+		s3SecretKey            = flag.String("s3secret", "", "S3 Secret key")
+		s3Bucket               = flag.String("s3bucket", "", "S3 Bucket")
+		s3Prefix               = flag.String("s3prefix", "", "S3 object prefix")
 	)
 
 	flag.Parse()
 
-	svr, err := contented.NewServer(&contented.ServerOptions{
+	opts := contented.ServerOptions{
-		DataDirectory:          *dataDir,
 		DBPath:                 *dbPath,
 		BandwidthLimit:         int64(*maxUploadSpeed),
 		TrustXForwardedFor:     *trustXForwardedFor,
@@ -41,7 +45,26 @@ func main() {
 			AppTitle:       *appTitle,
 			MaxUploadBytes: int64(*maxUploadMb) * 1024 * 1024,
 		},
-	})
+	}
+
+	if len(*s3AccessKey) > 0 {
+		opts.StorageType = contented.STORAGE_S3
+		opts.DataS3Options.Hostname = *s3Host
+		opts.DataS3Options.AccessKey = *s3AccessKey
+		opts.DataS3Options.SecretKey = *s3SecretKey
+		opts.DataS3Options.Bucket = *s3Bucket
+		opts.DataS3Options.Prefix = *s3Prefix
+
+	} else if len(*dataDir) > 0 {
+		opts.StorageType = contented.STORAGE_LOCAL
+		opts.DataDirectory = *dataDir
+
+	} else {
+		log.Println("Please specify either the -data or -s3__ options.")
+		os.Exit(1)
+	}
+
+	svr, err := contented.NewServer(&opts)
 	if err != nil {
 		log.Println(err.Error())
 		os.Exit(1)

download.go

@@ -4,7 +4,6 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"path/filepath"
 )
 
 func (this *Server) handleView(w http.ResponseWriter, r *http.Request, fileID string) {
@@ -28,7 +27,7 @@ func (this *Server) handleViewInternal(w http.ResponseWriter, r *http.Request, f
 	}
 
 	// Load file
-	f, err := os.Open(filepath.Join(this.opts.DataDirectory, m.FileHash))
+	f, err := this.ReadFile(r.Context(), m.FileHash)
 	if err != nil {
 		return err
 	}
@@ -50,7 +49,18 @@ func (this *Server) handleViewInternal(w http.ResponseWriter, r *http.Request, f
 		w.Header().Set(`Content-Type`, m.MimeType)
 	}
 
-	http.ServeContent(w, r, "", m.UploadTime, f)
+	/*
+		if _, ok := f.(io.ReadSeeker); ! ok {
+			// Stream directly, no support for bytes/etag
+			w.Header().Set(`Content-Length`, fmt.Sprintf("%d", m.FileSize))
+			_, err := io.Copy(w, f)
+			return err
 
+		}
+	*/
+
+	// Allow range requests, if-modified-since, and so on
+	http.ServeContent(w, r, "", m.UploadTime, f)
 	return nil
+
 }

preview.go

@@ -8,7 +8,6 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strings"
 	"time"
 )
@@ -121,7 +120,7 @@ html, body {
 
 		if m.MimeType == ALBUM_MIMETYPE {
 			// Special handling for albums
-			f, err := os.Open(filepath.Join(this.opts.DataDirectory, m.FileHash))
+			f, err := this.ReadFile(ctx, m.FileHash)
 			if err != nil {
 				log.Printf("Opening file '%s' for preview of album '%s': %s", m.FileHash, fileID, err.Error())
 				http.Error(w, "Internal error", 500)

storage.go

@@ -0,0 +1,57 @@
+package contented
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/minio/minio-go/v7"
+)
+
+func (this *Server) ReadFile(ctx context.Context, fileHash string) (io.ReadSeekCloser, error) {
+	if this.opts.StorageType == STORAGE_LOCAL {
+		fh, err := os.Open(filepath.Join(this.opts.DataDirectory, fileHash))
+		return fh, err
+
+	} else if this.opts.StorageType == STORAGE_S3 {
+		obj, err := this.s3client.GetObject(ctx, this.opts.DataS3Options.Bucket, this.opts.DataS3Options.Prefix+fileHash, minio.GetObjectOptions{})
+		if err != nil {
+			return nil, err
+		}
+
+		return obj, nil
+
+	} else {
+		panic("bad StorageType")
+	}
+}
+
+func (this *Server) SaveFile(ctx context.Context, fileHash string, srcLen int64, src io.Reader) error {
+	if this.opts.StorageType == STORAGE_LOCAL {
+
+		// Save file to disk
+		dest, err := os.OpenFile(filepath.Join(this.opts.DataDirectory, fileHash), os.O_CREATE|os.O_WRONLY, this.opts.FileMode())
+		if err != nil {
+			if os.IsExist(err) {
+				return nil // hash matches existing upload
+			}
+
+			return err // Real error
+		}
+		defer dest.Close()
+
+		_, err = io.CopyN(dest, src, int64(srcLen))
+		if err != nil {
+			return err
+		}
+		return nil
+
+	} else if this.opts.StorageType == STORAGE_S3 {
+		_, err := this.s3client.PutObject(ctx, this.opts.DataS3Options.Bucket, this.opts.DataS3Options.Prefix+fileHash, src, srcLen, minio.PutObjectOptions{})
+		return err
+
+	} else {
+		panic("bad StorageType")
+	}
+}

thumb.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
+	"os"
 	"path/filepath"
 
 	"code.ivysaur.me/thumbnail"
@@ -98,7 +100,35 @@ func (this *Server) handleThumbInternal(ctx context.Context, w http.ResponseWrit
 		return errors.New("Don't want to thumbnail very large files, sorry")
 	}
 
-	filePath := filepath.Join(this.opts.DataDirectory, m.FileHash)
+	var filePath string
+
+	if this.opts.StorageType == STORAGE_LOCAL {
+		filePath = filepath.Join(this.opts.DataDirectory, m.FileHash)
+
+	} else if this.opts.StorageType == STORAGE_S3 {
+		// Need to temporarily download it for thumbnailing (slow and costs money)
+
+		destFh, err := os.CreateTemp("", "contented-thumbcache-*")
+		defer os.Remove(destFh.Name())
+
+		srcFh, err := this.ReadFile(ctx, m.FileHash)
+		if err != nil {
+			return err
+		}
+
+		_, err = io.CopyN(destFh, srcFh, m.FileSize)
+		srcFh.Close()
+		if err != nil {
+			return err
+		}
+
+		destFh.Seek(0, io.SeekStart)
+		filePath = destFh.Name()
+
+	} else {
+		panic("bad StorageType")
+	}
+
 	thumb, err := t.RenderFileAs(filePath, m.MimeType)
 	if err != nil {
 		return err

upload.go

@@ -1,6 +1,7 @@
 package contented
 
 import (
+	"context"
 	"crypto/sha512"
 	"encoding/hex"
 	"encoding/json"
@@ -9,9 +10,7 @@ import (
 	"mime"
 	"mime/multipart"
 	"net/http"
-	"os"
 	"path"
-	"path/filepath"
 	"strings"
 	"time"
 )
@@ -94,27 +93,14 @@ func (this *Server) handleUploadFile(src multipart.File, hdr *multipart.FileHead
 		return "", err
 	}
 
-	// Save file to disk
 	fileHash := hex.EncodeToString(hasher.Sum(nil))
-	dest, err := os.OpenFile(filepath.Join(this.opts.DataDirectory, fileHash), os.O_CREATE|os.O_WRONLY, this.opts.FileMode())
+
-	shouldSave := true
+	// Save file to disk/s3
-	if err != nil && os.IsExist(err) {
+	err = this.SaveFile(context.Background(), fileHash, srcLen, src)
-		// hash matches existing upload
+	if err != nil {
-		// That's fine - but still persist the metadata separately
-		shouldSave = false
-	} else if err != nil {
 		return "", err
 	}
 
-	if shouldSave {
-		defer dest.Close()
-
-		_, err = io.CopyN(dest, src, int64(srcLen))
-		if err != nil {
-			return "", err
-		}
-	}
-
 	// Determine mime type
 	ctype := hdr.Header.Get("Content-Type")
 	if ctype == "" {