From 36b4b124f7f4ab08e861d270fd792b4645eefcc7 Mon Sep 17 00:00:00 2001 From: mappu Date: Fri, 19 May 2023 19:13:55 +1200 Subject: [PATCH] s3 backend support --- Server.go | 30 ++++++++++++++++++++++- cmd/contented/main.go | 29 +++++++++++++++++++--- download.go | 16 +++++++++--- preview.go | 3 +-- storage.go | 57 +++++++++++++++++++++++++++++++++++++++++++ thumb.go | 32 +++++++++++++++++++++++- upload.go | 24 ++++-------------- 7 files changed, 162 insertions(+), 29 deletions(-) create mode 100644 storage.go diff --git a/Server.go b/Server.go index 762eda3..1c72d9b 100644 --- a/Server.go +++ b/Server.go @@ -3,6 +3,7 @@ package contented import ( "embed" "encoding/json" + "fmt" "io/fs" "log" "net/http" @@ -11,6 +12,8 @@ import ( "strings" "time" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/mxk/go-flowrate/flowrate" bolt "go.etcd.io/bbolt" ) @@ -26,6 +29,8 @@ const ( ALBUM_MIMETYPE = `contented/album` + STORAGE_LOCAL int = 0 + STORAGE_S3 int = 1 ) type ServerPublicProperties struct { @@ -35,7 +40,16 @@ type ServerPublicProperties struct { } type ServerOptions struct { - DataDirectory string + StorageType int // STORAGE_xx + DataDirectory string + DataS3Options struct { + Hostname string + AccessKey string + SecretKey string + Bucket string + Prefix string + } + DBPath string DiskFilesWorldReadable bool BandwidthLimit int64 @@ -57,6 +71,7 @@ func (this *ServerOptions) FileMode() os.FileMode { type Server struct { opts ServerOptions + s3client *minio.Client db *bolt.DB startTime time.Time thumbnailSem chan struct{} @@ -83,6 +98,19 @@ func NewServer(opts *ServerOptions) (*Server, error) { s.staticDir, _ = fs.Sub(staticAssets, `static`) // can't fail + // Maybe open s3 connection + if s.opts.StorageType == STORAGE_S3 { + cl, err := minio.New(opts.DataS3Options.Hostname, &minio.Options{ + Creds: credentials.NewStaticV4(opts.DataS3Options.AccessKey, opts.DataS3Options.SecretKey, ""), + Secure: true, + }) + if err != nil { + return nil, fmt.Errorf("Connecting to S3 host: %w", err) + } + + s.s3client = cl + } + // "fill" the thumbnailer semaphore s.thumbnailSem = make(chan struct{}, s.opts.MaxConcurrentThumbs) for i := 0; i < s.opts.MaxConcurrentThumbs; i += 1 { diff --git a/cmd/contented/main.go b/cmd/contented/main.go index 45b9d3f..ea87143 100644 --- a/cmd/contented/main.go +++ b/cmd/contented/main.go @@ -24,12 +24,16 @@ func main() { enableUpload = flag.Bool("enableUpload", true, "Enable uploads (disable for read-only mode)") diskFilesWorldReadable = flag.Bool("diskFilesWorldReadable", false, "Save files as 0644 instead of 0600") maxConcurrentThumbs = flag.Int("concurrentthumbs", contented.DEFAULT_MAX_CONCURRENT_THUMBS, "Simultaneous thumbnail generation") + s3Host = flag.String("s3hostname", "", "S3 Server hostname") + s3AccessKey = flag.String("s3access", "", "S3 Access key") + s3SecretKey = flag.String("s3secret", "", "S3 Secret key") + s3Bucket = flag.String("s3bucket", "", "S3 Bucket") + s3Prefix = flag.String("s3prefix", "", "S3 object prefix") ) flag.Parse() - svr, err := contented.NewServer(&contented.ServerOptions{ - DataDirectory: *dataDir, + opts := contented.ServerOptions{ DBPath: *dbPath, BandwidthLimit: int64(*maxUploadSpeed), TrustXForwardedFor: *trustXForwardedFor, @@ -41,7 +45,26 @@ func main() { AppTitle: *appTitle, MaxUploadBytes: int64(*maxUploadMb) * 1024 * 1024, }, - }) + } + + if len(*s3AccessKey) > 0 { + opts.StorageType = contented.STORAGE_S3 + opts.DataS3Options.Hostname = *s3Host + opts.DataS3Options.AccessKey = *s3AccessKey + opts.DataS3Options.SecretKey = *s3SecretKey + opts.DataS3Options.Bucket = *s3Bucket + opts.DataS3Options.Prefix = *s3Prefix + + } else if len(*dataDir) > 0 { + opts.StorageType = contented.STORAGE_LOCAL + opts.DataDirectory = *dataDir + + } else { + log.Println("Please specify either the -data or -s3__ options.") + os.Exit(1) + } + + svr, err := contented.NewServer(&opts) if err != nil { log.Println(err.Error()) os.Exit(1) diff --git a/download.go b/download.go index 3c09b2e..750c46c 100644 --- a/download.go +++ b/download.go @@ -4,7 +4,6 @@ import ( "log" "net/http" "os" - "path/filepath" ) func (this *Server) handleView(w http.ResponseWriter, r *http.Request, fileID string) { @@ -28,7 +27,7 @@ func (this *Server) handleViewInternal(w http.ResponseWriter, r *http.Request, f } // Load file - f, err := os.Open(filepath.Join(this.opts.DataDirectory, m.FileHash)) + f, err := this.ReadFile(r.Context(), m.FileHash) if err != nil { return err } @@ -50,7 +49,18 @@ func (this *Server) handleViewInternal(w http.ResponseWriter, r *http.Request, f w.Header().Set(`Content-Type`, m.MimeType) } - http.ServeContent(w, r, "", m.UploadTime, f) + /* + if _, ok := f.(io.ReadSeeker); ! ok { + // Stream directly, no support for bytes/etag + w.Header().Set(`Content-Length`, fmt.Sprintf("%d", m.FileSize)) + _, err := io.Copy(w, f) + return err + } + */ + + // Allow range requests, if-modified-since, and so on + http.ServeContent(w, r, "", m.UploadTime, f) return nil + } diff --git a/preview.go b/preview.go index 4de5862..9fbcf3b 100644 --- a/preview.go +++ b/preview.go @@ -8,7 +8,6 @@ import ( "log" "net/http" "os" - "path/filepath" "strings" "time" ) @@ -121,7 +120,7 @@ html, body { if m.MimeType == ALBUM_MIMETYPE { // Special handling for albums - f, err := os.Open(filepath.Join(this.opts.DataDirectory, m.FileHash)) + f, err := this.ReadFile(ctx, m.FileHash) if err != nil { log.Printf("Opening file '%s' for preview of album '%s': %s", m.FileHash, fileID, err.Error()) http.Error(w, "Internal error", 500) diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..1c16896 --- /dev/null +++ b/storage.go @@ -0,0 +1,57 @@ +package contented + +import ( + "context" + "io" + "os" + "path/filepath" + + "github.com/minio/minio-go/v7" +) + +func (this *Server) ReadFile(ctx context.Context, fileHash string) (io.ReadSeekCloser, error) { + if this.opts.StorageType == STORAGE_LOCAL { + fh, err := os.Open(filepath.Join(this.opts.DataDirectory, fileHash)) + return fh, err + + } else if this.opts.StorageType == STORAGE_S3 { + obj, err := this.s3client.GetObject(ctx, this.opts.DataS3Options.Bucket, this.opts.DataS3Options.Prefix+fileHash, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + + return obj, nil + + } else { + panic("bad StorageType") + } +} + +func (this *Server) SaveFile(ctx context.Context, fileHash string, srcLen int64, src io.Reader) error { + if this.opts.StorageType == STORAGE_LOCAL { + + // Save file to disk + dest, err := os.OpenFile(filepath.Join(this.opts.DataDirectory, fileHash), os.O_CREATE|os.O_WRONLY, this.opts.FileMode()) + if err != nil { + if os.IsExist(err) { + return nil // hash matches existing upload + } + + return err // Real error + } + defer dest.Close() + + _, err = io.CopyN(dest, src, int64(srcLen)) + if err != nil { + return err + } + return nil + + } else if this.opts.StorageType == STORAGE_S3 { + _, err := this.s3client.PutObject(ctx, this.opts.DataS3Options.Bucket, this.opts.DataS3Options.Prefix+fileHash, src, srcLen, minio.PutObjectOptions{}) + return err + + } else { + panic("bad StorageType") + } +} diff --git a/thumb.go b/thumb.go index 20a01d6..fbe9203 100644 --- a/thumb.go +++ b/thumb.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "io" "log" "net/http" + "os" "path/filepath" "code.ivysaur.me/thumbnail" @@ -98,7 +100,35 @@ func (this *Server) handleThumbInternal(ctx context.Context, w http.ResponseWrit return errors.New("Don't want to thumbnail very large files, sorry") } - filePath := filepath.Join(this.opts.DataDirectory, m.FileHash) + var filePath string + + if this.opts.StorageType == STORAGE_LOCAL { + filePath = filepath.Join(this.opts.DataDirectory, m.FileHash) + + } else if this.opts.StorageType == STORAGE_S3 { + // Need to temporarily download it for thumbnailing (slow and costs money) + + destFh, err := os.CreateTemp("", "contented-thumbcache-*") + defer os.Remove(destFh.Name()) + + srcFh, err := this.ReadFile(ctx, m.FileHash) + if err != nil { + return err + } + + _, err = io.CopyN(destFh, srcFh, m.FileSize) + srcFh.Close() + if err != nil { + return err + } + + destFh.Seek(0, io.SeekStart) + filePath = destFh.Name() + + } else { + panic("bad StorageType") + } + thumb, err := t.RenderFileAs(filePath, m.MimeType) if err != nil { return err diff --git a/upload.go b/upload.go index 9bf9279..b1a6fa0 100644 --- a/upload.go +++ b/upload.go @@ -1,6 +1,7 @@ package contented import ( + "context" "crypto/sha512" "encoding/hex" "encoding/json" @@ -9,9 +10,7 @@ import ( "mime" "mime/multipart" "net/http" - "os" "path" - "path/filepath" "strings" "time" ) @@ -94,27 +93,14 @@ func (this *Server) handleUploadFile(src multipart.File, hdr *multipart.FileHead return "", err } - // Save file to disk fileHash := hex.EncodeToString(hasher.Sum(nil)) - dest, err := os.OpenFile(filepath.Join(this.opts.DataDirectory, fileHash), os.O_CREATE|os.O_WRONLY, this.opts.FileMode()) - shouldSave := true - if err != nil && os.IsExist(err) { - // hash matches existing upload - // That's fine - but still persist the metadata separately - shouldSave = false - } else if err != nil { + + // Save file to disk/s3 + err = this.SaveFile(context.Background(), fileHash, srcLen, src) + if err != nil { return "", err } - if shouldSave { - defer dest.Close() - - _, err = io.CopyN(dest, src, int64(srcLen)) - if err != nil { - return "", err - } - } - // Determine mime type ctype := hdr.Header.Get("Content-Type") if ctype == "" {