code.ivysaur.me
/
contented
1
0
Fork 0
Code Issues 8 Pull Requests Releases 5 Wiki Activity

s3 backend support

This commit is contained in:
mappu 2023-05-19 19:13:55 +12:00
parent 8d40031edc
commit 36b4b124f7
7 changed files with 162 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Server.go
View File

@ -3,6 +3,7 @@ package contented
import (
"embed"
"encoding/json"
"fmt"
"io/fs"
"log"
"net/http"
@ -11,6 +12,8 @@ import (
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/mxk/go-flowrate/flowrate"
bolt "go.etcd.io/bbolt"
)
@ -26,6 +29,8 @@ const (
ALBUM_MIMETYPE = `contented/album`
STORAGE_LOCAL int = 0
STORAGE_S3 int = 1
)
type ServerPublicProperties struct {
@ -35,7 +40,16 @@ type ServerPublicProperties struct {
}
type ServerOptions struct {
DataDirectory string
StorageType int // STORAGE_xx
DataDirectory string
DataS3Options struct {
Hostname string
AccessKey string
SecretKey string
Bucket string
Prefix string
}
DBPath string
DiskFilesWorldReadable bool
BandwidthLimit int64
@ -57,6 +71,7 @@ func (this *ServerOptions) FileMode() os.FileMode {
type Server struct {
opts ServerOptions
s3client *minio.Client
db *bolt.DB
startTime time.Time
thumbnailSem chan struct{}
@ -83,6 +98,19 @@ func NewServer(opts *ServerOptions) (*Server, error) {
s.staticDir, _ = fs.Sub(staticAssets, `static`) // can't fail
// Maybe open s3 connection
if s.opts.StorageType == STORAGE_S3 {
cl, err := minio.New(opts.DataS3Options.Hostname, &minio.Options{
Creds: credentials.NewStaticV4(opts.DataS3Options.AccessKey, opts.DataS3Options.SecretKey, ""),
Secure: true,
})
if err != nil {
return nil, fmt.Errorf("Connecting to S3 host: %w", err)
}
s.s3client = cl
}
// "fill" the thumbnailer semaphore
s.thumbnailSem = make(chan struct{}, s.opts.MaxConcurrentThumbs)
for i := 0; i < s.opts.MaxConcurrentThumbs; i += 1 {

29
cmd/contented/main.go
View File

@ -24,12 +24,16 @@ func main() {
enableUpload = flag.Bool("enableUpload", true, "Enable uploads (disable for read-only mode)")
diskFilesWorldReadable = flag.Bool("diskFilesWorldReadable", false, "Save files as 0644 instead of 0600")
maxConcurrentThumbs = flag.Int("concurrentthumbs", contented.DEFAULT_MAX_CONCURRENT_THUMBS, "Simultaneous thumbnail generation")
s3Host = flag.String("s3hostname", "", "S3 Server hostname")
s3AccessKey = flag.String("s3access", "", "S3 Access key")
s3SecretKey = flag.String("s3secret", "", "S3 Secret key")
s3Bucket = flag.String("s3bucket", "", "S3 Bucket")
s3Prefix = flag.String("s3prefix", "", "S3 object prefix")
)
flag.Parse()
svr, err := contented.NewServer(&contented.ServerOptions{
DataDirectory: *dataDir,
opts := contented.ServerOptions{
DBPath: *dbPath,
BandwidthLimit: int64(*maxUploadSpeed),
TrustXForwardedFor: *trustXForwardedFor,
@ -41,7 +45,26 @@ func main() {
AppTitle: *appTitle,
MaxUploadBytes: int64(*maxUploadMb) * 1024 * 1024,
},
})
}
if len(*s3AccessKey) > 0 {
opts.StorageType = contented.STORAGE_S3
opts.DataS3Options.Hostname = *s3Host
opts.DataS3Options.AccessKey = *s3AccessKey
opts.DataS3Options.SecretKey = *s3SecretKey
opts.DataS3Options.Bucket = *s3Bucket
opts.DataS3Options.Prefix = *s3Prefix
} else if len(*dataDir) > 0 {
opts.StorageType = contented.STORAGE_LOCAL
opts.DataDirectory = *dataDir
} else {
log.Println("Please specify either the -data or -s3__ options.")
os.Exit(1)
}
svr, err := contented.NewServer(&opts)
if err != nil {
log.Println(err.Error())
os.Exit(1)

16
download.go
View File

@ -4,7 +4,6 @@ import (
"log"
"net/http"
"os"
"path/filepath"
)
func (this *Server) handleView(w http.ResponseWriter, r *http.Request, fileID string) {
@ -28,7 +27,7 @@ func (this *Server) handleViewInternal(w http.ResponseWriter, r *http.Request, f
}
// Load file
f, err := os.Open(filepath.Join(this.opts.DataDirectory, m.FileHash))
f, err := this.ReadFile(r.Context(), m.FileHash)
if err != nil {
return err
}
@ -50,7 +49,18 @@ func (this *Server) handleViewInternal(w http.ResponseWriter, r *http.Request, f
w.Header().Set(`Content-Type`, m.MimeType)
}
http.ServeContent(w, r, "", m.UploadTime, f)
/*
if _, ok := f.(io.ReadSeeker); ! ok {
// Stream directly, no support for bytes/etag
w.Header().Set(`Content-Length`, fmt.Sprintf("%d", m.FileSize))
_, err := io.Copy(w, f)
return err
}
*/
// Allow range requests, if-modified-since, and so on
http.ServeContent(w, r, "", m.UploadTime, f)
return nil
}

3
preview.go
View File

@ -8,7 +8,6 @@ import (
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
@ -121,7 +120,7 @@ html, body {
if m.MimeType == ALBUM_MIMETYPE {
// Special handling for albums
f, err := os.Open(filepath.Join(this.opts.DataDirectory, m.FileHash))
f, err := this.ReadFile(ctx, m.FileHash)
if err != nil {
log.Printf("Opening file '%s' for preview of album '%s': %s", m.FileHash, fileID, err.Error())
http.Error(w, "Internal error", 500)

57
storage.go Normal file
View File

@ -0,0 +1,57 @@
package contented
import (
"context"
"io"
"os"
"path/filepath"
"github.com/minio/minio-go/v7"
)
func (this *Server) ReadFile(ctx context.Context, fileHash string) (io.ReadSeekCloser, error) {
if this.opts.StorageType == STORAGE_LOCAL {
fh, err := os.Open(filepath.Join(this.opts.DataDirectory, fileHash))
return fh, err
} else if this.opts.StorageType == STORAGE_S3 {
obj, err := this.s3client.GetObject(ctx, this.opts.DataS3Options.Bucket, this.opts.DataS3Options.Prefix+fileHash, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
return obj, nil
} else {
panic("bad StorageType")
}
}
func (this *Server) SaveFile(ctx context.Context, fileHash string, srcLen int64, src io.Reader) error {
if this.opts.StorageType == STORAGE_LOCAL {
// Save file to disk
dest, err := os.OpenFile(filepath.Join(this.opts.DataDirectory, fileHash), os.O_CREATE|os.O_WRONLY, this.opts.FileMode())
if err != nil {
if os.IsExist(err) {
return nil // hash matches existing upload
}
return err // Real error
}
defer dest.Close()
_, err = io.CopyN(dest, src, int64(srcLen))
if err != nil {
return err
}
return nil
} else if this.opts.StorageType == STORAGE_S3 {
_, err := this.s3client.PutObject(ctx, this.opts.DataS3Options.Bucket, this.opts.DataS3Options.Prefix+fileHash, src, srcLen, minio.PutObjectOptions{})
return err
} else {
panic("bad StorageType")
}
}

32
thumb.go
View File

@ -4,8 +4,10 @@ import (
"context"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"code.ivysaur.me/thumbnail"
@ -98,7 +100,35 @@ func (this *Server) handleThumbInternal(ctx context.Context, w http.ResponseWrit
return errors.New("Don't want to thumbnail very large files, sorry")
}
filePath := filepath.Join(this.opts.DataDirectory, m.FileHash)
var filePath string
if this.opts.StorageType == STORAGE_LOCAL {
filePath = filepath.Join(this.opts.DataDirectory, m.FileHash)
} else if this.opts.StorageType == STORAGE_S3 {
// Need to temporarily download it for thumbnailing (slow and costs money)
destFh, err := os.CreateTemp("", "contented-thumbcache-*")
defer os.Remove(destFh.Name())
srcFh, err := this.ReadFile(ctx, m.FileHash)
if err != nil {
return err
}
_, err = io.CopyN(destFh, srcFh, m.FileSize)
srcFh.Close()
if err != nil {
return err
}
destFh.Seek(0, io.SeekStart)
filePath = destFh.Name()
} else {
panic("bad StorageType")
}
thumb, err := t.RenderFileAs(filePath, m.MimeType)
if err != nil {
return err

24
upload.go
View File

@ -1,6 +1,7 @@
package contented
import (
"context"
"crypto/sha512"
"encoding/hex"
"encoding/json"
@ -9,9 +10,7 @@ import (
"mime"
"mime/multipart"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"time"
)
@ -94,27 +93,14 @@ func (this *Server) handleUploadFile(src multipart.File, hdr *multipart.FileHead
return "", err
}
// Save file to disk
fileHash := hex.EncodeToString(hasher.Sum(nil))
dest, err := os.OpenFile(filepath.Join(this.opts.DataDirectory, fileHash), os.O_CREATE|os.O_WRONLY, this.opts.FileMode())
shouldSave := true
if err != nil && os.IsExist(err) {
// hash matches existing upload
// That's fine - but still persist the metadata separately
shouldSave = false
} else if err != nil {
// Save file to disk/s3
err = this.SaveFile(context.Background(), fileHash, srcLen, src)
if err != nil {
return "", err
}
if shouldSave {
defer dest.Close()
_, err = io.CopyN(dest, src, int64(srcLen))
if err != nil {
return "", err
}
}
// Determine mime type
ctype := hdr.Header.Get("Content-Type")
if ctype == "" {