changelog for 1.3.0

Issues are now tracked at git.ivysaur.me.

.gitignore

@@ -1,5 +1,3 @@
-syntax: glob
-
 cmd/contented/contented
 build/
 _dist/

.hgtags

@@ -1,12 +0,0 @@
-e2250a7fd29052ea767f18e1459cabea4cd7efd3 v1.0.0
-b8975b9e75648a7c2a5003c67db92cf2216e01c0 v1.0.1
-c7b699105bd166e7a01aa0e678d34624680bf81e v1.1.0
-c7b699105bd166e7a01aa0e678d34624680bf81e v1.1.0
-0000000000000000000000000000000000000000 v1.1.0
-0000000000000000000000000000000000000000 v1.1.0
-cfb1e028fd0627614aa01184893f9f29f20a347e v1.1.0
-cfb1e028fd0627614aa01184893f9f29f20a347e v1.1.0
-0000000000000000000000000000000000000000 v1.1.0
-0000000000000000000000000000000000000000 v1.1.0
-98da2ebf0d50dffe8b625457a639bc2f15519714 v1.1.0
-0f021da52854175cd48084249c45b1d3fa8b58ca v1.2.0

Gopkg.lock

@@ -1,57 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  name = "code.ivysaur.me/imagequant"
-  packages = ["."]
-  revision = "6a468707fb1bb44c4bb71113273cc73daf401976"
-
-[[projects]]
-  branch = "master"
-  name = "code.ivysaur.me/thumbnail"
-  packages = ["."]
-  revision = "13e0990a33026cba5f746c13c85782e562e61fa6"
-
-[[projects]]
-  name = "github.com/boltdb/bolt"
-  packages = ["."]
-  revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8"
-  version = "v1.3.1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [".","simplelru"]
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mxk/go-flowrate"
-  packages = ["flowrate"]
-  revision = "cca7078d478f8520f85629ad7c68962d31ed7682"
-
-[[projects]]
-  name = "github.com/speps/go-hashids"
-  packages = ["."]
-  revision = "d1d57a886aa7e3ef6092b70ceab077e35ee8e0ce"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/image"
-  packages = ["bmp","draw","math/f64","riff","vp8","vp8l","webp"]
-  revision = "af66defab954cb421ca110193eed9477c8541e2a"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "9527bec2660bd847c050fda93a0f0c6dee0800bb"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "907370ab34b6a574b2845b17f9033acb75dfc5c8348266167f95f0112f38e89a"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,38 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-[[constraint]]
-  name = "code.ivysaur.me/thumbnail"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/boltdb/bolt"
-  version = "1.3.1"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mxk/go-flowrate"
-
-[[constraint]]
-  name = "github.com/speps/go-hashids"
-  version = "1.0.0"

Makefile

@@ -2,7 +2,7 @@
 # Makefile for contented
 #
 
-VERSION:=1.2.1
+VERSION:=1.2.2
 
 SOURCES:=Makefile \
 	static \
@@ -69,5 +69,5 @@ _dist/contented-$(VERSION)-win32.7z: build/win32/contented.exe
 	)
 	
 _dist/contented-$(VERSION)-src.zip: $(SOURCES)
-	 hg archive --type=zip _dist/contented-$(VERSION)-src.zip
+	git archive HEAD -o _dist/contented-$(VERSION)-src.zip
 	

README.md

@@ -1,15 +1,17 @@
-A file / image / paste upload server with a focus on embedding.
+# contented
 
-Written in Go
+[![](doc/image1.thumb.png)](doc/image1.png)
+
+A file / image / paste upload server with a focus on embedding.
 
 You can use contented as a standalone upload server, or you can use the SDK to embed its upload widget into another website.
 
-=FEATURES=
+## Features
 
 - Drag and drop upload
 - Multiple files upload
 - Pastebin upload
-- Custom drawing upload ([url=https://github.com/Leimi/drawingboard.js]via[/url])
+- Custom drawing upload ([via drawingboard.js](https://github.com/Leimi/drawingboard.js))
 - Ctrl-V upload
 - SDK-oriented design for embedding, including CORS support
 - Mobile friendly HTML interface
@@ -17,12 +19,13 @@ You can use contented as a standalone upload server, or you can use the SDK to e
 - Hash verification (SHA512/256)
 - Detect duplicate upload content and reuse storage
 - Options to limit the upload filesize and the upload bandwidth
-- Short URLs (using [url=http://hashids.org]Hashids[/url] algorithm)
+- Short URLs (using [Hashids](http://hashids.org) algorithm)
 - Image thumbnailing
 
-=USAGE (SERVER)=
+## Usage (Server)
 
-`Usage of contented:
+```
+Usage of contented:
   -data string
         Directory for stored content (default "")
   -db string
@@ -41,39 +44,55 @@ You can use contented as a standalone upload server, or you can use the SDK to e
         Title used in web interface (default "contented")
   -trustXForwardedFor
         Trust X-Forwarded-For reverse proxy headers
-`
+  -concurrentthumbs
+        Simultaneous thumbnail generation (default 16)
+```
 
 If you are hosting behind a reverse proxy, remember to set its post body size parameter appropriately (e.g. `client_max_body_size` for nginx).
 
-=USAGE (HTTP)=
+## Usage (HTTP)
 
 The server responds on the following URLs:
 
-- `/get/{ID}`: Download item content
-- `/info/{ID}`: Get item content metadata (JSON)
-- `/thumb/{Type}/{ID}`: Get item thumbnail image
-- `/about`: Get server metadata (JSON)
+URL                  |Method |Description
+---------------------|-------|---
+`/get/{ID}`          |`GET`  |Download item content
+`/info/{ID}`         |`GET`  |Get item content metadata (JSON)
+`/thumb/{Type}/{ID}` |`GET`  |Get item thumbnail image
+`/about`             |`GET`  |Get server metadata (JSON)
 
-=USAGE (EMBEDDING FOR WEB)=
+## Usage (Embedding for web)
 
 Your webpage should load the SDK from the contented server, then call the `contented.init` function to display the upload widget over the top of an existing DOM element. Your callback will be passed an array of file IDs of any uploaded items.
 
-`<script type="text/javascript" src="SERVER_ADDR/sdk.js"></script>
+```html
+<script type="text/javascript" src="SERVER_ADDR/sdk.js"></script>
 contented.init("#target", function(/* String[] */ items) {});
-`
+```
 
-=CHANGELOG=
+## Changelog
+
+2020-07-25: 1.3.0
+- Feature: Option to limit concurrent thumbnail generation
+- Enhancement: Set charset=UTF-8 when serving user-submitted text/plain content
+- Fix an issue with large memory usage for multipart file uploads
 
 2018-06-09: 1.2.1
 - Feature: Add OpenGraph tags on preview pages, for rich metadata in chat applications
 - Update thumbnailing library to improve quality
 - Use dep for vendoring
+- [⬇️ contented-1.2.1-win32.7z](https://git.ivysaur.me/attachments/88dea4f7-e314-4325-a957-096dcf8cdecc) *(1.51 MiB)*
+- [⬇️ contented-1.2.1-src.zip](https://git.ivysaur.me/attachments/6fd2b963-3be4-48a6-a5bf-6f273bcaea24) *(1.49 MiB)*
+- [⬇️ contented-1.2.1-linux64.tar.gz](https://git.ivysaur.me/attachments/c536f764-0250-4d67-886a-4797946e1124) *(2.21 MiB)*
 
 2017-11-18: 1.2.0
 - Feature: Thumbnail support
 - Feature: File preview page
 - Feature: Album mode (via URL `/p/{file1}-{file2}-...`)
 - Feature: New `-diskFilesWorldReadable` option to save files with `0644` mode
+- [⬇️ contented-1.2.0-win32.7z](https://git.ivysaur.me/attachments/f3453b62-b2a7-4e77-9b04-44c99dec35ba) *(1.36 MiB)*
+- [⬇️ contented-1.2.0-src.zip](https://git.ivysaur.me/attachments/a6c1ecfb-fd6a-44b5-9dc8-aea7c439d1e6) *(178.94 KiB)*
+- [⬇️ contented-1.2.0-linux64.tar.gz](https://git.ivysaur.me/attachments/6234754b-af17-4a72-8b66-56a5db21c7c7) *(2.03 MiB)*
 
 2017-10-15: 1.1.0
 - Feature: Drawing mode
@@ -87,11 +106,20 @@ contented.init("#target", function(/* String[] */ items) {});
 - Include drawingboard.js 0.4.6 (MIT license)
 - Fix a cosmetic issue with javascript console output
 - Fix a cosmetic issue with error messages if an upload failed
+- [⬇️ contented-1.1.0-win32.7z](https://git.ivysaur.me/attachments/bfb0a7fe-bf95-4d0e-933b-8137bc8071a4) *(1.11 MiB)*
+- [⬇️ contented-1.1.0-src.zip](https://git.ivysaur.me/attachments/67401341-724f-4ea2-b9c7-44d08ab9d38a) *(142.82 KiB)*
+- [⬇️ contented-1.1.0-linux64.tar.gz](https://git.ivysaur.me/attachments/a13752dd-5228-4830-b61d-0f7cc568b2ae) *(1.67 MiB)*
 
 2017-10-08: 1.0.1
 - Fix an issue with CORS preflight requests
 - Fix an issue with index URLs
+- [⬇️ contented-1.0.1-win32.7z](https://git.ivysaur.me/attachments/a873d510-da09-4797-95e9-ffcad690a77b) *(1.10 MiB)*
+- [⬇️ contented-1.0.1-src.zip](https://git.ivysaur.me/attachments/43ac17d6-b6f1-4da7-98e9-b8af6fb5551a) *(109.08 KiB)*
+- [⬇️ contented-1.0.1-linux64.tar.gz](https://git.ivysaur.me/attachments/34d74bed-db3f-4cef-a76f-266f0b9e6017) *(1.65 MiB)*
 
 2017-10-08: 1.0.0
 - Initial public release
 - Include jQuery 1.12.4 (MIT license)
+- [⬇️ contented-1.0.0-win32.7z](https://git.ivysaur.me/attachments/4ef132cf-dac8-4bcf-9da7-14ca1366e815) *(1.10 MiB)*
+- [⬇️ contented-1.0.0-src.zip](https://git.ivysaur.me/attachments/74d77b3f-557b-44bf-9645-7b3b25ab17c1) *(102.45 KiB)*
+- [⬇️ contented-1.0.0-linux64.tar.gz](https://git.ivysaur.me/attachments/1c28a913-686b-44cf-b63d-db22968a93b6) *(1.65 MiB)*

Server.go

@@ -16,6 +16,8 @@ import (
 
 var SERVER_HEADER string = `contented/0.0.0-dev`
 
+const DEFAULT_MAX_CONCURRENT_THUMBS = 16
+
 type ServerPublicProperties struct {
 	AppTitle         string
 	MaxUploadBytes   int64
@@ -29,6 +31,7 @@ type ServerOptions struct {
 	BandwidthLimit         int64
 	TrustXForwardedFor     bool
 	EnableHomepage         bool
+	MaxConcurrentThumbs    int
 	ServerPublicProperties
 }
 
@@ -44,6 +47,7 @@ type Server struct {
 	opts           ServerOptions
 	db             *bolt.DB
 	startTime      time.Time
+	thumbnailSem   chan struct{}
 	metadataBucket []byte
 }
 
@@ -54,6 +58,17 @@ func NewServer(opts *ServerOptions) (*Server, error) {
 		startTime:      time.Now(),
 	}
 
+	if s.opts.MaxConcurrentThumbs <= 0 {
+		s.opts.MaxConcurrentThumbs = DEFAULT_MAX_CONCURRENT_THUMBS // default
+		log.Printf("Allowing %d concurrent thumbnails", s.opts.MaxConcurrentThumbs)
+	}
+
+	// "fill" the thumbnailer semaphore
+	s.thumbnailSem = make(chan struct{}, s.opts.MaxConcurrentThumbs)
+	for i := 0; i < s.opts.MaxConcurrentThumbs; i += 1 {
+		s.thumbnailSem <- struct{}{}
+	}
+
 	b, err := bolt.Open(opts.DBPath, 0644, bolt.DefaultOptions)
 	if err != nil {
 		return nil, err

TODO.txt

@@ -1,22 +0,0 @@
-TODO
-
-- View server-wide recent uploads history / all upload history
-
-- Display 'my uploads' (id + metadata history kept in localstorage)
-
-- Encrypted at rest (anti- provider snooping)
-
-- Prevent selecting around the toggle buttons (firefox for android)
-
-- Detect when pasting creates one large file and one ~150 byte file(?? why does this happen?)
-
-- Detect when pasting an image URL, offer to download it (clientside?)
-
-- Add caching headers on thumbnails and on content endpoints
-
-- Add content-security headers on html pages
-
-- If file is larger than the upload limit, warn before the upload happens
-
-- If multiple files are larger than the upload limit, break into multiple uploads
-

cmd/contented/main.go

@@ -21,6 +21,7 @@ func main() {
 	trustXForwardedFor := flag.Bool("trustXForwardedFor", false, "Trust X-Forwarded-For reverse proxy headers")
 	enableHomepage := flag.Bool("enableHomepage", true, "Enable homepage (disable for embedded use only)")
 	diskFilesWorldReadable := flag.Bool("diskFilesWorldReadable", false, "Save files as 0644 instead of 0600")
+	maxConcurrentThumbs := flag.Int("concurrentthumbs", contented.DEFAULT_MAX_CONCURRENT_THUMBS, "Simultaneous thumbnail generation")
 
 	flag.Parse()
 
@@ -31,6 +32,7 @@ func main() {
 		TrustXForwardedFor:     *trustXForwardedFor,
 		EnableHomepage:         *enableHomepage,
 		DiskFilesWorldReadable: *diskFilesWorldReadable,
+		MaxConcurrentThumbs:    *maxConcurrentThumbs,
 		ServerPublicProperties: contented.ServerPublicProperties{
 			AppTitle:       *appTitle,
 			MaxUploadBytes: int64(*maxUploadMb) * 1024 * 1024,

download.go

@@ -37,9 +37,17 @@ func (this *Server) handleViewInternal(w http.ResponseWriter, r *http.Request, f
 
 	// ServeContent only uses the filename to get the mime type, which we can
 	// set accurately (including blacklist)
+
+	switch m.MimeType {
+	case `text/plain`:
+		w.Header().Set(`Content-Type`, `text/plain; charset=UTF-8`)
+
+	case `application/octet-stream`:
 		w.Header().Set(`Content-Type`, m.MimeType)
-	if m.MimeType == `application/octet-stream` {
 		w.Header().Set(`Content-Disposition`, `attachment; filename="`+m.Filename+`"`)
+
+	default:
+		w.Header().Set(`Content-Type`, m.MimeType)
 	}
 
 	http.ServeContent(w, r, "", m.UploadTime, f)

go.mod

@@ -0,0 +1,12 @@
+module code.ivysaur.me/contented
+
+require (
+	code.ivysaur.me/imagequant v2.12.2-go1.2+incompatible
+	code.ivysaur.me/thumbnail v1.0.1
+	github.com/boltdb/bolt v1.3.1
+	github.com/hashicorp/golang-lru v0.5.0
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f
+	github.com/speps/go-hashids v1.0.0
+	golang.org/x/image v0.0.0-20180601115456-af66defab954
+	golang.org/x/sys v0.0.0-20180606202747-9527bec2660b
+)

go.sum

@@ -0,0 +1,16 @@
+code.ivysaur.me/imagequant v0.0.0-20180609052806-6a468707fb1b/go.mod h1:1Pi+M0oJFDYLtGuCkPGPpb4OGCYudvp/SG6jdVcO+WU=
+code.ivysaur.me/imagequant v2.12.2-go1.2+incompatible/go.mod h1:1Pi+M0oJFDYLtGuCkPGPpb4OGCYudvp/SG6jdVcO+WU=
+code.ivysaur.me/thumbnail v1.0.1 h1:vBvueRPu9Ed+PuULNdn0pbfmtt6SVNHJ+8ezZBl8pIg=
+code.ivysaur.me/thumbnail v1.0.1/go.mod h1:oOapO2ddhCg0OBnAMFOW98mSKg6cRpWqCivVFcQnn5A=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/speps/go-hashids v1.0.0 h1:jdFC07PrExRM4Og5Ev4411Tox75aFpkC77NlmutadNI=
+github.com/speps/go-hashids v1.0.0/go.mod h1:P7hqPzMdnZOfyIk+xrlG1QaSMw+gCBdHKsBDnhpaZvc=
+golang.org/x/image v0.0.0-20180601115456-af66defab954 h1:n7UB+yxe5jyWxOA3BTAfwi23lhfKEIddaB/so7YOYe0=
+golang.org/x/image v0.0.0-20180601115456-af66defab954/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

thumb.go

@@ -1,6 +1,7 @@
 package contented
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"log"
@@ -51,6 +52,8 @@ func getThumbnailerConfig(t byte) (*thumbnail.Config, error) {
 }
 
 func (this *Server) handleThumb(w http.ResponseWriter, r *http.Request, thumbnailType byte, fileId string) {
+	ctx := r.Context()
+
 	opts, err := getThumbnailerConfig(thumbnailType)
 	if err != nil {
 		log.Printf("%s Thumbnail failed: %s\n", this.remoteIP(r), err.Error())
@@ -58,9 +61,18 @@ func (this *Server) handleThumb(w http.ResponseWriter, r *http.Request, thumbnai
 		return
 	}
 
+	// Only a limited number of thumbnails can be generated concurrently
+	<-this.thumbnailSem
+	defer func() { this.thumbnailSem <- struct{}{} }()
+
+	if ctx.Err() != nil {
+		// The request was already cancelled
+		return
+	}
+
 	t := thumbnail.NewThumbnailerEx(opts)
 
-	err = this.handleThumbInternal(w, t, fileId)
+	err = this.handleThumbInternal(ctx, w, t, fileId)
 	if err != nil {
 		log.Printf("%s Thumbnail failed: %s\n", this.remoteIP(r), err.Error())
 
@@ -69,7 +81,7 @@ func (this *Server) handleThumb(w http.ResponseWriter, r *http.Request, thumbnai
 	}
 }
 
-func (this *Server) handleThumbInternal(w http.ResponseWriter, t thumbnail.Thumbnailer, fileId string) error {
+func (this *Server) handleThumbInternal(ctx context.Context, w http.ResponseWriter, t thumbnail.Thumbnailer, fileId string) error {
 
 	// Load metadata
 	m, err := this.Metadata(fileId)

upload.go

@@ -20,7 +20,7 @@ func (this *Server) handleUpload(w http.ResponseWriter, r *http.Request) {
 
 	remoteIP := this.remoteIP(r)
 
-	err := r.ParseMultipartForm(this.opts.MaxUploadBytes * 2)
+	err := r.ParseMultipartForm(0) // buffer upload in temporary files on disk, not memory
 	if err != nil {
 		log.Printf("%s Invalid request: %s\n", remoteIP, err.Error())
 		http.Error(w, "Invalid request", 400)