From 9ceef928f96caf4745412af81ff9d24fe1be9c09 Mon Sep 17 00:00:00 2001 From: mappu Date: Wed, 6 Jun 2018 19:35:54 +1200 Subject: [PATCH] contented: initial work on file upload support --- NTFConfig.go | 2 + NTFServer.go | 20 ++++++++ contented.go | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 contented.go diff --git a/NTFConfig.go b/NTFConfig.go index c5a84d4..3a44a32 100644 --- a/NTFConfig.go +++ b/NTFConfig.go @@ -12,6 +12,8 @@ type NTFConfig struct { HubDescription string HubIgnoreNicks []string // Nicknames of Hub-Security/bots to exclude (e.g. "PtokaX"). HubNickMinChars int + ContentedURL string + ContentedMaxMB int BotAPIKey string GroupChatID int64 diff --git a/NTFServer.go b/NTFServer.go index dc730a7..8344204 100644 --- a/NTFServer.go +++ b/NTFServer.go @@ -5,6 +5,7 @@ import ( "fmt" "html" "log" + "net/url" "regexp" "sort" "strings" @@ -32,6 +33,8 @@ type NTFServer struct { conns map[string]*libnmdc.HubConnection // hubnick -> hubconn verbose bool + contentedMaxBytes int64 + // Except the coalesce buffer, that requires a background worker. coalesceBufferMut sync.Mutex coalesceBuffer map[string]time.Time @@ -65,6 +68,23 @@ func NewNTFServer(configFile string, verbose bool) (*NTFServer, error) { ret.config = cfg + // Validate contented URL (if present) + if len(ret.config.ContentedURL) > 0 { + _, err := url.Parse(ret.config.ContentedURL) + if err != nil { + log.Printf("Ignoring malformed URL to contented server '%s': %s", ret.config.ContentedURL, err.Error()) + ret.config.ContentedURL = "" // clear + } + + // Valid. Enforce trailing slash + if !strings.HasSuffix(ret.config.ContentedURL, `/`) { + ret.config.ContentedURL += `/` + } + + // Only set this if contented URL is valid - zero otherwise + ret.contentedMaxBytes = int64(ret.config.ContentedMaxMB) * 1024 * 1024 + } + // Coalesce background worker go ret.coalesceWorker() diff --git a/contented.go b/contented.go new file mode 100644 index 0000000..4da198f --- /dev/null +++ b/contented.go @@ -0,0 +1,140 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "mime" + "mime/multipart" + "net/http" + + telegram "github.com/go-telegram-bot-api/telegram-bot-api" +) + +func (this *NTFServer) ContentedEnabled() bool { + return this.contentedMaxBytes > 0 +} + +func (this *NTFServer) ContentedUploadFromSync(main *telegram.File, thumbs *[]telegram.PhotoSize) (string, error) { + + // If file fits under size limit, take it + if int64(main.FileSize) < this.contentedMaxBytes { + return this.ContentedUploadSync(main.FileID, int64(main.FileSize)) + } + + // Otherwise, we'll settle for the highest-res thumbnail we can take + + if thumbs == nil || len(*thumbs) == 0 { + return "", errors.New("The file was too large for the image host server (and no smaller thumbnail is available)") + } + + // + + bestKnownIdx := -1 + bestKnownMpx := int64(-1) + for idx, img := range *thumbs { + if int64(img.FileSize) > this.contentedMaxBytes { + continue // no good + } + + // Highest total pixel count + mpx := int64(img.Width) * int64(img.Height) + if mpx > bestKnownMpx { + bestKnownIdx = idx + bestKnownMpx = mpx + } + } + + if bestKnownIdx == -1 { + return "", errors.New("The file was too large for the image host server (and no smaller thumbnail is available)") + } + + return this.ContentedUploadSync((*thumbs)[bestKnownIdx].FileID, int64((*thumbs)[bestKnownIdx].FileSize)) +} + +func (this *NTFServer) ContentedUploadSync(fileId string, expectSizeBytes int64) (string, error) { + + // Prepare to download the file from telegram + url, err := this.bot.GetFileDirectURL(fileId) + if err != nil { + return "", fmt.Errorf("TelegramGetFileDirectURL: %s", err.Error()) + } + + if this.verbose { + log.Printf("Downloading telegram file %s", url) + } + + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("HttpGet: %s", err.Error()) + } + defer resp.Body.Close() + + // Determine a suitable pseudo-filename from the telegram content-type + + fileContentType := resp.Header.Get("Content-Type") // is it blank? who knows? + + if this.verbose { + log.Printf("Downloaded telegram file response headers: %#v\n", resp.Header) + } + + fileName := "telegram-upload" + if fileExts, err := mime.ExtensionsByType(fileContentType); err == nil && len(fileExts) > 0 { + fileName += fileExts[0] // Just pick the first one. They all contain a leading period + } + + // Contented uploads are in multipart mime format + uploadBody := bytes.Buffer{} + formValues := multipart.NewWriter(&uploadBody) + + filePart, err := formValues.CreateFormFile("f", fileName) + if err != nil { + return "", err + } + + // Download + _, err = io.CopyN(filePart, resp.Body, expectSizeBytes) // CopyN asserts either err!=nil, or copiedBytes == expectSizeBytes. So we're safe + if err != nil { + return "", fmt.Errorf("CopyN: %s", err.Error()) + } + + // Upload + err = formValues.Close() + if err != nil { + return "", fmt.Errorf("Close: %s", err.Error()) + } + + req, err := http.NewRequest("POST", this.config.ContentedURL+"upload", &uploadBody) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", fileContentType) + uploadResp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("Upload failed: %s", err.Error()) + } + + defer uploadResp.Body.Close() + + // Read back the response from contented + // Should be a 200 error code, JSON content type, array of strings of shortcodes to the uploaded file(s) + if uploadResp.StatusCode != 200 { + return "", fmt.Errorf("Unexpected HTTP %d response back from file hosting server", uploadResp.StatusCode) + } + + shortCodes := make([]string, 0, 1) + err = json.NewDecoder(uploadResp.Body).Decode(&shortCodes) + if err != nil { + return "", fmt.Errorf("Decoding response from file hosting server: %s", err.Error()) + } + + if len(shortCodes) != 1 || len(shortCodes[0]) == 0 { + return "", errors.New("Unexpected blank response back from file hosting server") + } + + // DONE + return this.config.ContentedURL + "p/" + shortCodes[0], nil +}