contented: initial work on file upload support

This commit is contained in:
mappu 2018-06-06 19:35:54 +12:00
parent 2668a6abbd
commit 9ceef928f9
3 changed files with 162 additions and 0 deletions

View File

@ -12,6 +12,8 @@ type NTFConfig struct {
HubDescription string HubDescription string
HubIgnoreNicks []string // Nicknames of Hub-Security/bots to exclude (e.g. "PtokaX"). HubIgnoreNicks []string // Nicknames of Hub-Security/bots to exclude (e.g. "PtokaX").
HubNickMinChars int HubNickMinChars int
ContentedURL string
ContentedMaxMB int
BotAPIKey string BotAPIKey string
GroupChatID int64 GroupChatID int64

View File

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"html" "html"
"log" "log"
"net/url"
"regexp" "regexp"
"sort" "sort"
"strings" "strings"
@ -32,6 +33,8 @@ type NTFServer struct {
conns map[string]*libnmdc.HubConnection // hubnick -> hubconn conns map[string]*libnmdc.HubConnection // hubnick -> hubconn
verbose bool verbose bool
contentedMaxBytes int64
// Except the coalesce buffer, that requires a background worker. // Except the coalesce buffer, that requires a background worker.
coalesceBufferMut sync.Mutex coalesceBufferMut sync.Mutex
coalesceBuffer map[string]time.Time coalesceBuffer map[string]time.Time
@ -65,6 +68,23 @@ func NewNTFServer(configFile string, verbose bool) (*NTFServer, error) {
ret.config = cfg ret.config = cfg
// Validate contented URL (if present)
if len(ret.config.ContentedURL) > 0 {
_, err := url.Parse(ret.config.ContentedURL)
if err != nil {
log.Printf("Ignoring malformed URL to contented server '%s': %s", ret.config.ContentedURL, err.Error())
ret.config.ContentedURL = "" // clear
}
// Valid. Enforce trailing slash
if !strings.HasSuffix(ret.config.ContentedURL, `/`) {
ret.config.ContentedURL += `/`
}
// Only set this if contented URL is valid - zero otherwise
ret.contentedMaxBytes = int64(ret.config.ContentedMaxMB) * 1024 * 1024
}
// Coalesce background worker // Coalesce background worker
go ret.coalesceWorker() go ret.coalesceWorker()

140
contented.go Normal file
View File

@ -0,0 +1,140 @@
package main
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"mime"
"mime/multipart"
"net/http"
telegram "github.com/go-telegram-bot-api/telegram-bot-api"
)
func (this *NTFServer) ContentedEnabled() bool {
return this.contentedMaxBytes > 0
}
func (this *NTFServer) ContentedUploadFromSync(main *telegram.File, thumbs *[]telegram.PhotoSize) (string, error) {
// If file fits under size limit, take it
if int64(main.FileSize) < this.contentedMaxBytes {
return this.ContentedUploadSync(main.FileID, int64(main.FileSize))
}
// Otherwise, we'll settle for the highest-res thumbnail we can take
if thumbs == nil || len(*thumbs) == 0 {
return "", errors.New("The file was too large for the image host server (and no smaller thumbnail is available)")
}
//
bestKnownIdx := -1
bestKnownMpx := int64(-1)
for idx, img := range *thumbs {
if int64(img.FileSize) > this.contentedMaxBytes {
continue // no good
}
// Highest total pixel count
mpx := int64(img.Width) * int64(img.Height)
if mpx > bestKnownMpx {
bestKnownIdx = idx
bestKnownMpx = mpx
}
}
if bestKnownIdx == -1 {
return "", errors.New("The file was too large for the image host server (and no smaller thumbnail is available)")
}
return this.ContentedUploadSync((*thumbs)[bestKnownIdx].FileID, int64((*thumbs)[bestKnownIdx].FileSize))
}
func (this *NTFServer) ContentedUploadSync(fileId string, expectSizeBytes int64) (string, error) {
// Prepare to download the file from telegram
url, err := this.bot.GetFileDirectURL(fileId)
if err != nil {
return "", fmt.Errorf("TelegramGetFileDirectURL: %s", err.Error())
}
if this.verbose {
log.Printf("Downloading telegram file %s", url)
}
resp, err := http.Get(url)
if err != nil {
return "", fmt.Errorf("HttpGet: %s", err.Error())
}
defer resp.Body.Close()
// Determine a suitable pseudo-filename from the telegram content-type
fileContentType := resp.Header.Get("Content-Type") // is it blank? who knows?
if this.verbose {
log.Printf("Downloaded telegram file response headers: %#v\n", resp.Header)
}
fileName := "telegram-upload"
if fileExts, err := mime.ExtensionsByType(fileContentType); err == nil && len(fileExts) > 0 {
fileName += fileExts[0] // Just pick the first one. They all contain a leading period
}
// Contented uploads are in multipart mime format
uploadBody := bytes.Buffer{}
formValues := multipart.NewWriter(&uploadBody)
filePart, err := formValues.CreateFormFile("f", fileName)
if err != nil {
return "", err
}
// Download
_, err = io.CopyN(filePart, resp.Body, expectSizeBytes) // CopyN asserts either err!=nil, or copiedBytes == expectSizeBytes. So we're safe
if err != nil {
return "", fmt.Errorf("CopyN: %s", err.Error())
}
// Upload
err = formValues.Close()
if err != nil {
return "", fmt.Errorf("Close: %s", err.Error())
}
req, err := http.NewRequest("POST", this.config.ContentedURL+"upload", &uploadBody)
if err != nil {
return "", err
}
req.Header.Set("Content-Type", fileContentType)
uploadResp, err := http.DefaultClient.Do(req)
if err != nil {
return "", fmt.Errorf("Upload failed: %s", err.Error())
}
defer uploadResp.Body.Close()
// Read back the response from contented
// Should be a 200 error code, JSON content type, array of strings of shortcodes to the uploaded file(s)
if uploadResp.StatusCode != 200 {
return "", fmt.Errorf("Unexpected HTTP %d response back from file hosting server", uploadResp.StatusCode)
}
shortCodes := make([]string, 0, 1)
err = json.NewDecoder(uploadResp.Body).Decode(&shortCodes)
if err != nil {
return "", fmt.Errorf("Decoding response from file hosting server: %s", err.Error())
}
if len(shortCodes) != 1 || len(shortCodes[0]) == 0 {
return "", errors.New("Unexpected blank response back from file hosting server")
}
// DONE
return this.config.ContentedURL + "p/" + shortCodes[0], nil
}