package tumblr
import (
"encoding/json"
"errors"
"strings"
"github.com/iawia002/annie/downloader"
"github.com/iawia002/annie/extractors"
"github.com/iawia002/annie/parser"
"github.com/iawia002/annie/request"
"github.com/iawia002/annie/utils"
)
type imageList struct {
List []string `json:"@list"`
}
type tumblrImageList struct {
Image imageList `json:"image"`
}
type tumblrImage struct {
Image string `json:"image"`
}
func genURLData(url, referer string) (downloader.URL, int64, error) {
size, err := request.Size(url, referer)
if err != nil {
return downloader.URL{}, 0, err
}
_, ext, err := utils.GetNameAndExt(url)
if err != nil {
return downloader.URL{}, 0, err
}
data := downloader.URL{
URL: url,
Size: size,
Ext: ext,
}
return data, size, nil
}
func tumblrImageDownload(url, html, title string) ([]downloader.Data, error) {
jsonStrings := utils.MatchOneOf(
html, ``,
)
if jsonStrings == nil || len(jsonStrings) < 2 {
return nil, extractors.ErrURLParseFailed
}
jsonString := jsonStrings[1]
var totalSize int64
var urls []downloader.URL
if strings.Contains(jsonString, `"image":{"@list"`) {
// there are two data structures in the same field(image)
var imageList tumblrImageList
if err := json.Unmarshal([]byte(jsonString), &imageList); err != nil {
return nil, err
}
for _, u := range imageList.Image.List {
urlData, size, err := genURLData(u, url)
if err != nil {
return nil, err
}
totalSize += size
urls = append(urls, urlData)
}
} else {
var image tumblrImage
if err := json.Unmarshal([]byte(jsonString), &image); err != nil {
return nil, err
}
urlData, size, err := genURLData(image.Image, url)
if err != nil {
return nil, err
}
totalSize = size
urls = append(urls, urlData)
}
streams := map[string]downloader.Stream{
"default": {
URLs: urls,
Size: totalSize,
},
}
return []downloader.Data{
{
Site: "Tumblr tumblr.com",
Title: title,
Type: "image",
Streams: streams,
URL: url,
},
}, nil
}
func tumblrVideoDownload(url, html, title string) ([]downloader.Data, error) {
videoURLs := utils.MatchOneOf(html, `