package tumblr import ( "encoding/json" "errors" "strings" "github.com/iawia002/annie/downloader" "github.com/iawia002/annie/extractors" "github.com/iawia002/annie/parser" "github.com/iawia002/annie/request" "github.com/iawia002/annie/utils" ) type imageList struct { List []string `json:"@list"` } type tumblrImageList struct { Image imageList `json:"image"` } type tumblrImage struct { Image string `json:"image"` } func genURLData(url, referer string) (downloader.URL, int64, error) { size, err := request.Size(url, referer) if err != nil { return downloader.URL{}, 0, err } _, ext, err := utils.GetNameAndExt(url) if err != nil { return downloader.URL{}, 0, err } data := downloader.URL{ URL: url, Size: size, Ext: ext, } return data, size, nil } func tumblrImageDownload(url, html, title string) ([]downloader.Data, error) { jsonStrings := utils.MatchOneOf( html, ``, ) if jsonStrings == nil || len(jsonStrings) < 2 { return nil, extractors.ErrURLParseFailed } jsonString := jsonStrings[1] var totalSize int64 var urls []downloader.URL if strings.Contains(jsonString, `"image":{"@list"`) { // there are two data structures in the same field(image) var imageList tumblrImageList if err := json.Unmarshal([]byte(jsonString), &imageList); err != nil { return nil, err } for _, u := range imageList.Image.List { urlData, size, err := genURLData(u, url) if err != nil { return nil, err } totalSize += size urls = append(urls, urlData) } } else { var image tumblrImage if err := json.Unmarshal([]byte(jsonString), &image); err != nil { return nil, err } urlData, size, err := genURLData(image.Image, url) if err != nil { return nil, err } totalSize = size urls = append(urls, urlData) } streams := map[string]downloader.Stream{ "default": { URLs: urls, Size: totalSize, }, } return []downloader.Data{ { Site: "Tumblr tumblr.com", Title: title, Type: "image", Streams: streams, URL: url, }, }, nil } func tumblrVideoDownload(url, html, title string) ([]downloader.Data, error) { videoURLs := utils.MatchOneOf(html, `