package tangdou import ( "github.com/iawia002/annie/config" "github.com/iawia002/annie/downloader" "github.com/iawia002/annie/extractors" "github.com/iawia002/annie/request" "github.com/iawia002/annie/utils" ) const referer = "http://www.tangdou.com/html/playlist/view/4173" // Extract is the main function for extracting data func Extract(uri string) ([]downloader.Data, error) { if !config.Playlist { return []downloader.Data{tangdouDownload(uri)}, nil } html, err := request.Get(uri, referer, nil) if err != nil { return nil, err } videoIDs := utils.MatchAll(html, ``) needDownloadItems := utils.NeedDownloadList(len(videoIDs)) extractedData := make([]downloader.Data, len(needDownloadItems)) wgp := utils.NewWaitGroupPool(config.ThreadNumber) dataIndex := 0 for index, videoID := range videoIDs { if !utils.ItemInSlice(index+1, needDownloadItems) || len(videoID) < 2 { continue } wgp.Add() go func(index int, videURI string, extractedData []downloader.Data) { defer wgp.Done() extractedData[index] = tangdouDownload(videURI) }(dataIndex, videoID[1], extractedData) dataIndex++ } wgp.Wait() return extractedData, nil } // tangdouDownload download function for single url func tangdouDownload(uri string) downloader.Data { html, err := request.Get(uri, referer, nil) if err != nil { return downloader.EmptyData(uri, err) } titles := utils.MatchOneOf( html, `
(.+?)
`, `(.+?)`, ) if titles == nil || len(titles) < 2 { return downloader.EmptyData(uri, extractors.ErrURLParseFailed) } title := titles[1] var realURL string videoURLs := utils.MatchOneOf( html, `video:'(.+?)'`, `video:"(.+?)"`, `\s*