package archive import ( "bufio" "errors" "fmt" "html" "io/ioutil" "math" "net/http" "net/url" "os" "regexp" "sort" "strings" "time" ) const ( pageNotSet int = -1 ) type ArchiveState struct { svr *ArchiveServer log *LogSource logBestSlug string query string queryIsRegex bool isStats bool ym YearMonth page int highestPage int } func NewArchiveState(svr *ArchiveServer) *ArchiveState { return &ArchiveState{ svr: svr, page: pageNotSet, } } func (this *ArchiveState) showPageURLs() bool { return this.log != nil && len(this.query) == 0 && !this.isStats } func (this *ArchiveState) URL() string { if this.isStats { return fmt.Sprintf(`/%s/stats`, this.logBestSlug) } else if len(this.query) > 0 { if this.queryIsRegex { return fmt.Sprintf(`/%s/rx/%s`, this.logBestSlug, url.QueryEscape(this.query)) } else { return fmt.Sprintf(`/%s/search/%s`, this.logBestSlug, url.QueryEscape(this.query)) } } else { if this.page == pageNotSet { return fmt.Sprintf(`/%s/%s/%s`, this.logBestSlug, this.ym.Year, this.ym.Month) } else { return fmt.Sprintf(`/%s/%s/%s/page-%d`, this.logBestSlug, this.ym.Year, this.ym.Month, this.page) } } } func (this *ArchiveState) selectSource(log *LogSource, slug string) { this.log = log this.logBestSlug = slug } // renderView renders a single page of log entries. // - Mandatory: log, ym // - Optional: page func (this *ArchiveState) renderView(w http.ResponseWriter) { fname, err := this.svr.LogFile(this.log, this.ym) if err != nil { this.renderError(w, err.Error()) return } fc, err := ioutil.ReadFile(fname) if err != nil { this.renderError(w, err.Error()) return } lines := strings.Split(string(fc), "\n") // Work around #newpage: if the last line is blank, skip if len(lines) > 0 && len(lines[len(lines)-1]) == 0 { lines = lines[:len(lines)-1] } this.highestPage = int(math.Ceil(float64(len(lines))/float64(this.svr.cfg.LinesPerPage))) - 1 if this.page == pageNotSet || this.page > this.highestPage { this.page = this.highestPage } if this.page < 0 { this.page = 0 } startLine := this.svr.cfg.LinesPerPage * this.page endLine := this.svr.cfg.LinesPerPage * (this.page + 1) if endLine > len(lines) { endLine = len(lines) } output := "" for i := startLine; i < endLine; i += 1 { output += html.EscapeString(lines[i]) + "\n" } this.renderTemplate(w, []byte(output)) } // parseStatsFor loads user post statistics for a single yearMonth in a single log source. func (this *ArchiveState) parseStatsFor(ls *LogSource, ym YearMonth, into map[string]int) error { fname, err := this.svr.LogFile(ls, ym) if err != nil { return err } fc, err := ioutil.ReadFile(fname) if err != nil { return err } rxUser := regexp.MustCompile(`(?ms)^[^<\r\n]*<([^>\r\n]+)>.+?$`) matches := rxUser.FindAllSubmatch(fc, -1) if matches == nil || len(matches) == 0 { return errors.New("No matches") } for _, match := range matches { username := string(match[1]) if ct, ok := into[username]; ok { into[username] = ct + 1 } else { into[username] = 1 } } return nil } func (this *ArchiveState) renderStats(w http.ResponseWriter) { // Lines per year // Users / posts/year startTime := time.Now() yearsToUsersToPostCount := make(map[int]map[string]int, 0) totalErrors := 0 var lastError error = nil ym := this.log.EarliestDate() orderedYears := make([]int, 0) for { usersToPostCount, ok := yearsToUsersToPostCount[ym.Year] if !ok { usersToPostCount = make(map[string]int) orderedYears = append(orderedYears, ym.Year) } err := this.parseStatsFor(this.log, ym, usersToPostCount) if err != nil { //log.Printf("Stats(%s): %s", this.logBestSlug, err.Error()) totalErrors += 1 lastError = err } //fmt.Printf("%#v\n", usersToPostCount) yearsToUsersToPostCount[ym.Year] = usersToPostCount if ym.Equals(this.log.LatestDate()) { break } ym = ym.Next() } ret := make([]byte, 0) if lastError != nil { ret = append(ret, []byte(fmt.Sprintf("Got %d errors, including: '%s'\n\n", totalErrors, lastError.Error()))...) } // allUsersExistence := make(map[string]struct{}) for _, usersMap := range yearsToUsersToPostCount { for username, _ := range usersMap { allUsersExistence[username] = struct{}{} } } allUsernames := make([]string, 0, len(allUsersExistence)) for username, _ := range allUsersExistence { allUsernames = append(allUsernames, username) } sort.Strings(allUsernames) // ret = append(ret, []byte(``)...) for _, year := range orderedYears { ret = append(ret, []byte(fmt.Sprintf(``, year))...) } ret = append(ret, []byte("\n")...) for _, username := range allUsernames { ret = append(ret, []byte(fmt.Sprintf(``, html.EscapeString(username)))...) for _, year := range orderedYears { usersMap := yearsToUsersToPostCount[year] posts, _ /*ok*/ := usersMap[username] ret = append(ret, []byte(fmt.Sprintf(``, posts))...) } ret = append(ret, []byte("\n")...) } ret = append(ret, []byte(``)...) for _, year := range orderedYears { postsTotalForYear := 0 for _, userPostCount := range yearsToUsersToPostCount[year] { postsTotalForYear += userPostCount } ret = append(ret, []byte(fmt.Sprintf(``, postsTotalForYear))...) } duration := time.Now().Sub(startTime) ret = append(ret, []byte(fmt.Sprintf("
 %d
%s%d
TOTAL:%d
\n\n%d total users\nStatistics generated in %s", len(allUsernames), duration.String()))...) this.renderTemplate(w, ret) } // renderSearch renders the search results. // - Mandatory: log, query, queryIsRegex func (this *ArchiveState) renderSearch(w http.ResponseWriter) { var matcher func(line string) bool if this.queryIsRegex { rx, err := regexp.Compile(`(?i)` + this.query) if err != nil { this.renderError(w, "Invalid regular expression "+this.query+" ("+err.Error()+")") return } matcher = rx.MatchString } else { queryLower := strings.ToLower(this.query) matcher = func(line string) bool { return strings.Contains(strings.ToLower(line), queryLower) } } this.renderTemplateHead(w) totalResults := 0 w.Write([]byte(``)) if totalResults == 0 { w.Write([]byte(`No search results for "` + html.EscapeString(this.query) + `"`)) } else { w.Write([]byte(`
Found ` + fmt.Sprintf("%d", totalResults) + ` total result(s).

`)) } this.renderTemplateFoot(w) } // renderError renders a plain text string, escaping it for HTML use. func (this *ArchiveState) renderError(w http.ResponseWriter, msg string) { this.renderTemplate(w, []byte(html.EscapeString(msg))) } func (this *ArchiveState) renderTemplate(w http.ResponseWriter, body []byte) { this.renderTemplateHead(w) w.Write(body) this.renderTemplateFoot(w) } func (this *ArchiveState) renderTemplateHead(w http.ResponseWriter) { w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`) w.Header().Set(`X-UA-Compatible`, `IE=Edge`) w.WriteHeader(200) title := `Archives` if this.log != nil { title = this.log.Description + ` Archives` } latestUrl := `/` if this.log != nil { latestUrl = fmt.Sprintf(`/%s/%d/%d`, url.PathEscape(this.logBestSlug), this.log.LatestDate().Year, this.log.LatestDate().Month) } statsUrl := fmt.Sprintf(`/%s/stats`, url.PathEscape(this.logBestSlug)) w.Write([]byte(` ` + html.EscapeString(title) + `
`, )) // Header ends } func (this *ArchiveState) renderTemplateFoot(w http.ResponseWriter) { w.Write([]byte(`
`)) }