statistics page

This commit is contained in:
mappu 2017-12-10 15:19:20 +13:00
parent 917b9c1a53
commit dea0cffc90
3 changed files with 153 additions and 5 deletions

View File

@ -18,7 +18,7 @@ type ArchiveServer struct {
cfg *Config cfg *Config
startup time.Time startup time.Time
rxViewRoot, rxViewPage, rxSearch, rxSearchRx *regexp.Regexp rxViewRoot, rxViewPage, rxSearch, rxSearchRx, rxStats *regexp.Regexp
} }
func NewArchiveServer(cfg *Config) (*ArchiveServer, error) { func NewArchiveServer(cfg *Config) (*ArchiveServer, error) {
@ -49,6 +49,7 @@ func NewArchiveServer(cfg *Config) (*ArchiveServer, error) {
rxViewPage: regexp.MustCompile(`^/([^/]+)/(\d+)/(\d+)/(?:page-)?(\d+)$`), rxViewPage: regexp.MustCompile(`^/([^/]+)/(\d+)/(\d+)/(?:page-)?(\d+)$`),
rxSearch: regexp.MustCompile(`^/([^/]+)/search/(.*)$`), rxSearch: regexp.MustCompile(`^/([^/]+)/search/(.*)$`),
rxSearchRx: regexp.MustCompile(`^/([^/]+)/rx/(.*)$`), rxSearchRx: regexp.MustCompile(`^/([^/]+)/rx/(.*)$`),
rxStats: regexp.MustCompile((`^/([^/]+)/stats/?$`)),
}, nil }, nil
} }

View File

@ -2,6 +2,7 @@ package archive
import ( import (
"bufio" "bufio"
"errors"
"fmt" "fmt"
"html" "html"
"io/ioutil" "io/ioutil"
@ -10,7 +11,9 @@ import (
"net/url" "net/url"
"os" "os"
"regexp" "regexp"
"sort"
"strings" "strings"
"time"
) )
const ( const (
@ -23,6 +26,7 @@ type ArchiveState struct {
logBestSlug string logBestSlug string
query string query string
queryIsRegex bool queryIsRegex bool
isStats bool
ym YearMonth ym YearMonth
page int page int
highestPage int highestPage int
@ -35,13 +39,21 @@ func NewArchiveState(svr *ArchiveServer) *ArchiveState {
} }
} }
func (this *ArchiveState) showPageURLs() bool {
return this.log != nil && len(this.query) == 0 && !this.isStats
}
func (this *ArchiveState) URL() string { func (this *ArchiveState) URL() string {
if len(this.query) > 0 { if this.isStats {
return fmt.Sprintf(`/%s/stats`, this.logBestSlug)
} else if len(this.query) > 0 {
if this.queryIsRegex { if this.queryIsRegex {
return fmt.Sprintf(`/%s/rx/%s`, this.logBestSlug, url.QueryEscape(this.query)) return fmt.Sprintf(`/%s/rx/%s`, this.logBestSlug, url.QueryEscape(this.query))
} else { } else {
return fmt.Sprintf(`/%s/search/%s`, this.logBestSlug, url.QueryEscape(this.query)) return fmt.Sprintf(`/%s/search/%s`, this.logBestSlug, url.QueryEscape(this.query))
} }
} else { } else {
if this.page == pageNotSet { if this.page == pageNotSet {
return fmt.Sprintf(`/%s/%s/%s`, this.logBestSlug, this.ym.Year, this.ym.Month) return fmt.Sprintf(`/%s/%s/%s`, this.logBestSlug, this.ym.Year, this.ym.Month)
@ -101,6 +113,130 @@ func (this *ArchiveState) renderView(w http.ResponseWriter) {
this.renderTemplate(w, []byte(output)) this.renderTemplate(w, []byte(output))
} }
// parseStatsFor loads user post statistics for a single yearMonth in a single log source.
func (this *ArchiveState) parseStatsFor(ls *LogSource, ym YearMonth, into map[string]int) error {
fname, err := this.svr.LogFile(ls, ym)
if err != nil {
return err
}
fc, err := ioutil.ReadFile(fname)
if err != nil {
return err
}
rxUser := regexp.MustCompile(`(?ms)^[^<\r\n]*<([^>\r\n]+)>.+?$`)
matches := rxUser.FindAllSubmatch(fc, -1)
if matches == nil || len(matches) == 0 {
return errors.New("No matches")
}
for _, match := range matches {
username := string(match[1])
if ct, ok := into[username]; ok {
into[username] = ct + 1
} else {
into[username] = 1
}
}
return nil
}
func (this *ArchiveState) renderStats(w http.ResponseWriter) {
// Lines per year
// Users / posts/year
startTime := time.Now()
yearsToUsersToPostCount := make(map[int]map[string]int, 0)
totalErrors := 0
var lastError error = nil
ym := this.log.EarliestDate()
orderedYears := make([]int, 0)
for {
usersToPostCount, ok := yearsToUsersToPostCount[ym.Year]
if !ok {
usersToPostCount = make(map[string]int)
orderedYears = append(orderedYears, ym.Year)
}
err := this.parseStatsFor(this.log, ym, usersToPostCount)
if err != nil {
//log.Printf("Stats(%s): %s", this.logBestSlug, err.Error())
totalErrors += 1
lastError = err
}
//fmt.Printf("%#v\n", usersToPostCount)
yearsToUsersToPostCount[ym.Year] = usersToPostCount
if ym.Equals(this.log.LatestDate()) {
break
}
ym = ym.Next()
}
ret := make([]byte, 0)
if lastError != nil {
ret = append(ret, []byte(fmt.Sprintf("<em>Got %d errors, including: '%s'</em>\n\n", totalErrors, lastError.Error()))...)
}
//
allUsersExistence := make(map[string]struct{})
for _, usersMap := range yearsToUsersToPostCount {
for username, _ := range usersMap {
allUsersExistence[username] = struct{}{}
}
}
allUsernames := make([]string, 0, len(allUsersExistence))
for username, _ := range allUsersExistence {
allUsernames = append(allUsernames, username)
}
sort.Strings(allUsernames)
//
ret = append(ret, []byte(`<table><thead><tr><th>&nbsp;</th>`)...)
for _, year := range orderedYears {
ret = append(ret, []byte(fmt.Sprintf(`<th>%d</th>`, year))...)
}
ret = append(ret, []byte("</tr><tbody>\n")...)
for _, username := range allUsernames {
ret = append(ret, []byte(fmt.Sprintf(`<tr><td>%s</td>`, html.EscapeString(username)))...)
for _, year := range orderedYears {
usersMap := yearsToUsersToPostCount[year]
posts, _ /*ok*/ := usersMap[username]
ret = append(ret, []byte(fmt.Sprintf(`<td>%d</td>`, posts))...)
}
ret = append(ret, []byte("</tr>\n")...)
}
ret = append(ret, []byte(`</tbody><tfoot><th style="text-align:right;">TOTAL:</th>`)...)
for _, year := range orderedYears {
postsTotalForYear := 0
for _, userPostCount := range yearsToUsersToPostCount[year] {
postsTotalForYear += userPostCount
}
ret = append(ret, []byte(fmt.Sprintf(`<th>%d</th>`, postsTotalForYear))...)
}
duration := time.Now().Sub(startTime)
ret = append(ret, []byte(fmt.Sprintf("</tfoot></table>\n\n<em>%d total users\nStatistics generated in %s</em>", len(allUsernames), duration.String()))...)
this.renderTemplate(w, ret)
}
// renderSearch renders the search results. // renderSearch renders the search results.
// - Mandatory: log, query, queryIsRegex // - Mandatory: log, query, queryIsRegex
func (this *ArchiveState) renderSearch(w http.ResponseWriter) { func (this *ArchiveState) renderSearch(w http.ResponseWriter) {
@ -191,12 +327,11 @@ func (this *ArchiveState) renderTemplateHead(w http.ResponseWriter) {
title = this.log.Description + ` Archives` title = this.log.Description + ` Archives`
} }
showPageURLs := (this.log != nil && len(this.query) == 0)
latestUrl := `/` latestUrl := `/`
if this.log != nil { if this.log != nil {
latestUrl = fmt.Sprintf(`/%s/%d/%d`, url.PathEscape(this.logBestSlug), this.log.LatestDate().Year, this.log.LatestDate().Month) latestUrl = fmt.Sprintf(`/%s/%d/%d`, url.PathEscape(this.logBestSlug), this.log.LatestDate().Year, this.log.LatestDate().Month)
} }
statsUrl := fmt.Sprintf(`/%s/stats`, url.PathEscape(this.logBestSlug))
w.Write([]byte(`<!DOCTYPE html> w.Write([]byte(`<!DOCTYPE html>
<html> <html>
@ -214,6 +349,7 @@ func (this *ArchiveState) renderTemplateHead(w http.ResponseWriter) {
<div id="tr2"></div> <div id="tr2"></div>
<div class="ddmenu"> <div class="ddmenu">
<a href="` + html.EscapeString(latestUrl) + `">Latest</a> <a href="` + html.EscapeString(latestUrl) + `">Latest</a>
<a href="` + html.EscapeString(statsUrl) + `">Statistics</a>
<a onclick="fontSize(1);">Font increase</a> <a onclick="fontSize(1);">Font increase</a>
<a onclick="fontSize(-1);">Font decrease</a> <a onclick="fontSize(-1);">Font decrease</a>
<a href="/download" onclick="return confirm('Are you sure you want to download a backup?');">Download backup</a> <a href="/download" onclick="return confirm('Are you sure you want to download a backup?');">Download backup</a>
@ -251,7 +387,7 @@ func (this *ArchiveState) renderTemplateHead(w http.ResponseWriter) {
</select> </select>
`)) `))
if showPageURLs { if this.showPageURLs() {
w.Write([]byte(` w.Write([]byte(`
<select onchange="window.location.pathname = this.value;"> <select onchange="window.location.pathname = this.value;">
`)) `))

View File

@ -139,6 +139,17 @@ func (this *ArchiveServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} else { } else {
arc.renderError(w, fmt.Sprintf("Unknown source '%s'", matches[1])) arc.renderError(w, fmt.Sprintf("Unknown source '%s'", matches[1]))
} }
} else if matches := this.rxStats.FindStringSubmatch(r.URL.Path); len(matches) > 0 {
if ls := this.lookupSource(matches[1]); ls != nil {
arc.selectSource(ls, matches[1])
arc.isStats = true
arc.renderStats(w)
} else {
arc.renderError(w, fmt.Sprintf("Unknown source '%s'", matches[1]))
}
} else { } else {
arc.renderError(w, "Unknown route.") arc.renderError(w, "Unknown route.")