statistics page

ArchiveServer.go

@@ -18,7 +18,7 @@ type ArchiveServer struct {
 	cfg      *Config
 	startup  time.Time
 
-	rxViewRoot, rxViewPage, rxSearch, rxSearchRx *regexp.Regexp
+	rxViewRoot, rxViewPage, rxSearch, rxSearchRx, rxStats *regexp.Regexp
 }
 
 func NewArchiveServer(cfg *Config) (*ArchiveServer, error) {
@@ -49,6 +49,7 @@ func NewArchiveServer(cfg *Config) (*ArchiveServer, error) {
 		rxViewPage: regexp.MustCompile(`^/([^/]+)/(\d+)/(\d+)/(?:page-)?(\d+)$`),
 		rxSearch:   regexp.MustCompile(`^/([^/]+)/search/(.*)$`),
 		rxSearchRx: regexp.MustCompile(`^/([^/]+)/rx/(.*)$`),
+		rxStats:    regexp.MustCompile((`^/([^/]+)/stats/?$`)),
 	}, nil
 }
 

ArchiveState.go

@@ -2,6 +2,7 @@ package archive
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"html"
 	"io/ioutil"
@@ -10,7 +11,9 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"sort"
 	"strings"
+	"time"
 )
 
 const (
@@ -23,6 +26,7 @@ type ArchiveState struct {
 	logBestSlug  string
 	query        string
 	queryIsRegex bool
+	isStats      bool
 	ym           YearMonth
 	page         int
 	highestPage  int
@@ -35,13 +39,21 @@ func NewArchiveState(svr *ArchiveServer) *ArchiveState {
 	}
 }
 
+func (this *ArchiveState) showPageURLs() bool {
+	return this.log != nil && len(this.query) == 0 && !this.isStats
+}
+
 func (this *ArchiveState) URL() string {
-	if len(this.query) > 0 {
+	if this.isStats {
+		return fmt.Sprintf(`/%s/stats`, this.logBestSlug)
+
+	} else if len(this.query) > 0 {
 		if this.queryIsRegex {
 			return fmt.Sprintf(`/%s/rx/%s`, this.logBestSlug, url.QueryEscape(this.query))
 		} else {
 			return fmt.Sprintf(`/%s/search/%s`, this.logBestSlug, url.QueryEscape(this.query))
 		}
+
 	} else {
 		if this.page == pageNotSet {
 			return fmt.Sprintf(`/%s/%s/%s`, this.logBestSlug, this.ym.Year, this.ym.Month)
@@ -101,6 +113,130 @@ func (this *ArchiveState) renderView(w http.ResponseWriter) {
 	this.renderTemplate(w, []byte(output))
 }
 
+// parseStatsFor loads user post statistics for a single yearMonth in a single log source.
+func (this *ArchiveState) parseStatsFor(ls *LogSource, ym YearMonth, into map[string]int) error {
+
+	fname, err := this.svr.LogFile(ls, ym)
+	if err != nil {
+		return err
+	}
+
+	fc, err := ioutil.ReadFile(fname)
+	if err != nil {
+		return err
+	}
+
+	rxUser := regexp.MustCompile(`(?ms)^[^<\r\n]*<([^>\r\n]+)>.+?$`)
+	matches := rxUser.FindAllSubmatch(fc, -1)
+	if matches == nil || len(matches) == 0 {
+		return errors.New("No matches")
+	}
+
+	for _, match := range matches {
+		username := string(match[1])
+
+		if ct, ok := into[username]; ok {
+			into[username] = ct + 1
+		} else {
+			into[username] = 1
+		}
+	}
+
+	return nil
+}
+
+func (this *ArchiveState) renderStats(w http.ResponseWriter) {
+	// Lines per year
+	// Users / posts/year
+
+	startTime := time.Now()
+
+	yearsToUsersToPostCount := make(map[int]map[string]int, 0)
+
+	totalErrors := 0
+	var lastError error = nil
+
+	ym := this.log.EarliestDate()
+	orderedYears := make([]int, 0)
+	for {
+		usersToPostCount, ok := yearsToUsersToPostCount[ym.Year]
+		if !ok {
+			usersToPostCount = make(map[string]int)
+			orderedYears = append(orderedYears, ym.Year)
+		}
+
+		err := this.parseStatsFor(this.log, ym, usersToPostCount)
+		if err != nil {
+			//log.Printf("Stats(%s): %s", this.logBestSlug, err.Error())
+			totalErrors += 1
+			lastError = err
+		}
+
+		//fmt.Printf("%#v\n", usersToPostCount)
+
+		yearsToUsersToPostCount[ym.Year] = usersToPostCount
+
+		if ym.Equals(this.log.LatestDate()) {
+			break
+		}
+
+		ym = ym.Next()
+	}
+
+	ret := make([]byte, 0)
+
+	if lastError != nil {
+		ret = append(ret, []byte(fmt.Sprintf("<em>Got %d errors, including: '%s'</em>\n\n", totalErrors, lastError.Error()))...)
+	}
+
+	//
+
+	allUsersExistence := make(map[string]struct{})
+	for _, usersMap := range yearsToUsersToPostCount {
+		for username, _ := range usersMap {
+			allUsersExistence[username] = struct{}{}
+		}
+	}
+	allUsernames := make([]string, 0, len(allUsersExistence))
+	for username, _ := range allUsersExistence {
+		allUsernames = append(allUsernames, username)
+	}
+	sort.Strings(allUsernames)
+
+	//
+
+	ret = append(ret, []byte(`<table><thead><tr><th>&nbsp;</th>`)...)
+	for _, year := range orderedYears {
+		ret = append(ret, []byte(fmt.Sprintf(`<th>%d</th>`, year))...)
+	}
+	ret = append(ret, []byte("</tr><tbody>\n")...)
+	for _, username := range allUsernames {
+
+		ret = append(ret, []byte(fmt.Sprintf(`<tr><td>%s</td>`, html.EscapeString(username)))...)
+		for _, year := range orderedYears {
+			usersMap := yearsToUsersToPostCount[year]
+			posts, _ /*ok*/ := usersMap[username]
+			ret = append(ret, []byte(fmt.Sprintf(`<td>%d</td>`, posts))...)
+		}
+		ret = append(ret, []byte("</tr>\n")...)
+	}
+	ret = append(ret, []byte(`</tbody><tfoot><th style="text-align:right;">TOTAL:</th>`)...)
+
+	for _, year := range orderedYears {
+		postsTotalForYear := 0
+		for _, userPostCount := range yearsToUsersToPostCount[year] {
+			postsTotalForYear += userPostCount
+		}
+		ret = append(ret, []byte(fmt.Sprintf(`<th>%d</th>`, postsTotalForYear))...)
+	}
+
+	duration := time.Now().Sub(startTime)
+
+	ret = append(ret, []byte(fmt.Sprintf("</tfoot></table>\n\n<em>%d total users\nStatistics generated in %s</em>", len(allUsernames), duration.String()))...)
+
+	this.renderTemplate(w, ret)
+}
+
 // renderSearch renders the search results.
 // - Mandatory: log, query, queryIsRegex
 func (this *ArchiveState) renderSearch(w http.ResponseWriter) {
@@ -191,12 +327,11 @@ func (this *ArchiveState) renderTemplateHead(w http.ResponseWriter) {
 		title = this.log.Description + ` Archives`
 	}
 
-	showPageURLs := (this.log != nil && len(this.query) == 0)
-
 	latestUrl := `/`
 	if this.log != nil {
 		latestUrl = fmt.Sprintf(`/%s/%d/%d`, url.PathEscape(this.logBestSlug), this.log.LatestDate().Year, this.log.LatestDate().Month)
 	}
+	statsUrl := fmt.Sprintf(`/%s/stats`, url.PathEscape(this.logBestSlug))
 
 	w.Write([]byte(`<!DOCTYPE html>
 <html>
@@ -214,6 +349,7 @@ func (this *ArchiveState) renderTemplateHead(w http.ResponseWriter) {
 				<div id="tr2"></div>
 				<div class="ddmenu">
 					<a href="` + html.EscapeString(latestUrl) + `">Latest</a>
+					<a href="` + html.EscapeString(statsUrl) + `">Statistics</a>
 					<a onclick="fontSize(1);">Font increase</a>
 					<a onclick="fontSize(-1);">Font decrease</a>
 					<a href="/download" onclick="return confirm('Are you sure you want to download a backup?');">Download backup</a>
@@ -251,7 +387,7 @@ func (this *ArchiveState) renderTemplateHead(w http.ResponseWriter) {
 				</select>
 	`))
 
-	if showPageURLs {
+	if this.showPageURLs() {
 		w.Write([]byte(`
 				<select onchange="window.location.pathname = this.value;">
 `))

Router.go

@@ -139,6 +139,17 @@ func (this *ArchiveServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else {
 			arc.renderError(w, fmt.Sprintf("Unknown source '%s'", matches[1]))
 		}
+
+	} else if matches := this.rxStats.FindStringSubmatch(r.URL.Path); len(matches) > 0 {
+		if ls := this.lookupSource(matches[1]); ls != nil {
+			arc.selectSource(ls, matches[1])
+			arc.isStats = true
+			arc.renderStats(w)
+
+		} else {
+			arc.renderError(w, fmt.Sprintf("Unknown source '%s'", matches[1]))
+		}
+
 	} else {
 		arc.renderError(w, "Unknown route.")