doc/changelog: v1.2.1

For codesite-migrated repositories, looking at the oldest commit is
preferable to determine the "create date". But for forked projects,
looking at the oldest commit is incorrect for when we started the project
If inferring the real create date has to be manual, then let's rely on
the Gitea metadata - it's faster and can be modified by hand if needed

README.md

@@ -31,6 +31,16 @@ dokku storage:mount teafolio /srv/teafolio-dokku/config.toml:/app/config.toml
 
 ## CHANGELOG
 
+2021-04-12 v1.2.1
+- Exclude 'build:success' tag from article-only repositories, and re-host the image locally
+- Also search root repo directory for images
+- Enhance mobile viewport and table styling
+
+2020-11-19 v1.2.0
+- Cache homepage repositories, sync changes in the background
+- Consider the updated time to be the most recent commit, not the Gitea repository metadata update field
+- Add extra logging to startup
+
 2020-11-08 v1.1.1
 - Fix an issue with newer versions of Gitea that paginate repoistory list responses
 - Fix an issue with blocking semaphores for a cancelled network request

api.go

@@ -16,8 +16,35 @@ import (
 type Repo struct {
 	Name         string    `json:"name"`
 	Description  string    `json:"description"`
-	CreatedAt   time.Time `json:"created_at"`
-	UpdatedAt   time.Time `json:"updated_at"`
+	GiteaCreated time.Time `json:"created_at"`
+	GiteaUpdated time.Time `json:"updated_at"`
+
+	newestCommit time.Time
+	topics       []string
+}
+
+func (this *Application) populateCommitInfo(ctx context.Context, rr *Repo) {
+
+	// The most recent commit will be the head of one of the branches (easy to find)
+
+	brs, err := this.branches(ctx, rr.Name)
+	if err != nil {
+		log.Printf("loading branches for '%s': %s", rr.Name, err)
+		rr.newestCommit = rr.GiteaUpdated // best guess
+
+	} else {
+
+		newestCommit := time.Unix(0, 0) // sentinel
+		for _, br := range brs {
+			if br.Commit.Timestamp.After(newestCommit) {
+				newestCommit = br.Commit.Timestamp
+			}
+		}
+		if !newestCommit.Equal(time.Unix(0, 0)) {
+			rr.newestCommit = newestCommit // replace it
+		}
+	}
+
 }
 
 type ContentsResponse struct {
@@ -46,31 +73,116 @@ type MarkdownRequest struct {
 	Wiki    bool
 }
 
-// reposPage gets a single page of the list of Git repositories in this organisation.
-func (this *Application) reposPage(ctx context.Context, page, limit int) ([]Repo, error) {
+type BranchCommit struct {
+	ID        string    `json:"id"`
+	Message   string    `json:"message"`
+	Timestamp time.Time `json:"timestamp"`
+}
+
+type Branch struct {
+	Name   string       `json:"name"`
+	Commit BranchCommit `json:"commit"`
+}
+
+type AuthorInfo struct {
+	Name  string    `json:"name"`
+	Email string    `json:"email"`
+	Date  time.Time `json:"date"`
+}
+
+type CommitListEntryCommit struct {
+	Message   string     `json:"message"`
+	Author    AuthorInfo `json:"author"`
+	Committer AuthorInfo `json:"committer"`
+}
+
+type CommitListEntry struct {
+	ID     string                `json:"sha"`
+	Commit CommitListEntryCommit `json:"commit"`
+}
+
+func (this *Application) apiRequest(ctx context.Context, endpoint string, target interface{}) error {
 	err := this.apiSem.Acquire(ctx, 1)
 	if err != nil {
-		return nil, err // e.g. ctx closed
+		return err // e.g. ctx closed
 	}
 	defer this.apiSem.Release(1)
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), nil)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+endpoint, nil)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
+		return fmt.Errorf("HTTP %d", resp.StatusCode)
 	}
 
+	err = json.NewDecoder(resp.Body).Decode(target)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (this *Application) branches(ctx context.Context, repo string) ([]Branch, error) {
+	var branches []Branch
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/branches`, &branches)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return branches, nil
+}
+
+func (this *Application) commitsPage(ctx context.Context, repo, ref string, page, limit int) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	err := this.apiRequest(ctx, fmt.Sprintf(`api/v1/repos/%s/%s/commits?page=%d&limit=%d`, url.PathEscape(this.cfg.Gitea.Org), url.PathEscape(repo), page, limit), &ret)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return ret, nil
+}
+
+func (this *Application) commits(ctx context.Context, repo, ref string) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := this.commitsPage(ctx, repo, ref, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(page) == 0 && len(ret) > 0 {
+			break // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+
+	if len(ret) == 0 {
+		return nil, fmt.Errorf("no commits found")
+	}
+
+	return ret, nil
+}
+
+// reposPage gets a single page of the list of Git repositories in this organisation.
+func (this *Application) reposPage(ctx context.Context, page, limit int) ([]Repo, error) {
 	var repos []Repo
-	err = json.NewDecoder(resp.Body).Decode(&repos)
+
+	err := this.apiRequest(ctx, `api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), &repos)
 	if err != nil {
 		return nil, err
 	}
@@ -93,43 +205,23 @@ func (this *Application) repos(ctx context.Context) ([]Repo, error) {
 			return nil, err
 		}
 
-		log.Printf("Page %d with %d results", nextPage, len(page))
-
 		if len(page) == 0 && len(ret) > 0 {
-			return ret, nil // Found enough already
+			break // Found enough already
 		}
 
 		ret = append(ret, page...)
 		nextPage += 1
 	}
+
+	return ret, nil
 }
 
 // repoFile gets a single file from the default branch of the git repository
 // Usually the default branch is `master`.
 func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
 	var cr ContentsResponse
-	err = json.NewDecoder(resp.Body).Decode(&cr)
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), &cr)
 	if err != nil {
 		return nil, err
 	}
@@ -181,12 +273,12 @@ func (this *Application) filesInDirectory(ctx context.Context, repo, dir string)
 }
 
 // imageFilesForRepo finds documentation images for the repository.
-// It searches the dist/ and doc/ subdirectories.
+// It searches the top-level directory and the dist/ and doc/ subdirectories.
 func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]ReaddirEntry, error) {
 
 	ret := []ReaddirEntry{}
 
-	for _, dirName := range []string{`dist`, `doc`} {
+	for _, dirName := range []string{``, `dist`, `doc`} {
 
 		files, err := this.filesInDirectory(ctx, repo, dirName)
 		if err != nil {
@@ -204,29 +296,9 @@ func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]
 }
 
 func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
 	var tr TopicsResponse
-	err = json.NewDecoder(resp.Body).Decode(&tr)
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, &tr)
 	if err != nil {
 		return nil, err
 	}

main.go

@@ -1,17 +1,13 @@
 package main
 
 import (
-	"bytes"
-	"encoding/base64"
+	"context"
 	"flag"
-	"fmt"
-	"html"
 	"log"
 	"net/http"
-	"net/url"
 	"regexp"
-	"sort"
 	"strings"
+	"sync"
 
 	"github.com/BurntSushi/toml"
 	"golang.org/x/sync/semaphore"
@@ -36,339 +32,10 @@ type Application struct {
 
 	rxRepoPage, rxRepoImage *regexp.Regexp
 	apiSem                  *semaphore.Weighted
-}
-
-func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
-
-	pageTitle := this.cfg.Template.AppName
-	if pageDesc != "" {
-		pageTitle = pageDesc + ` | ` + pageTitle
-	}
-
-	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
-	w.WriteHeader(200)
-	fmt.Fprint(w, `<!DOCTYPE html>
-<html lang="en">
-	<head>
-		<meta http-equiv="X-UA-Compatible" content="IE=edge">
-		<meta charset="UTF-8">
-		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-		<meta name="viewport" content="width=960">
-		<title>`+html.EscapeString(pageTitle)+`</title>
-		`+extraHead+`
-		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
-		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
-		<link type="text/css" rel="stylesheet" href="/static/style.css">
-	</head>
-	<body>
-		<div id="container">
-			<div id="content">
-				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
-	`)
-	cb()
-	fmt.Fprint(w, `
-	</body>
-	<script type="text/javascript" src="/static/site.js"></script>
-</html>
-`)
-
-}
-
-func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
-	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
-	http.Error(w, "An internal error occurred.", 500)
-}
-
-func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-
-	repos, err := this.repos(ctx)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing repos: %w", err))
-		return
-	}
-
-	topics := make(map[string][]string)
-	for _, repo := range repos {
-		if t, err := this.topicsForRepo(ctx, repo.Name); err == nil {
-			topics[repo.Name] = t
-		}
-	}
-
-	// Sort repos once alphabetically, to get alphabetical indexes...
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].Name < repos[j].Name
-	})
-	alphabeticalOrderIndexes := make(map[string]int, len(repos))
-	for idx, repo := range repos {
-		alphabeticalOrderIndexes[repo.Name] = idx
-	}
-
-	// But then make sure the final sort is by most-recently-created
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].CreatedAt.After(repos[j].CreatedAt)
-	})
-
-	// Ready for template
-
-	this.Templatepage(w, r, "", "", func() {
-		fmt.Fprint(w, `
-		`+this.cfg.Template.HomepageHeaderHTML+`		
-		<select id="sortorder" style="float:right;">
-			<option value="data-sort-al">Alphabetical</option>
-			<option value="data-sort-ls">Lifespan</option>
-			<option value="data-sort-ct" selected>Recent projects</option>
-			<option value="data-sort-mt">Recent updates</option>
-		</select>
-		
-		<h2>Projects <small>(`+fmt.Sprintf("%d", len(repos))+`)</small></h2>
-		<table id="projtable-main" class="projtable">
-`)
-		for _, repo := range repos {
-			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
-
-			normalisedDesc := repo.Description
-			normalisedDesc = strings.TrimRight(repo.Description, `.`)
-			if len(normalisedDesc) > 0 {
-				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
-				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
-				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
-
-				if !(isAcronymOrCamelCase || firstWord == `Go`) {
-					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
-				}
-
-				// Add leading `<COMMA><SPACE>` to separate from the repo title
-				normalisedDesc = `, ` + normalisedDesc
-			}
-
-			rowClass := ""
-			for _, topic := range topics[repo.Name] {
-				rowClass += `taggedWith-` + topic + ` `
-			}
-
-			fmt.Fprint(w, `
-			<tr
-				class="`+html.EscapeString(rowClass)+`"
-				data-sort-al="`+fmt.Sprintf("-%d", alphabeticalOrderIndexes[repo.Name])+`"
-				data-sort-ls="`+fmt.Sprintf("%.0f", repo.UpdatedAt.Sub(repo.CreatedAt).Seconds())+`"
-				data-sort-ct="`+fmt.Sprintf("%d", repo.CreatedAt.Unix())+`"
-				data-sort-mt="`+fmt.Sprintf("%d", repo.UpdatedAt.Unix())+`"
-			>
-				<td>
-					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
-				</td>
-				<td>
-					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
-					<a href="`+pageHref+`" class="article-read-more">more...</a>
-					<br>
-					<small>
-			`)
-			for _, topic := range topics[repo.Name] {
-				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
-			}
-			fmt.Fprint(w, `
-					</small>
-				</td>
-			</tr>
-			`)
-		}
-		fmt.Fprint(w, `
-		</table>
-		`)
-	})
-}
-
-func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
-	ctx := r.Context()
-
-	images, err := this.imageFilesForRepo(ctx, repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	if len(images) == 0 {
-		w.Header().Set(`Location`, `/static/no_image.png`)
-		w.WriteHeader(301)
-		return
-	}
-
-	w.Header().Set(`Location`, images[0].RawURL)
-	w.WriteHeader(301)
-}
-
-func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
-	ctx := r.Context()
-	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
-	extraHead := ""
-
-	readme, err := this.repoFile(ctx, repoName, `README.md`)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
-		return
-	}
-
-	lines := strings.Split(string(readme), "\n")
-
-	// We add some extra badges based on special text entries
-	extraBadgesMd := ` ![](https://img.shields.io/badge/build-success-brightgreen)`
-	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
-
-	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
-	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
-		lines[2] += ` ` + extraBadgesMd
-	} else {
-		// Push other lines down
-		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
-	}
-
-	readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
-		return
-	}
-
-	images, err := this.imageFilesForRepo(ctx, repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	// If the Git repository contains a top-level go.mod file, allow vanity imports
-	if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
-
-		// Check the first line should be `module MODULENAME\n`
-		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
-		if bytes.HasPrefix(firstLine, []byte("module ")) {
-			moduleName := firstLine[7:]
-			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
-		}
-	}
-
-	// De-escalate all headers in rendered markdown to match our style
-	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
-		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
-
-	// Ready for template
-
-	this.Templatepage(w, r, repoName, extraHead, func() {
-
-		projBodyclass := `projbody`
-		if len(images) > 0 {
-			projBodyclass += ` projbody_halfw`
-		}
-
-		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
-		repl.WriteString(w, string(readmeHtml))
-		fmt.Fprint(w, `</div>`)
-
-		if len(images) > 0 {
-			fmt.Fprint(w, `<div class="projimg">`)
-			for _, img := range images {
-				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
-			}
-			fmt.Fprint(w, `</div>`)
-		}
-
-		fmt.Fprint(w, `<div style="clear:both;"></div>`)
-		fmt.Fprint(w, `</div>`) // projbody
-	})
-}
-
-func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method == `GET` {
-		if r.URL.Path == `/` {
-			this.Homepage(w, r)
-
-		} else if r.URL.Path == `/favicon.ico` {
-			w.Header().Set(`Location`, `/static/logo.png`)
-			w.WriteHeader(301)
-
-		} else if r.URL.Path == `/robots.txt` {
-			http.Error(w, "not found", 404)
-
-		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
-			this.Bannerpage(w, r, parts[1])
-
-		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
-
-			// Support /repo.html URIs for backward compatibility
-			if strings.HasSuffix(parts[1], `.html`) {
-				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
-				w.WriteHeader(301)
-				return
-			}
-
-			// The regexp supports an optional trailing slash
-			// Redirect to canonical no-trailing-slash
-			if strings.HasSuffix(r.URL.Path, `/`) {
-				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
-				w.WriteHeader(301)
-				return
-			}
-
-			// Proper decoding of special characters in repo path component
-			repoName, err := url.PathUnescape(parts[1])
-			if err != nil {
-				http.Error(w, "malformed url encoding in repository name", 400)
-				return
-			}
-
-			// Maybe it's a redirected project (alternative name)
-			if rename, ok := this.cfg.Redirect[repoName]; ok {
-				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
-				w.WriteHeader(301)
-				return
-			}
-
-			this.Repopage(w, r, repoName)
-
-		} else if r.URL.Path == `/static/logo.png` {
-			if this.cfg.Template.CustomLogoPngBase64 != "" {
-
-				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
-				if err != nil {
-					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
-					return
-				}
-
-				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
-				w.Header().Set(`Content-Type`, `image/png`)
-				w.WriteHeader(200)
-				w.Write(logoPng)
-
-			} else {
-				r.URL.Path = r.URL.Path[8:]
-				http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-			}
-
-		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
-			r.URL.Path = r.URL.Path[8:]
-			http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-		} else if r.URL.Query().Get("go-get") == "1" {
-			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
-			// It must be a subpackage request
-			// We can't serve the proper go-import meta tag immediately because
-			// we haven't looked up the go.mod yet. Just redirect to the root
-			// package - `go get` will follow redirects and the resulting meta tag is correct
-
-			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
-
-			w.Header().Set(`Location`, `/`+slashParts[1])
-			w.WriteHeader(301)
-			return
-
-		} else {
-			http.Error(w, "not found", 404)
-		}
-
-	} else {
-		http.Error(w, "invalid method", 400)
-	}
 
+	reposMut               sync.RWMutex
+	reposCache             []Repo // Sorted by recently-created-first
+	reposAlphabeticalOrder map[string]int
 }
 
 func main() {
@@ -382,7 +49,7 @@ func main() {
 
 	_, err := toml.DecodeFile(*configFile, &app.cfg)
 	if err != nil {
-		panic(err)
+		log.Fatalf("toml.DecodeFile: %s", err.Error())
 	}
 
 	// Assert Gitea URL always has trailing slash
@@ -397,5 +64,9 @@ func main() {
 		app.apiSem = semaphore.NewWeighted(app.cfg.Gitea.MaxConnections)
 	}
 
+	// Sync worker
+	go app.syncWorker(context.Background())
+
+	log.Printf("Starting web server on [%s]...", app.cfg.BindTo)
 	log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app))
 }

pages.go

@@ -0,0 +1,261 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
+
+	pageTitle := this.cfg.Template.AppName
+	if pageDesc != "" {
+		pageTitle = pageDesc + ` | ` + pageTitle
+	}
+
+	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
+	w.WriteHeader(200)
+	fmt.Fprint(w, `<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta charset="UTF-8">
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+		<title>`+html.EscapeString(pageTitle)+`</title>
+		`+extraHead+`
+		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
+		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
+		<link type="text/css" rel="stylesheet" href="/static/style.css">
+	</head>
+	<body>
+		<div id="container">
+			<div id="content">
+				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
+	`)
+	cb()
+	fmt.Fprint(w, `
+	</body>
+	<script type="text/javascript" src="/static/site.js"></script>
+</html>
+`)
+
+}
+
+func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
+	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
+	http.Error(w, "An internal error occurred.", 500)
+}
+
+func (this *Application) Delay(w http.ResponseWriter, r *http.Request) {
+	this.Templatepage(w, r, "Loading...", "", func() {
+		fmt.Fprintf(w, `
+		<h2>Loading, please wait...</h2>
+		
+		<meta http-equiv="refresh" content="5">
+		`)
+	})
+}
+
+func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
+
+	this.reposMut.RLock()
+	defer this.reposMut.RUnlock()
+
+	if len(this.reposCache) == 0 {
+		// We haven't loaded the repositories from Gitea yet
+		this.Delay(w, r)
+		return
+	}
+
+	// Ready for template
+
+	this.Templatepage(w, r, "", "", func() {
+		fmt.Fprint(w, `
+		`+this.cfg.Template.HomepageHeaderHTML+`		
+		<select id="sortorder" style="float:right;">
+			<option value="data-sort-al">Alphabetical</option>
+			<option value="data-sort-ls">Lifespan</option>
+			<option value="data-sort-ct" selected>Recent projects</option>
+			<option value="data-sort-mt">Recent updates</option>
+		</select>
+		
+		<h2>Projects <small>(`+fmt.Sprintf("%d", len(this.reposCache))+`)</small></h2>
+		<table id="projtable-main" class="projtable">
+`)
+		for _, repo := range this.reposCache {
+			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
+
+			normalisedDesc := repo.Description
+			normalisedDesc = strings.TrimRight(repo.Description, `.`)
+			if len(normalisedDesc) > 0 {
+				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
+				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
+				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
+
+				if !(isAcronymOrCamelCase || firstWord == `Go`) {
+					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
+				}
+
+				// Add leading `<COMMA><SPACE>` to separate from the repo title
+				normalisedDesc = `, ` + normalisedDesc
+			}
+
+			rowClass := ""
+			for _, topic := range repo.topics {
+				rowClass += `taggedWith-` + topic + ` `
+			}
+
+			fmt.Fprint(w, `
+			<tr
+				class="`+html.EscapeString(rowClass)+`"
+				data-sort-al="`+fmt.Sprintf("-%d", this.reposAlphabeticalOrder[repo.Name])+`"
+				data-sort-ls="`+fmt.Sprintf("%.0f", repo.newestCommit.Sub(repo.GiteaCreated).Seconds())+`"
+				data-sort-ct="`+fmt.Sprintf("%d", repo.GiteaCreated.Unix())+`"
+				data-sort-mt="`+fmt.Sprintf("%d", repo.newestCommit.Unix())+`"
+			>
+				<td>
+					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
+				</td>
+				<td>
+					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
+					<a href="`+pageHref+`" class="article-read-more">more...</a>
+					<br>
+					<small>
+			`)
+			for _, topic := range repo.topics {
+				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
+			}
+			fmt.Fprint(w, `
+					</small>
+				</td>
+			</tr>
+			`)
+		}
+		fmt.Fprint(w, `
+		</table>
+		`)
+	})
+}
+
+func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
+
+	images, err := this.imageFilesForRepo(ctx, repoName)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
+		return
+	}
+
+	if len(images) == 0 {
+		w.Header().Set(`Location`, `/static/no_image.png`)
+		w.WriteHeader(301)
+		return
+	}
+
+	w.Header().Set(`Location`, images[0].RawURL)
+	w.WriteHeader(301)
+}
+
+func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
+	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
+	extraHead := ""
+
+	readme, err := this.repoFile(ctx, repoName, `README.md`)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
+		return
+	}
+
+	lines := strings.Split(string(readme), "\n")
+
+	// Check if this repo has the 'article' tag
+	hasArticleTag := false
+	this.reposMut.RLock()
+	for _, rr := range this.reposCache {
+		if rr.Name != repoName {
+			continue
+		}
+		for _, topic := range rr.topics {
+			if topic == "article" {
+				hasArticleTag = true
+				break
+			}
+		}
+	}
+	this.reposMut.RUnlock()
+
+	// We add some extra badges based on special text entries
+	extraBadgesMd := ``
+	if !hasArticleTag {
+		extraBadgesMd += ` %%REPLACEME__BADGE%%`
+	}
+	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
+
+	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
+	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
+		lines[2] += ` ` + extraBadgesMd
+	} else {
+		// Push other lines down
+		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
+	}
+
+	readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
+		return
+	}
+
+	readmeHtml = []byte(strings.Replace(string(readmeHtml), `%%REPLACEME__BADGE%%`, `<img src="/static/build_success_brightgreen.svg" style="width:90px;height:20px;">`, 1))
+
+	images, err := this.imageFilesForRepo(ctx, repoName)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
+		return
+	}
+
+	// If the Git repository contains a top-level go.mod file, allow vanity imports
+	if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
+
+		// Check the first line should be `module MODULENAME\n`
+		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
+		if bytes.HasPrefix(firstLine, []byte("module ")) {
+			moduleName := firstLine[7:]
+			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
+		}
+	}
+
+	// De-escalate all headers in rendered markdown to match our style
+	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
+		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
+
+	// Ready for template
+
+	this.Templatepage(w, r, repoName, extraHead, func() {
+
+		projBodyclass := `projbody`
+		if len(images) > 0 {
+			projBodyclass += ` projbody_halfw`
+		}
+
+		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
+		repl.WriteString(w, string(readmeHtml))
+		fmt.Fprint(w, `</div>`)
+
+		if len(images) > 0 {
+			fmt.Fprint(w, `<div class="projimg">`)
+			for _, img := range images {
+				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
+			}
+			fmt.Fprint(w, `</div>`)
+		}
+
+		fmt.Fprint(w, `<div style="clear:both;"></div>`)
+		fmt.Fprint(w, `</div>`) // projbody
+	})
+}

router.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method == `GET` {
+		if r.URL.Path == `/` {
+			this.Homepage(w, r)
+
+		} else if r.URL.Path == `/favicon.ico` {
+			w.Header().Set(`Location`, `/static/logo.png`)
+			w.WriteHeader(301)
+
+		} else if r.URL.Path == `/robots.txt` {
+			http.Error(w, "not found", 404)
+
+		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
+			this.Bannerpage(w, r, parts[1])
+
+		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
+
+			// Support /repo.html URIs for backward compatibility
+			if strings.HasSuffix(parts[1], `.html`) {
+				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
+				w.WriteHeader(301)
+				return
+			}
+
+			// The regexp supports an optional trailing slash
+			// Redirect to canonical no-trailing-slash
+			if strings.HasSuffix(r.URL.Path, `/`) {
+				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
+				w.WriteHeader(301)
+				return
+			}
+
+			// Proper decoding of special characters in repo path component
+			repoName, err := url.PathUnescape(parts[1])
+			if err != nil {
+				http.Error(w, "malformed url encoding in repository name", 400)
+				return
+			}
+
+			// Maybe it's a redirected project (alternative name)
+			if rename, ok := this.cfg.Redirect[repoName]; ok {
+				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
+				w.WriteHeader(301)
+				return
+			}
+
+			this.Repopage(w, r, repoName)
+
+		} else if r.URL.Path == `/static/logo.png` {
+			if this.cfg.Template.CustomLogoPngBase64 != "" {
+
+				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
+				if err != nil {
+					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
+					return
+				}
+
+				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
+				w.Header().Set(`Content-Type`, `image/png`)
+				w.WriteHeader(200)
+				w.Write(logoPng)
+
+			} else {
+				r.URL.Path = r.URL.Path[8:]
+				http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
+
+			}
+
+		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
+			r.URL.Path = r.URL.Path[8:]
+			http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
+
+		} else if r.URL.Query().Get("go-get") == "1" {
+			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
+			// It must be a subpackage request
+			// We can't serve the proper go-import meta tag immediately because
+			// we haven't looked up the go.mod yet. Just redirect to the root
+			// package - `go get` will follow redirects and the resulting meta tag is correct
+
+			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
+
+			w.Header().Set(`Location`, `/`+slashParts[1])
+			w.WriteHeader(301)
+			return
+
+		} else {
+			http.Error(w, "not found", 404)
+		}
+
+	} else {
+		http.Error(w, "invalid method", 400)
+	}
+
+}

static/build_success_brightgreen.svg

@@ -0,0 +1,2 @@
+
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="90" height="20" role="img" aria-label="build: success"><title>build: success</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="90" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="37" height="20" fill="#555"/><rect x="37" width="53" height="20" fill="#4c1"/><rect width="90" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="195" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="270">build</text><text x="195" y="140" transform="scale(.1)" fill="#fff" textLength="270">build</text><text aria-hidden="true" x="625" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">success</text><text x="625" y="140" transform="scale(.1)" fill="#fff" textLength="430">success</text></g></svg>

static/style.css

@@ -161,6 +161,25 @@ img[src*="shields.io"] {
 
 /* */
 
+.projbody table {
+	width: 100%;
+}
+
+.projbody table {
+    border-collapse: collapse;
+}
+
+.projbody table td, .projbody table th {
+	border: 1px solid #eee;
+    padding: 4px;
+}
+
+.projbody tr:hover td {
+    background: #f8f8f8;
+}
+
+/* */
+
 @media screen and (max-width:960px) {
 	
 	#container {

sync.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"context"
+	"log"
+	"sort"
+	"time"
+)
+
+func (this *Application) sync(ctx context.Context) (bool, error) {
+
+	// List repositories on Gitea
+	repos, err := this.repos(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	// Compare this list of repositories to our existing one
+	// If the repository is new, or if it's update-time has changed since we last
+	// saw it, then re-refresh its real git commit timestamps
+	// Otherwise copy them from the previous version
+
+	this.reposMut.RLock() // readonly
+
+	anyChanges := false
+	if len(repos) != len(this.reposCache) {
+		anyChanges = true
+	}
+
+	for i, rr := range repos {
+		if idx, ok := this.reposAlphabeticalOrder[rr.Name]; ok && this.reposCache[idx].GiteaUpdated == rr.GiteaUpdated {
+			// Already exists in cache with same Gitea update time
+			// Copy timestamps
+			repos[i] = this.reposCache[idx]
+
+		} else {
+
+			// New repo, or Gitea has updated timestamp
+			anyChanges = true
+
+			// Refresh timestamps
+			this.populateCommitInfo(ctx, &rr)
+
+			// Refresh topics
+			if t, err := this.topicsForRepo(ctx, rr.Name); err == nil {
+				rr.topics = t
+			}
+
+			// Save
+			repos[i] = rr
+		}
+	}
+
+	this.reposMut.RUnlock()
+
+	//
+	if !anyChanges {
+		return false, nil // nothing to do
+	}
+
+	// We have a final updated repos array
+
+	// Sort repos once alphabetically, to get alphabetical indexes...
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	alphabeticalOrderIndexes := make(map[string]int, len(repos))
+	for idx, repo := range repos {
+		alphabeticalOrderIndexes[repo.Name] = idx
+	}
+
+	// But then make sure the final sort is by most-recently-created
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GiteaCreated.After(repos[j].GiteaCreated)
+	})
+
+	// Commit our changes for the other threads to look at
+	this.reposMut.Lock()
+	this.reposCache = repos
+	this.reposAlphabeticalOrder = alphabeticalOrderIndexes
+	this.reposMut.Unlock()
+
+	// Done
+	return true, nil
+}
+
+func (this *Application) syncWorker(ctx context.Context) {
+
+	t := time.NewTicker(30 * time.Minute)
+	defer t.Stop()
+
+	for {
+		anyChanges, err := this.sync(ctx)
+		if err != nil {
+			// log and continue
+			log.Printf("Refreshing repositories: %s", err.Error())
+		}
+		if anyChanges {
+			log.Printf("Repositories updated")
+		}
+
+		select {
+		case <-t.C:
+			continue
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}