doc/changelog: v1.2.0

For codesite-migrated repositories, looking at the oldest commit is
preferable to determine the "create date". But for forked projects,
looking at the oldest commit is incorrect for when we started the project
If inferring the real create date has to be manual, then let's rely on
the Gitea metadata - it's faster and can be modified by hand if needed

README.md

@@ -31,6 +31,11 @@ dokku storage:mount teafolio /srv/teafolio-dokku/config.toml:/app/config.toml
 
 ## CHANGELOG
 
+2020-11-19 v1.2.0
+- Cache homepage repositories, sync changes in the background
+- Consider the updated time to be the most recent commit, not the Gitea repository metadata update field
+- Add extra logging to startup
+
 2020-11-08 v1.1.1
 - Fix an issue with newer versions of Gitea that paginate repoistory list responses
 - Fix an issue with blocking semaphores for a cancelled network request

api.go

@@ -6,13 +6,47 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"strings"
-
-	"code.gitea.io/sdk/gitea"
+	"time"
 )
 
+type Repo struct {
+	Name         string    `json:"name"`
+	Description  string    `json:"description"`
+	GiteaCreated time.Time `json:"created_at"`
+	GiteaUpdated time.Time `json:"updated_at"`
+
+	newestCommit time.Time
+	topics       []string
+}
+
+func (this *Application) populateCommitInfo(ctx context.Context, rr *Repo) {
+
+	// The most recent commit will be the head of one of the branches (easy to find)
+
+	brs, err := this.branches(ctx, rr.Name)
+	if err != nil {
+		log.Printf("loading branches for '%s': %s", rr.Name, err)
+		rr.newestCommit = rr.GiteaUpdated // best guess
+
+	} else {
+
+		newestCommit := time.Unix(0, 0) // sentinel
+		for _, br := range brs {
+			if br.Commit.Timestamp.After(newestCommit) {
+				newestCommit = br.Commit.Timestamp
+			}
+		}
+		if !newestCommit.Equal(time.Unix(0, 0)) {
+			rr.newestCommit = newestCommit // replace it
+		}
+	}
+
+}
+
 type ContentsResponse struct {
 	Content []byte `json:"content"` // Assume base64 "encoding" parameter in Gitea response, and use Go's auto decode
 }
@@ -39,15 +73,116 @@ type MarkdownRequest struct {
 	Wiki    bool
 }
 
-// reposPage gets a single page of the list of Git repositories in this organisation.
-func (this *Application) reposPage(ctx context.Context, page, limit int) ([]*gitea.Repository, error) {
+type BranchCommit struct {
+	ID        string    `json:"id"`
+	Message   string    `json:"message"`
+	Timestamp time.Time `json:"timestamp"`
+}
+
+type Branch struct {
+	Name   string       `json:"name"`
+	Commit BranchCommit `json:"commit"`
+}
+
+type AuthorInfo struct {
+	Name  string    `json:"name"`
+	Email string    `json:"email"`
+	Date  time.Time `json:"date"`
+}
+
+type CommitListEntryCommit struct {
+	Message   string     `json:"message"`
+	Author    AuthorInfo `json:"author"`
+	Committer AuthorInfo `json:"committer"`
+}
+
+type CommitListEntry struct {
+	ID     string                `json:"sha"`
+	Commit CommitListEntryCommit `json:"commit"`
+}
+
+func (this *Application) apiRequest(ctx context.Context, endpoint string, target interface{}) error {
 	err := this.apiSem.Acquire(ctx, 1)
 	if err != nil {
-		return nil, err // e.g. ctx closed
+		return err // e.g. ctx closed
 	}
 	defer this.apiSem.Release(1)
 
-	repos, _, err := this.gc.ListOrgRepos(this.cfg.Gitea.Org, gitea.ListOrgReposOptions{ListOptions: gitea.ListOptions{Page: page, PageSize: limit}})
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+endpoint, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+
+	err = json.NewDecoder(resp.Body).Decode(target)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (this *Application) branches(ctx context.Context, repo string) ([]Branch, error) {
+	var branches []Branch
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/branches`, &branches)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return branches, nil
+}
+
+func (this *Application) commitsPage(ctx context.Context, repo, ref string, page, limit int) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	err := this.apiRequest(ctx, fmt.Sprintf(`api/v1/repos/%s/%s/commits?page=%d&limit=%d`, url.PathEscape(this.cfg.Gitea.Org), url.PathEscape(repo), page, limit), &ret)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return ret, nil
+}
+
+func (this *Application) commits(ctx context.Context, repo, ref string) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := this.commitsPage(ctx, repo, ref, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(page) == 0 && len(ret) > 0 {
+			break // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+
+	if len(ret) == 0 {
+		return nil, fmt.Errorf("no commits found")
+	}
+
+	return ret, nil
+}
+
+// reposPage gets a single page of the list of Git repositories in this organisation.
+func (this *Application) reposPage(ctx context.Context, page, limit int) ([]Repo, error) {
+	var repos []Repo
+
+	err := this.apiRequest(ctx, `api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), &repos)
 	if err != nil {
 		return nil, err
 	}
@@ -57,11 +192,11 @@ func (this *Application) reposPage(ctx context.Context, page, limit int) ([]*git
 
 // repos gets a list of Git repositories in this organisation. It may have to
 // make multiple network requests.
-func (this *Application) repos(ctx context.Context) ([]*gitea.Repository, error) {
+func (this *Application) repos(ctx context.Context) ([]Repo, error) {
 	// Seems like gitea-1.13.0-rc1 returns 30 results by default, and supports up to a limit of 100
 	// Make a much larger request
 
-	ret := make([]*gitea.Repository, 0)
+	ret := make([]Repo, 0)
 
 	nextPage := 1 // Counting starts at 1
 	for {
@@ -71,29 +206,27 @@ func (this *Application) repos(ctx context.Context) ([]*gitea.Repository, error)
 		}
 
 		if len(page) == 0 && len(ret) > 0 {
-			return ret, nil // Found enough already
+			break // Found enough already
 		}
 
 		ret = append(ret, page...)
 		nextPage += 1
 	}
+
+	return ret, nil
 }
 
 // repoFile gets a single file from the default branch of the git repository
 // Usually the default branch is `master`.
 func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
+	var cr ContentsResponse
 
-	resp, _, err := this.gc.GetContents(this.cfg.Gitea.Org, repo, "", filename)
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), &cr)
 	if err != nil {
 		return nil, err
 	}
 
-	return []byte(*resp.Content), nil
+	return cr.Content, nil
 }
 
 func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) {
@@ -163,29 +296,9 @@ func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]
 }
 
 func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
 	var tr TopicsResponse
-	err = json.NewDecoder(resp.Body).Decode(&tr)
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, &tr)
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -3,7 +3,6 @@ module teafolio
 go 1.13
 
 require (
-	code.gitea.io/sdk/gitea v0.13.1
 	github.com/BurntSushi/toml v0.3.1
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 )

go.sum

@@ -1,15 +1,4 @@
-code.gitea.io/sdk v0.11.0 h1:R3VdjBCxObyLKnv4Svd/TM6oGsXzN8JORbzgkEFb83w=
-code.gitea.io/sdk/gitea v0.13.1 h1:Y7bpH2iO6Q0KhhMJfjP/LZ0AmiYITeRQlCD8b0oYqhk=
-code.gitea.io/sdk/gitea v0.13.1/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
-github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -1,19 +1,14 @@
 package main
 
 import (
-	"bytes"
-	"encoding/base64"
+	"context"
 	"flag"
-	"fmt"
-	"html"
 	"log"
 	"net/http"
-	"net/url"
 	"regexp"
-	"sort"
 	"strings"
+	"sync"
 
-	"code.gitea.io/sdk/gitea"
 	"github.com/BurntSushi/toml"
 	"golang.org/x/sync/semaphore"
 )
@@ -37,340 +32,10 @@ type Application struct {
 
 	rxRepoPage, rxRepoImage *regexp.Regexp
 	apiSem                  *semaphore.Weighted
-	gc                      *gitea.Client
-}
-
-func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
-
-	pageTitle := this.cfg.Template.AppName
-	if pageDesc != "" {
-		pageTitle = pageDesc + ` | ` + pageTitle
-	}
-
-	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
-	w.WriteHeader(200)
-	fmt.Fprint(w, `<!DOCTYPE html>
-<html lang="en">
-	<head>
-		<meta http-equiv="X-UA-Compatible" content="IE=edge">
-		<meta charset="UTF-8">
-		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-		<meta name="viewport" content="width=960">
-		<title>`+html.EscapeString(pageTitle)+`</title>
-		`+extraHead+`
-		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
-		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
-		<link type="text/css" rel="stylesheet" href="/static/style.css">
-	</head>
-	<body>
-		<div id="container">
-			<div id="content">
-				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
-	`)
-	cb()
-	fmt.Fprint(w, `
-	</body>
-	<script type="text/javascript" src="/static/site.js"></script>
-</html>
-`)
-
-}
-
-func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
-	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
-	http.Error(w, "An internal error occurred.", 500)
-}
-
-func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-
-	repos, err := this.repos(ctx)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing repos: %w", err))
-		return
-	}
-
-	topics := make(map[string][]string)
-	for _, repo := range repos {
-		if t, err := this.topicsForRepo(ctx, repo.Name); err == nil {
-			topics[repo.Name] = t
-		}
-	}
-
-	// Sort repos once alphabetically, to get alphabetical indexes...
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].Name < repos[j].Name
-	})
-	alphabeticalOrderIndexes := make(map[string]int, len(repos))
-	for idx, repo := range repos {
-		alphabeticalOrderIndexes[repo.Name] = idx
-	}
-
-	// But then make sure the final sort is by most-recently-created
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].Created.After(repos[j].Created)
-	})
-
-	// Ready for template
-
-	this.Templatepage(w, r, "", "", func() {
-		fmt.Fprint(w, `
-		`+this.cfg.Template.HomepageHeaderHTML+`		
-		<select id="sortorder" style="float:right;">
-			<option value="data-sort-al">Alphabetical</option>
-			<option value="data-sort-ls">Lifespan</option>
-			<option value="data-sort-ct" selected>Recent projects</option>
-			<option value="data-sort-mt">Recent updates</option>
-		</select>
-		
-		<h2>Projects <small>(`+fmt.Sprintf("%d", len(repos))+`)</small></h2>
-		<table id="projtable-main" class="projtable">
-`)
-		for _, repo := range repos {
-			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
-
-			normalisedDesc := repo.Description
-			normalisedDesc = strings.TrimRight(repo.Description, `.`)
-			if len(normalisedDesc) > 0 {
-				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
-				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
-				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
-
-				if !(isAcronymOrCamelCase || firstWord == `Go`) {
-					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
-				}
-
-				// Add leading `<COMMA><SPACE>` to separate from the repo title
-				normalisedDesc = `, ` + normalisedDesc
-			}
-
-			rowClass := ""
-			for _, topic := range topics[repo.Name] {
-				rowClass += `taggedWith-` + topic + ` `
-			}
-
-			fmt.Fprint(w, `
-			<tr
-				class="`+html.EscapeString(rowClass)+`"
-				data-sort-al="`+fmt.Sprintf("-%d", alphabeticalOrderIndexes[repo.Name])+`"
-				data-sort-ls="`+fmt.Sprintf("%.0f", repo.Updated.Sub(repo.Created).Seconds())+`"
-				data-sort-ct="`+fmt.Sprintf("%d", repo.Created.Unix())+`"
-				data-sort-mt="`+fmt.Sprintf("%d", repo.Updated.Unix())+`"
-			>
-				<td>
-					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
-				</td>
-				<td>
-					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
-					<a href="`+pageHref+`" class="article-read-more">more...</a>
-					<br>
-					<small>
-			`)
-			for _, topic := range topics[repo.Name] {
-				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
-			}
-			fmt.Fprint(w, `
-					</small>
-				</td>
-			</tr>
-			`)
-		}
-		fmt.Fprint(w, `
-		</table>
-		`)
-	})
-}
-
-func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
-	ctx := r.Context()
-
-	images, err := this.imageFilesForRepo(ctx, repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	if len(images) == 0 {
-		w.Header().Set(`Location`, `/static/no_image.png`)
-		w.WriteHeader(301)
-		return
-	}
-
-	w.Header().Set(`Location`, images[0].RawURL)
-	w.WriteHeader(301)
-}
-
-func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
-	ctx := r.Context()
-	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
-	extraHead := ""
-
-	readme, err := this.repoFile(ctx, repoName, `README.md`)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
-		return
-	}
-
-	lines := strings.Split(string(readme), "\n")
-
-	// We add some extra badges based on special text entries
-	extraBadgesMd := ` ![](https://img.shields.io/badge/build-success-brightgreen)`
-	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
-
-	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
-	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
-		lines[2] += ` ` + extraBadgesMd
-	} else {
-		// Push other lines down
-		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
-	}
-
-	readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
-		return
-	}
-
-	images, err := this.imageFilesForRepo(ctx, repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	// If the Git repository contains a top-level go.mod file, allow vanity imports
-	if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
-
-		// Check the first line should be `module MODULENAME\n`
-		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
-		if bytes.HasPrefix(firstLine, []byte("module ")) {
-			moduleName := firstLine[7:]
-			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
-		}
-	}
-
-	// De-escalate all headers in rendered markdown to match our style
-	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
-		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
-
-	// Ready for template
-
-	this.Templatepage(w, r, repoName, extraHead, func() {
-
-		projBodyclass := `projbody`
-		if len(images) > 0 {
-			projBodyclass += ` projbody_halfw`
-		}
-
-		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
-		repl.WriteString(w, string(readmeHtml))
-		fmt.Fprint(w, `</div>`)
-
-		if len(images) > 0 {
-			fmt.Fprint(w, `<div class="projimg">`)
-			for _, img := range images {
-				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
-			}
-			fmt.Fprint(w, `</div>`)
-		}
-
-		fmt.Fprint(w, `<div style="clear:both;"></div>`)
-		fmt.Fprint(w, `</div>`) // projbody
-	})
-}
-
-func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method == `GET` {
-		if r.URL.Path == `/` {
-			this.Homepage(w, r)
-
-		} else if r.URL.Path == `/favicon.ico` {
-			w.Header().Set(`Location`, `/static/logo.png`)
-			w.WriteHeader(301)
-
-		} else if r.URL.Path == `/robots.txt` {
-			http.Error(w, "not found", 404)
-
-		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
-			this.Bannerpage(w, r, parts[1])
-
-		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
-
-			// Support /repo.html URIs for backward compatibility
-			if strings.HasSuffix(parts[1], `.html`) {
-				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
-				w.WriteHeader(301)
-				return
-			}
-
-			// The regexp supports an optional trailing slash
-			// Redirect to canonical no-trailing-slash
-			if strings.HasSuffix(r.URL.Path, `/`) {
-				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
-				w.WriteHeader(301)
-				return
-			}
-
-			// Proper decoding of special characters in repo path component
-			repoName, err := url.PathUnescape(parts[1])
-			if err != nil {
-				http.Error(w, "malformed url encoding in repository name", 400)
-				return
-			}
-
-			// Maybe it's a redirected project (alternative name)
-			if rename, ok := this.cfg.Redirect[repoName]; ok {
-				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
-				w.WriteHeader(301)
-				return
-			}
-
-			this.Repopage(w, r, repoName)
-
-		} else if r.URL.Path == `/static/logo.png` {
-			if this.cfg.Template.CustomLogoPngBase64 != "" {
-
-				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
-				if err != nil {
-					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
-					return
-				}
-
-				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
-				w.Header().Set(`Content-Type`, `image/png`)
-				w.WriteHeader(200)
-				w.Write(logoPng)
-
-			} else {
-				r.URL.Path = r.URL.Path[8:]
-				http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-			}
-
-		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
-			r.URL.Path = r.URL.Path[8:]
-			http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-		} else if r.URL.Query().Get("go-get") == "1" {
-			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
-			// It must be a subpackage request
-			// We can't serve the proper go-import meta tag immediately because
-			// we haven't looked up the go.mod yet. Just redirect to the root
-			// package - `go get` will follow redirects and the resulting meta tag is correct
-
-			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
-
-			w.Header().Set(`Location`, `/`+slashParts[1])
-			w.WriteHeader(301)
-			return
-
-		} else {
-			http.Error(w, "not found", 404)
-		}
-
-	} else {
-		http.Error(w, "invalid method", 400)
-	}
 
+	reposMut               sync.RWMutex
+	reposCache             []Repo // Sorted by recently-created-first
+	reposAlphabeticalOrder map[string]int
 }
 
 func main() {
@@ -399,11 +64,9 @@ func main() {
 		app.apiSem = semaphore.NewWeighted(app.cfg.Gitea.MaxConnections)
 	}
 
-	gc, err := gitea.NewClient(app.cfg.Gitea.URL)
-	if err != nil {
-		log.Fatalf("gitea.NewClient: %s", err.Error())
-	}
-	app.gc = gc
+	// Sync worker
+	go app.syncWorker(context.Background())
 
+	log.Printf("Starting web server on [%s]...", app.cfg.BindTo)
 	log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app))
 }

pages.go

@@ -0,0 +1,240 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
+
+	pageTitle := this.cfg.Template.AppName
+	if pageDesc != "" {
+		pageTitle = pageDesc + ` | ` + pageTitle
+	}
+
+	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
+	w.WriteHeader(200)
+	fmt.Fprint(w, `<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta charset="UTF-8">
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+		<meta name="viewport" content="width=960">
+		<title>`+html.EscapeString(pageTitle)+`</title>
+		`+extraHead+`
+		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
+		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
+		<link type="text/css" rel="stylesheet" href="/static/style.css">
+	</head>
+	<body>
+		<div id="container">
+			<div id="content">
+				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
+	`)
+	cb()
+	fmt.Fprint(w, `
+	</body>
+	<script type="text/javascript" src="/static/site.js"></script>
+</html>
+`)
+
+}
+
+func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
+	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
+	http.Error(w, "An internal error occurred.", 500)
+}
+
+func (this *Application) Delay(w http.ResponseWriter, r *http.Request) {
+	this.Templatepage(w, r, "Loading...", "", func() {
+		fmt.Fprintf(w, `
+		<h2>Loading, please wait...</h2>
+		
+		<meta http-equiv="refresh" content="5">
+		`)
+	})
+}
+
+func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
+
+	this.reposMut.RLock()
+	defer this.reposMut.RUnlock()
+
+	if len(this.reposCache) == 0 {
+		// We haven't loaded the repositories from Gitea yet
+		this.Delay(w, r)
+		return
+	}
+
+	// Ready for template
+
+	this.Templatepage(w, r, "", "", func() {
+		fmt.Fprint(w, `
+		`+this.cfg.Template.HomepageHeaderHTML+`		
+		<select id="sortorder" style="float:right;">
+			<option value="data-sort-al">Alphabetical</option>
+			<option value="data-sort-ls">Lifespan</option>
+			<option value="data-sort-ct" selected>Recent projects</option>
+			<option value="data-sort-mt">Recent updates</option>
+		</select>
+		
+		<h2>Projects <small>(`+fmt.Sprintf("%d", len(this.reposCache))+`)</small></h2>
+		<table id="projtable-main" class="projtable">
+`)
+		for _, repo := range this.reposCache {
+			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
+
+			normalisedDesc := repo.Description
+			normalisedDesc = strings.TrimRight(repo.Description, `.`)
+			if len(normalisedDesc) > 0 {
+				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
+				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
+				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
+
+				if !(isAcronymOrCamelCase || firstWord == `Go`) {
+					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
+				}
+
+				// Add leading `<COMMA><SPACE>` to separate from the repo title
+				normalisedDesc = `, ` + normalisedDesc
+			}
+
+			rowClass := ""
+			for _, topic := range repo.topics {
+				rowClass += `taggedWith-` + topic + ` `
+			}
+
+			fmt.Fprint(w, `
+			<tr
+				class="`+html.EscapeString(rowClass)+`"
+				data-sort-al="`+fmt.Sprintf("-%d", this.reposAlphabeticalOrder[repo.Name])+`"
+				data-sort-ls="`+fmt.Sprintf("%.0f", repo.newestCommit.Sub(repo.GiteaCreated).Seconds())+`"
+				data-sort-ct="`+fmt.Sprintf("%d", repo.GiteaCreated.Unix())+`"
+				data-sort-mt="`+fmt.Sprintf("%d", repo.newestCommit.Unix())+`"
+			>
+				<td>
+					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
+				</td>
+				<td>
+					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
+					<a href="`+pageHref+`" class="article-read-more">more...</a>
+					<br>
+					<small>
+			`)
+			for _, topic := range repo.topics {
+				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
+			}
+			fmt.Fprint(w, `
+					</small>
+				</td>
+			</tr>
+			`)
+		}
+		fmt.Fprint(w, `
+		</table>
+		`)
+	})
+}
+
+func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
+
+	images, err := this.imageFilesForRepo(ctx, repoName)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
+		return
+	}
+
+	if len(images) == 0 {
+		w.Header().Set(`Location`, `/static/no_image.png`)
+		w.WriteHeader(301)
+		return
+	}
+
+	w.Header().Set(`Location`, images[0].RawURL)
+	w.WriteHeader(301)
+}
+
+func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
+	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
+	extraHead := ""
+
+	readme, err := this.repoFile(ctx, repoName, `README.md`)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
+		return
+	}
+
+	lines := strings.Split(string(readme), "\n")
+
+	// We add some extra badges based on special text entries
+	extraBadgesMd := ` ![](https://img.shields.io/badge/build-success-brightgreen)`
+	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
+
+	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
+	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
+		lines[2] += ` ` + extraBadgesMd
+	} else {
+		// Push other lines down
+		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
+	}
+
+	readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
+		return
+	}
+
+	images, err := this.imageFilesForRepo(ctx, repoName)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
+		return
+	}
+
+	// If the Git repository contains a top-level go.mod file, allow vanity imports
+	if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
+
+		// Check the first line should be `module MODULENAME\n`
+		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
+		if bytes.HasPrefix(firstLine, []byte("module ")) {
+			moduleName := firstLine[7:]
+			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
+		}
+	}
+
+	// De-escalate all headers in rendered markdown to match our style
+	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
+		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
+
+	// Ready for template
+
+	this.Templatepage(w, r, repoName, extraHead, func() {
+
+		projBodyclass := `projbody`
+		if len(images) > 0 {
+			projBodyclass += ` projbody_halfw`
+		}
+
+		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
+		repl.WriteString(w, string(readmeHtml))
+		fmt.Fprint(w, `</div>`)
+
+		if len(images) > 0 {
+			fmt.Fprint(w, `<div class="projimg">`)
+			for _, img := range images {
+				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
+			}
+			fmt.Fprint(w, `</div>`)
+		}
+
+		fmt.Fprint(w, `<div style="clear:both;"></div>`)
+		fmt.Fprint(w, `</div>`) // projbody
+	})
+}

router.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method == `GET` {
+		if r.URL.Path == `/` {
+			this.Homepage(w, r)
+
+		} else if r.URL.Path == `/favicon.ico` {
+			w.Header().Set(`Location`, `/static/logo.png`)
+			w.WriteHeader(301)
+
+		} else if r.URL.Path == `/robots.txt` {
+			http.Error(w, "not found", 404)
+
+		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
+			this.Bannerpage(w, r, parts[1])
+
+		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
+
+			// Support /repo.html URIs for backward compatibility
+			if strings.HasSuffix(parts[1], `.html`) {
+				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
+				w.WriteHeader(301)
+				return
+			}
+
+			// The regexp supports an optional trailing slash
+			// Redirect to canonical no-trailing-slash
+			if strings.HasSuffix(r.URL.Path, `/`) {
+				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
+				w.WriteHeader(301)
+				return
+			}
+
+			// Proper decoding of special characters in repo path component
+			repoName, err := url.PathUnescape(parts[1])
+			if err != nil {
+				http.Error(w, "malformed url encoding in repository name", 400)
+				return
+			}
+
+			// Maybe it's a redirected project (alternative name)
+			if rename, ok := this.cfg.Redirect[repoName]; ok {
+				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
+				w.WriteHeader(301)
+				return
+			}
+
+			this.Repopage(w, r, repoName)
+
+		} else if r.URL.Path == `/static/logo.png` {
+			if this.cfg.Template.CustomLogoPngBase64 != "" {
+
+				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
+				if err != nil {
+					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
+					return
+				}
+
+				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
+				w.Header().Set(`Content-Type`, `image/png`)
+				w.WriteHeader(200)
+				w.Write(logoPng)
+
+			} else {
+				r.URL.Path = r.URL.Path[8:]
+				http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
+
+			}
+
+		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
+			r.URL.Path = r.URL.Path[8:]
+			http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
+
+		} else if r.URL.Query().Get("go-get") == "1" {
+			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
+			// It must be a subpackage request
+			// We can't serve the proper go-import meta tag immediately because
+			// we haven't looked up the go.mod yet. Just redirect to the root
+			// package - `go get` will follow redirects and the resulting meta tag is correct
+
+			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
+
+			w.Header().Set(`Location`, `/`+slashParts[1])
+			w.WriteHeader(301)
+			return
+
+		} else {
+			http.Error(w, "not found", 404)
+		}
+
+	} else {
+		http.Error(w, "invalid method", 400)
+	}
+
+}

sync.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"context"
+	"log"
+	"sort"
+	"time"
+)
+
+func (this *Application) sync(ctx context.Context) (bool, error) {
+
+	// List repositories on Gitea
+	repos, err := this.repos(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	// Compare this list of repositories to our existing one
+	// If the repository is new, or if it's update-time has changed since we last
+	// saw it, then re-refresh its real git commit timestamps
+	// Otherwise copy them from the previous version
+
+	this.reposMut.RLock() // readonly
+
+	anyChanges := false
+	if len(repos) != len(this.reposCache) {
+		anyChanges = true
+	}
+
+	for i, rr := range repos {
+		if idx, ok := this.reposAlphabeticalOrder[rr.Name]; ok && this.reposCache[idx].GiteaUpdated == rr.GiteaUpdated {
+			// Already exists in cache with same Gitea update time
+			// Copy timestamps
+			repos[i] = this.reposCache[idx]
+
+		} else {
+
+			// New repo, or Gitea has updated timestamp
+			anyChanges = true
+
+			// Refresh timestamps
+			this.populateCommitInfo(ctx, &rr)
+
+			// Refresh topics
+			if t, err := this.topicsForRepo(ctx, rr.Name); err == nil {
+				rr.topics = t
+			}
+
+			// Save
+			repos[i] = rr
+		}
+	}
+
+	this.reposMut.RUnlock()
+
+	//
+	if !anyChanges {
+		return false, nil // nothing to do
+	}
+
+	// We have a final updated repos array
+
+	// Sort repos once alphabetically, to get alphabetical indexes...
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	alphabeticalOrderIndexes := make(map[string]int, len(repos))
+	for idx, repo := range repos {
+		alphabeticalOrderIndexes[repo.Name] = idx
+	}
+
+	// But then make sure the final sort is by most-recently-created
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GiteaCreated.After(repos[j].GiteaCreated)
+	})
+
+	// Commit our changes for the other threads to look at
+	this.reposMut.Lock()
+	this.reposCache = repos
+	this.reposAlphabeticalOrder = alphabeticalOrderIndexes
+	this.reposMut.Unlock()
+
+	// Done
+	return true, nil
+}
+
+func (this *Application) syncWorker(ctx context.Context) {
+
+	t := time.NewTicker(30 * time.Minute)
+	defer t.Stop()
+
+	for {
+		anyChanges, err := this.sync(ctx)
+		if err != nil {
+			// log and continue
+			log.Printf("Refreshing repositories: %s", err.Error())
+		}
+		if anyChanges {
+			log.Printf("Repositories updated")
+		}
+
+		select {
+		case <-t.C:
+			continue
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}