doc/changelog: v1.2.0

For codesite-migrated repositories, looking at the oldest commit is
preferable to determine the "create date". But for forked projects,
looking at the oldest commit is incorrect for when we started the project
If inferring the real create date has to be manual, then let's rely on
the Gitea metadata - it's faster and can be modified by hand if needed

README.md

@@ -31,6 +31,15 @@ dokku storage:mount teafolio /srv/teafolio-dokku/config.toml:/app/config.toml
 
 ## CHANGELOG
 
+2020-11-19 v1.2.0
+- Cache homepage repositories, sync changes in the background
+- Consider the updated time to be the most recent commit, not the Gitea repository metadata update field
+- Add extra logging to startup
+
+2020-11-08 v1.1.1
+- Fix an issue with newer versions of Gitea that paginate repoistory list responses
+- Fix an issue with blocking semaphores for a cancelled network request
+
 2020-05-24 v1.1.0
 - Support limiting the number of concurrent API requests to Gitea
 - Display total number of projects

api.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"strings"
@@ -15,8 +16,35 @@ import (
 type Repo struct {
 	Name         string    `json:"name"`
 	Description  string    `json:"description"`
-	CreatedAt   time.Time `json:"created_at"`
+	GiteaCreated time.Time `json:"created_at"`
-	UpdatedAt   time.Time `json:"updated_at"`
+	GiteaUpdated time.Time `json:"updated_at"`
+
+	newestCommit time.Time
+	topics       []string
+}
+
+func (this *Application) populateCommitInfo(ctx context.Context, rr *Repo) {
+
+	// The most recent commit will be the head of one of the branches (easy to find)
+
+	brs, err := this.branches(ctx, rr.Name)
+	if err != nil {
+		log.Printf("loading branches for '%s': %s", rr.Name, err)
+		rr.newestCommit = rr.GiteaUpdated // best guess
+
+	} else {
+
+		newestCommit := time.Unix(0, 0) // sentinel
+		for _, br := range brs {
+			if br.Commit.Timestamp.After(newestCommit) {
+				newestCommit = br.Commit.Timestamp
+			}
+		}
+		if !newestCommit.Equal(time.Unix(0, 0)) {
+			rr.newestCommit = newestCommit // replace it
+		}
+	}
+
 }
 
 type ContentsResponse struct {
@@ -45,28 +73,116 @@ type MarkdownRequest struct {
 	Wiki    bool
 }
 
-// repos gets a list of Git repositories in this organisation.
+type BranchCommit struct {
-func (this *Application) repos(ctx context.Context) ([]Repo, error) {
+	ID        string    `json:"id"`
-	this.apiSem.Acquire(ctx, 1)
+	Message   string    `json:"message"`
+	Timestamp time.Time `json:"timestamp"`
+}
+
+type Branch struct {
+	Name   string       `json:"name"`
+	Commit BranchCommit `json:"commit"`
+}
+
+type AuthorInfo struct {
+	Name  string    `json:"name"`
+	Email string    `json:"email"`
+	Date  time.Time `json:"date"`
+}
+
+type CommitListEntryCommit struct {
+	Message   string     `json:"message"`
+	Author    AuthorInfo `json:"author"`
+	Committer AuthorInfo `json:"committer"`
+}
+
+type CommitListEntry struct {
+	ID     string                `json:"sha"`
+	Commit CommitListEntryCommit `json:"commit"`
+}
+
+func (this *Application) apiRequest(ctx context.Context, endpoint string, target interface{}) error {
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+`/repos`, nil)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+endpoint, nil)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
+		return fmt.Errorf("HTTP %d", resp.StatusCode)
 	}
 
+	err = json.NewDecoder(resp.Body).Decode(target)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (this *Application) branches(ctx context.Context, repo string) ([]Branch, error) {
+	var branches []Branch
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/branches`, &branches)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return branches, nil
+}
+
+func (this *Application) commitsPage(ctx context.Context, repo, ref string, page, limit int) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	err := this.apiRequest(ctx, fmt.Sprintf(`api/v1/repos/%s/%s/commits?page=%d&limit=%d`, url.PathEscape(this.cfg.Gitea.Org), url.PathEscape(repo), page, limit), &ret)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return ret, nil
+}
+
+func (this *Application) commits(ctx context.Context, repo, ref string) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := this.commitsPage(ctx, repo, ref, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(page) == 0 && len(ret) > 0 {
+			break // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+
+	if len(ret) == 0 {
+		return nil, fmt.Errorf("no commits found")
+	}
+
+	return ret, nil
+}
+
+// reposPage gets a single page of the list of Git repositories in this organisation.
+func (this *Application) reposPage(ctx context.Context, page, limit int) ([]Repo, error) {
 	var repos []Repo
-	err = json.NewDecoder(resp.Body).Decode(&repos)
+
+	err := this.apiRequest(ctx, `api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), &repos)
 	if err != nil {
 		return nil, err
 	}
@@ -74,29 +190,38 @@ func (this *Application) repos(ctx context.Context) ([]Repo, error) {
 	return repos, nil
 }
 
+// repos gets a list of Git repositories in this organisation. It may have to
+// make multiple network requests.
+func (this *Application) repos(ctx context.Context) ([]Repo, error) {
+	// Seems like gitea-1.13.0-rc1 returns 30 results by default, and supports up to a limit of 100
+	// Make a much larger request
+
+	ret := make([]Repo, 0)
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := this.reposPage(ctx, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(page) == 0 && len(ret) > 0 {
+			break // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+
+	return ret, nil
+}
+
 // repoFile gets a single file from the default branch of the git repository
 // Usually the default branch is `master`.
 func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) {
-	this.apiSem.Acquire(ctx, 1)
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
 	var cr ContentsResponse
-	err = json.NewDecoder(resp.Body).Decode(&cr)
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), &cr)
 	if err != nil {
 		return nil, err
 	}
@@ -105,7 +230,10 @@ func (this *Application) repoFile(ctx context.Context, repo, filename string) ([
 }
 
 func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) {
-	this.apiSem.Acquire(ctx, 1)
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped
@@ -168,26 +296,9 @@ func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]
 }
 
 func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
-	this.apiSem.Acquire(ctx, 1)
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
 	var tr TopicsResponse
-	err = json.NewDecoder(resp.Body).Decode(&tr)
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, &tr)
 	if err != nil {
 		return nil, err
 	}
@@ -197,7 +308,10 @@ func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]stri
 
 // renderMarkdown calls the remote Gitea server's own markdown renderer.
 func (this *Application) renderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) {
-	this.apiSem.Acquire(ctx, 1)
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
 	jb, err := json.Marshal(MarkdownRequest{
@@ -231,7 +345,10 @@ func (this *Application) renderMarkdown(ctx context.Context, repoName string, bo
 
 // renderMarkdownRaw calls the remote Gitea server's own markdown renderer.
 func (this *Application) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) {
-	this.apiSem.Acquire(ctx, 1)
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/raw`, bytes.NewReader(body))

main.go

@@ -1,17 +1,13 @@
 package main
 
 import (
-	"bytes"
+	"context"
-	"encoding/base64"
 	"flag"
-	"fmt"
-	"html"
 	"log"
 	"net/http"
-	"net/url"
 	"regexp"
-	"sort"
 	"strings"
+	"sync"
 
 	"github.com/BurntSushi/toml"
 	"golang.org/x/sync/semaphore"
@@ -36,339 +32,10 @@ type Application struct {
 
 	rxRepoPage, rxRepoImage *regexp.Regexp
 	apiSem                  *semaphore.Weighted
-}
-
-func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
-
-	pageTitle := this.cfg.Template.AppName
-	if pageDesc != "" {
-		pageTitle = pageDesc + ` | ` + pageTitle
-	}
-
-	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
-	w.WriteHeader(200)
-	fmt.Fprint(w, `<!DOCTYPE html>
-<html lang="en">
-	<head>
-		<meta http-equiv="X-UA-Compatible" content="IE=edge">
-		<meta charset="UTF-8">
-		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-		<meta name="viewport" content="width=960">
-		<title>`+html.EscapeString(pageTitle)+`</title>
-		`+extraHead+`
-		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
-		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
-		<link type="text/css" rel="stylesheet" href="/static/style.css">
-	</head>
-	<body>
-		<div id="container">
-			<div id="content">
-				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
-	`)
-	cb()
-	fmt.Fprint(w, `
-	</body>
-	<script type="text/javascript" src="/static/site.js"></script>
-</html>
-`)
-
-}
-
-func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
-	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
-	http.Error(w, "An internal error occurred.", 500)
-}
-
-func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-
-	repos, err := this.repos(ctx)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing repos: %w", err))
-		return
-	}
-
-	topics := make(map[string][]string)
-	for _, repo := range repos {
-		if t, err := this.topicsForRepo(ctx, repo.Name); err == nil {
-			topics[repo.Name] = t
-		}
-	}
-
-	// Sort repos once alphabetically, to get alphabetical indexes...
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].Name < repos[j].Name
-	})
-	alphabeticalOrderIndexes := make(map[string]int, len(repos))
-	for idx, repo := range repos {
-		alphabeticalOrderIndexes[repo.Name] = idx
-	}
-
-	// But then make sure the final sort is by most-recently-created
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].CreatedAt.After(repos[j].CreatedAt)
-	})
-
-	// Ready for template
-
-	this.Templatepage(w, r, "", "", func() {
-		fmt.Fprint(w, `
-		`+this.cfg.Template.HomepageHeaderHTML+`		
-		<select id="sortorder" style="float:right;">
-			<option value="data-sort-al">Alphabetical</option>
-			<option value="data-sort-ls">Lifespan</option>
-			<option value="data-sort-ct" selected>Recent projects</option>
-			<option value="data-sort-mt">Recent updates</option>
-		</select>
-		
-		<h2>Projects <small>(`+fmt.Sprintf("%d", len(repos))+`)</small></h2>
-		<table id="projtable-main" class="projtable">
-`)
-		for _, repo := range repos {
-			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
-
-			normalisedDesc := repo.Description
-			normalisedDesc = strings.TrimRight(repo.Description, `.`)
-			if len(normalisedDesc) > 0 {
-				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
-				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
-				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
-
-				if !(isAcronymOrCamelCase || firstWord == `Go`) {
-					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
-				}
-
-				// Add leading `<COMMA><SPACE>` to separate from the repo title
-				normalisedDesc = `, ` + normalisedDesc
-			}
-
-			rowClass := ""
-			for _, topic := range topics[repo.Name] {
-				rowClass += `taggedWith-` + topic + ` `
-			}
-
-			fmt.Fprint(w, `
-			<tr
-				class="`+html.EscapeString(rowClass)+`"
-				data-sort-al="`+fmt.Sprintf("-%d", alphabeticalOrderIndexes[repo.Name])+`"
-				data-sort-ls="`+fmt.Sprintf("%.0f", repo.UpdatedAt.Sub(repo.CreatedAt).Seconds())+`"
-				data-sort-ct="`+fmt.Sprintf("%d", repo.CreatedAt.Unix())+`"
-				data-sort-mt="`+fmt.Sprintf("%d", repo.UpdatedAt.Unix())+`"
-			>
-				<td>
-					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
-				</td>
-				<td>
-					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
-					<a href="`+pageHref+`" class="article-read-more">more...</a>
-					<br>
-					<small>
-			`)
-			for _, topic := range topics[repo.Name] {
-				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
-			}
-			fmt.Fprint(w, `
-					</small>
-				</td>
-			</tr>
-			`)
-		}
-		fmt.Fprint(w, `
-		</table>
-		`)
-	})
-}
-
-func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
-	ctx := r.Context()
-
-	images, err := this.imageFilesForRepo(ctx, repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	if len(images) == 0 {
-		w.Header().Set(`Location`, `/static/no_image.png`)
-		w.WriteHeader(301)
-		return
-	}
-
-	w.Header().Set(`Location`, images[0].RawURL)
-	w.WriteHeader(301)
-}
-
-func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
-	ctx := r.Context()
-	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
-	extraHead := ""
-
-	readme, err := this.repoFile(ctx, repoName, `README.md`)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
-		return
-	}
-
-	lines := strings.Split(string(readme), "\n")
-
-	// We add some extra badges based on special text entries
-	extraBadgesMd := ` ![](https://img.shields.io/badge/build-success-brightgreen)`
-	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
-
-	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
-	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
-		lines[2] += ` ` + extraBadgesMd
-	} else {
-		// Push other lines down
-		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
-	}
-
-	readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
-		return
-	}
-
-	images, err := this.imageFilesForRepo(ctx, repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	// If the Git repository contains a top-level go.mod file, allow vanity imports
-	if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
-
-		// Check the first line should be `module MODULENAME\n`
-		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
-		if bytes.HasPrefix(firstLine, []byte("module ")) {
-			moduleName := firstLine[7:]
-			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
-		}
-	}
-
-	// De-escalate all headers in rendered markdown to match our style
-	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
-		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
-
-	// Ready for template
-
-	this.Templatepage(w, r, repoName, extraHead, func() {
-
-		projBodyclass := `projbody`
-		if len(images) > 0 {
-			projBodyclass += ` projbody_halfw`
-		}
-
-		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
-		repl.WriteString(w, string(readmeHtml))
-		fmt.Fprint(w, `</div>`)
-
-		if len(images) > 0 {
-			fmt.Fprint(w, `<div class="projimg">`)
-			for _, img := range images {
-				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
-			}
-			fmt.Fprint(w, `</div>`)
-		}
-
-		fmt.Fprint(w, `<div style="clear:both;"></div>`)
-		fmt.Fprint(w, `</div>`) // projbody
-	})
-}
-
-func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method == `GET` {
-		if r.URL.Path == `/` {
-			this.Homepage(w, r)
-
-		} else if r.URL.Path == `/favicon.ico` {
-			w.Header().Set(`Location`, `/static/logo.png`)
-			w.WriteHeader(301)
-
-		} else if r.URL.Path == `/robots.txt` {
-			http.Error(w, "not found", 404)
-
-		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
-			this.Bannerpage(w, r, parts[1])
-
-		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
-
-			// Support /repo.html URIs for backward compatibility
-			if strings.HasSuffix(parts[1], `.html`) {
-				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
-				w.WriteHeader(301)
-				return
-			}
-
-			// The regexp supports an optional trailing slash
-			// Redirect to canonical no-trailing-slash
-			if strings.HasSuffix(r.URL.Path, `/`) {
-				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
-				w.WriteHeader(301)
-				return
-			}
-
-			// Proper decoding of special characters in repo path component
-			repoName, err := url.PathUnescape(parts[1])
-			if err != nil {
-				http.Error(w, "malformed url encoding in repository name", 400)
-				return
-			}
-
-			// Maybe it's a redirected project (alternative name)
-			if rename, ok := this.cfg.Redirect[repoName]; ok {
-				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
-				w.WriteHeader(301)
-				return
-			}
-
-			this.Repopage(w, r, repoName)
-
-		} else if r.URL.Path == `/static/logo.png` {
-			if this.cfg.Template.CustomLogoPngBase64 != "" {
-
-				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
-				if err != nil {
-					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
-					return
-				}
-
-				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
-				w.Header().Set(`Content-Type`, `image/png`)
-				w.WriteHeader(200)
-				w.Write(logoPng)
-
-			} else {
-				r.URL.Path = r.URL.Path[8:]
-				http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-			}
-
-		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
-			r.URL.Path = r.URL.Path[8:]
-			http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-		} else if r.URL.Query().Get("go-get") == "1" {
-			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
-			// It must be a subpackage request
-			// We can't serve the proper go-import meta tag immediately because
-			// we haven't looked up the go.mod yet. Just redirect to the root
-			// package - `go get` will follow redirects and the resulting meta tag is correct
-
-			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
-
-			w.Header().Set(`Location`, `/`+slashParts[1])
-			w.WriteHeader(301)
-			return
-
-		} else {
-			http.Error(w, "not found", 404)
-		}
-
-	} else {
-		http.Error(w, "invalid method", 400)
-	}
 
+	reposMut               sync.RWMutex
+	reposCache             []Repo // Sorted by recently-created-first
+	reposAlphabeticalOrder map[string]int
 }
 
 func main() {
@@ -382,7 +49,7 @@ func main() {
 
 	_, err := toml.DecodeFile(*configFile, &app.cfg)
 	if err != nil {
-		panic(err)
+		log.Fatalf("toml.DecodeFile: %s", err.Error())
 	}
 
 	// Assert Gitea URL always has trailing slash
@@ -397,5 +64,9 @@ func main() {
 		app.apiSem = semaphore.NewWeighted(app.cfg.Gitea.MaxConnections)
 	}
 
+	// Sync worker
+	go app.syncWorker(context.Background())
+
+	log.Printf("Starting web server on [%s]...", app.cfg.BindTo)
 	log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app))
 }

pages.go

@@ -0,0 +1,240 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
+
+	pageTitle := this.cfg.Template.AppName
+	if pageDesc != "" {
+		pageTitle = pageDesc + ` | ` + pageTitle
+	}
+
+	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
+	w.WriteHeader(200)
+	fmt.Fprint(w, `<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta charset="UTF-8">
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+		<meta name="viewport" content="width=960">
+		<title>`+html.EscapeString(pageTitle)+`</title>
+		`+extraHead+`
+		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
+		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
+		<link type="text/css" rel="stylesheet" href="/static/style.css">
+	</head>
+	<body>
+		<div id="container">
+			<div id="content">
+				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
+	`)
+	cb()
+	fmt.Fprint(w, `
+	</body>
+	<script type="text/javascript" src="/static/site.js"></script>
+</html>
+`)
+
+}
+
+func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
+	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
+	http.Error(w, "An internal error occurred.", 500)
+}
+
+func (this *Application) Delay(w http.ResponseWriter, r *http.Request) {
+	this.Templatepage(w, r, "Loading...", "", func() {
+		fmt.Fprintf(w, `
+		<h2>Loading, please wait...</h2>
+		
+		<meta http-equiv="refresh" content="5">
+		`)
+	})
+}
+
+func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
+
+	this.reposMut.RLock()
+	defer this.reposMut.RUnlock()
+
+	if len(this.reposCache) == 0 {
+		// We haven't loaded the repositories from Gitea yet
+		this.Delay(w, r)
+		return
+	}
+
+	// Ready for template
+
+	this.Templatepage(w, r, "", "", func() {
+		fmt.Fprint(w, `
+		`+this.cfg.Template.HomepageHeaderHTML+`		
+		<select id="sortorder" style="float:right;">
+			<option value="data-sort-al">Alphabetical</option>
+			<option value="data-sort-ls">Lifespan</option>
+			<option value="data-sort-ct" selected>Recent projects</option>
+			<option value="data-sort-mt">Recent updates</option>
+		</select>
+		
+		<h2>Projects <small>(`+fmt.Sprintf("%d", len(this.reposCache))+`)</small></h2>
+		<table id="projtable-main" class="projtable">
+`)
+		for _, repo := range this.reposCache {
+			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
+
+			normalisedDesc := repo.Description
+			normalisedDesc = strings.TrimRight(repo.Description, `.`)
+			if len(normalisedDesc) > 0 {
+				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
+				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
+				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
+
+				if !(isAcronymOrCamelCase || firstWord == `Go`) {
+					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
+				}
+
+				// Add leading `<COMMA><SPACE>` to separate from the repo title
+				normalisedDesc = `, ` + normalisedDesc
+			}
+
+			rowClass := ""
+			for _, topic := range repo.topics {
+				rowClass += `taggedWith-` + topic + ` `
+			}
+
+			fmt.Fprint(w, `
+			<tr
+				class="`+html.EscapeString(rowClass)+`"
+				data-sort-al="`+fmt.Sprintf("-%d", this.reposAlphabeticalOrder[repo.Name])+`"
+				data-sort-ls="`+fmt.Sprintf("%.0f", repo.newestCommit.Sub(repo.GiteaCreated).Seconds())+`"
+				data-sort-ct="`+fmt.Sprintf("%d", repo.GiteaCreated.Unix())+`"
+				data-sort-mt="`+fmt.Sprintf("%d", repo.newestCommit.Unix())+`"
+			>
+				<td>
+					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
+				</td>
+				<td>
+					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
+					<a href="`+pageHref+`" class="article-read-more">more...</a>
+					<br>
+					<small>
+			`)
+			for _, topic := range repo.topics {
+				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
+			}
+			fmt.Fprint(w, `
+					</small>
+				</td>
+			</tr>
+			`)
+		}
+		fmt.Fprint(w, `
+		</table>
+		`)
+	})
+}
+
+func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
+
+	images, err := this.imageFilesForRepo(ctx, repoName)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
+		return
+	}
+
+	if len(images) == 0 {
+		w.Header().Set(`Location`, `/static/no_image.png`)
+		w.WriteHeader(301)
+		return
+	}
+
+	w.Header().Set(`Location`, images[0].RawURL)
+	w.WriteHeader(301)
+}
+
+func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
+	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
+	extraHead := ""
+
+	readme, err := this.repoFile(ctx, repoName, `README.md`)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
+		return
+	}
+
+	lines := strings.Split(string(readme), "\n")
+
+	// We add some extra badges based on special text entries
+	extraBadgesMd := ` ![](https://img.shields.io/badge/build-success-brightgreen)`
+	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
+
+	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
+	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
+		lines[2] += ` ` + extraBadgesMd
+	} else {
+		// Push other lines down
+		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
+	}
+
+	readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
+		return
+	}
+
+	images, err := this.imageFilesForRepo(ctx, repoName)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
+		return
+	}
+
+	// If the Git repository contains a top-level go.mod file, allow vanity imports
+	if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
+
+		// Check the first line should be `module MODULENAME\n`
+		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
+		if bytes.HasPrefix(firstLine, []byte("module ")) {
+			moduleName := firstLine[7:]
+			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
+		}
+	}
+
+	// De-escalate all headers in rendered markdown to match our style
+	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
+		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
+
+	// Ready for template
+
+	this.Templatepage(w, r, repoName, extraHead, func() {
+
+		projBodyclass := `projbody`
+		if len(images) > 0 {
+			projBodyclass += ` projbody_halfw`
+		}
+
+		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
+		repl.WriteString(w, string(readmeHtml))
+		fmt.Fprint(w, `</div>`)
+
+		if len(images) > 0 {
+			fmt.Fprint(w, `<div class="projimg">`)
+			for _, img := range images {
+				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
+			}
+			fmt.Fprint(w, `</div>`)
+		}
+
+		fmt.Fprint(w, `<div style="clear:both;"></div>`)
+		fmt.Fprint(w, `</div>`) // projbody
+	})
+}

router.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method == `GET` {
+		if r.URL.Path == `/` {
+			this.Homepage(w, r)
+
+		} else if r.URL.Path == `/favicon.ico` {
+			w.Header().Set(`Location`, `/static/logo.png`)
+			w.WriteHeader(301)
+
+		} else if r.URL.Path == `/robots.txt` {
+			http.Error(w, "not found", 404)
+
+		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
+			this.Bannerpage(w, r, parts[1])
+
+		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
+
+			// Support /repo.html URIs for backward compatibility
+			if strings.HasSuffix(parts[1], `.html`) {
+				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
+				w.WriteHeader(301)
+				return
+			}
+
+			// The regexp supports an optional trailing slash
+			// Redirect to canonical no-trailing-slash
+			if strings.HasSuffix(r.URL.Path, `/`) {
+				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
+				w.WriteHeader(301)
+				return
+			}
+
+			// Proper decoding of special characters in repo path component
+			repoName, err := url.PathUnescape(parts[1])
+			if err != nil {
+				http.Error(w, "malformed url encoding in repository name", 400)
+				return
+			}
+
+			// Maybe it's a redirected project (alternative name)
+			if rename, ok := this.cfg.Redirect[repoName]; ok {
+				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
+				w.WriteHeader(301)
+				return
+			}
+
+			this.Repopage(w, r, repoName)
+
+		} else if r.URL.Path == `/static/logo.png` {
+			if this.cfg.Template.CustomLogoPngBase64 != "" {
+
+				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
+				if err != nil {
+					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
+					return
+				}
+
+				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
+				w.Header().Set(`Content-Type`, `image/png`)
+				w.WriteHeader(200)
+				w.Write(logoPng)
+
+			} else {
+				r.URL.Path = r.URL.Path[8:]
+				http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
+
+			}
+
+		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
+			r.URL.Path = r.URL.Path[8:]
+			http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
+
+		} else if r.URL.Query().Get("go-get") == "1" {
+			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
+			// It must be a subpackage request
+			// We can't serve the proper go-import meta tag immediately because
+			// we haven't looked up the go.mod yet. Just redirect to the root
+			// package - `go get` will follow redirects and the resulting meta tag is correct
+
+			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
+
+			w.Header().Set(`Location`, `/`+slashParts[1])
+			w.WriteHeader(301)
+			return
+
+		} else {
+			http.Error(w, "not found", 404)
+		}
+
+	} else {
+		http.Error(w, "invalid method", 400)
+	}
+
+}

sync.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"context"
+	"log"
+	"sort"
+	"time"
+)
+
+func (this *Application) sync(ctx context.Context) (bool, error) {
+
+	// List repositories on Gitea
+	repos, err := this.repos(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	// Compare this list of repositories to our existing one
+	// If the repository is new, or if it's update-time has changed since we last
+	// saw it, then re-refresh its real git commit timestamps
+	// Otherwise copy them from the previous version
+
+	this.reposMut.RLock() // readonly
+
+	anyChanges := false
+	if len(repos) != len(this.reposCache) {
+		anyChanges = true
+	}
+
+	for i, rr := range repos {
+		if idx, ok := this.reposAlphabeticalOrder[rr.Name]; ok && this.reposCache[idx].GiteaUpdated == rr.GiteaUpdated {
+			// Already exists in cache with same Gitea update time
+			// Copy timestamps
+			repos[i] = this.reposCache[idx]
+
+		} else {
+
+			// New repo, or Gitea has updated timestamp
+			anyChanges = true
+
+			// Refresh timestamps
+			this.populateCommitInfo(ctx, &rr)
+
+			// Refresh topics
+			if t, err := this.topicsForRepo(ctx, rr.Name); err == nil {
+				rr.topics = t
+			}
+
+			// Save
+			repos[i] = rr
+		}
+	}
+
+	this.reposMut.RUnlock()
+
+	//
+	if !anyChanges {
+		return false, nil // nothing to do
+	}
+
+	// We have a final updated repos array
+
+	// Sort repos once alphabetically, to get alphabetical indexes...
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	alphabeticalOrderIndexes := make(map[string]int, len(repos))
+	for idx, repo := range repos {
+		alphabeticalOrderIndexes[repo.Name] = idx
+	}
+
+	// But then make sure the final sort is by most-recently-created
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GiteaCreated.After(repos[j].GiteaCreated)
+	})
+
+	// Commit our changes for the other threads to look at
+	this.reposMut.Lock()
+	this.reposCache = repos
+	this.reposAlphabeticalOrder = alphabeticalOrderIndexes
+	this.reposMut.Unlock()
+
+	// Done
+	return true, nil
+}
+
+func (this *Application) syncWorker(ctx context.Context) {
+
+	t := time.NewTicker(30 * time.Minute)
+	defer t.Stop()
+
+	for {
+		anyChanges, err := this.sync(ctx)
+		if err != nil {
+			// log and continue
+			log.Printf("Refreshing repositories: %s", err.Error())
+		}
+		if anyChanges {
+			log.Printf("Repositories updated")
+		}
+
+		select {
+		case <-t.C:
+			continue
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}