doc/changelog: v1.2.1

For codesite-migrated repositories, looking at the oldest commit is
preferable to determine the "create date". But for forked projects,
looking at the oldest commit is incorrect for when we started the project
If inferring the real create date has to be manual, then let's rely on
the Gitea metadata - it's faster and can be modified by hand if needed

README.md

@@ -28,3 +28,31 @@ By default, Dokku will proxy HTTP on port 5000.
 dokku apps:create teafolio
 dokku storage:mount teafolio /srv/teafolio-dokku/config.toml:/app/config.toml
 ```
+
+## CHANGELOG
+
+2021-04-12 v1.2.1
+- Exclude 'build:success' tag from article-only repositories, and re-host the image locally
+- Also search root repo directory for images
+- Enhance mobile viewport and table styling
+
+2020-11-19 v1.2.0
+- Cache homepage repositories, sync changes in the background
+- Consider the updated time to be the most recent commit, not the Gitea repository metadata update field
+- Add extra logging to startup
+
+2020-11-08 v1.1.1
+- Fix an issue with newer versions of Gitea that paginate repoistory list responses
+- Fix an issue with blocking semaphores for a cancelled network request
+
+2020-05-24 v1.1.0
+- Support limiting the number of concurrent API requests to Gitea
+- Display total number of projects
+- Fix cosmetic issues with page background image, page height, and margins around thumbnails
+
+2020-05-24 v1.0.1
+- Remove image dependency from static files
+- Fix a cosmetic issue with `h2`/`h3` margins
+
+2020-05-05 v1.0.0
+- Initial release

api.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"strings"
@@ -12,10 +14,37 @@ import (
 )
 
 type Repo struct {
-	Name        string    `json:"name"`
-	Description string    `json:"description"`
-	CreatedAt   time.Time `json:"created_at"`
-	UpdatedAt   time.Time `json:"updated_at"`
+	Name         string    `json:"name"`
+	Description  string    `json:"description"`
+	GiteaCreated time.Time `json:"created_at"`
+	GiteaUpdated time.Time `json:"updated_at"`
+
+	newestCommit time.Time
+	topics       []string
+}
+
+func (this *Application) populateCommitInfo(ctx context.Context, rr *Repo) {
+
+	// The most recent commit will be the head of one of the branches (easy to find)
+
+	brs, err := this.branches(ctx, rr.Name)
+	if err != nil {
+		log.Printf("loading branches for '%s': %s", rr.Name, err)
+		rr.newestCommit = rr.GiteaUpdated // best guess
+
+	} else {
+
+		newestCommit := time.Unix(0, 0) // sentinel
+		for _, br := range brs {
+			if br.Commit.Timestamp.After(newestCommit) {
+				newestCommit = br.Commit.Timestamp
+			}
+		}
+		if !newestCommit.Equal(time.Unix(0, 0)) {
+			rr.newestCommit = newestCommit // replace it
+		}
+	}
+
 }
 
 type ContentsResponse struct {
@@ -44,20 +73,116 @@ type MarkdownRequest struct {
 	Wiki    bool
 }
 
-// repos gets a list of Git repositories in this organisation.
-func (this *Application) repos() ([]Repo, error) {
-	resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/orgs/` + url.PathEscape(this.cfg.Gitea.Org) + `/repos`)
+type BranchCommit struct {
+	ID        string    `json:"id"`
+	Message   string    `json:"message"`
+	Timestamp time.Time `json:"timestamp"`
+}
+
+type Branch struct {
+	Name   string       `json:"name"`
+	Commit BranchCommit `json:"commit"`
+}
+
+type AuthorInfo struct {
+	Name  string    `json:"name"`
+	Email string    `json:"email"`
+	Date  time.Time `json:"date"`
+}
+
+type CommitListEntryCommit struct {
+	Message   string     `json:"message"`
+	Author    AuthorInfo `json:"author"`
+	Committer AuthorInfo `json:"committer"`
+}
+
+type CommitListEntry struct {
+	ID     string                `json:"sha"`
+	Commit CommitListEntryCommit `json:"commit"`
+}
+
+func (this *Application) apiRequest(ctx context.Context, endpoint string, target interface{}) error {
+	err := this.apiSem.Acquire(ctx, 1)
 	if err != nil {
-		return nil, err
+		return err // e.g. ctx closed
+	}
+	defer this.apiSem.Release(1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+endpoint, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
+		return fmt.Errorf("HTTP %d", resp.StatusCode)
 	}
 
+	err = json.NewDecoder(resp.Body).Decode(target)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (this *Application) branches(ctx context.Context, repo string) ([]Branch, error) {
+	var branches []Branch
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/branches`, &branches)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return branches, nil
+}
+
+func (this *Application) commitsPage(ctx context.Context, repo, ref string, page, limit int) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	err := this.apiRequest(ctx, fmt.Sprintf(`api/v1/repos/%s/%s/commits?page=%d&limit=%d`, url.PathEscape(this.cfg.Gitea.Org), url.PathEscape(repo), page, limit), &ret)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return ret, nil
+}
+
+func (this *Application) commits(ctx context.Context, repo, ref string) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := this.commitsPage(ctx, repo, ref, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(page) == 0 && len(ret) > 0 {
+			break // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+
+	if len(ret) == 0 {
+		return nil, fmt.Errorf("no commits found")
+	}
+
+	return ret, nil
+}
+
+// reposPage gets a single page of the list of Git repositories in this organisation.
+func (this *Application) reposPage(ctx context.Context, page, limit int) ([]Repo, error) {
 	var repos []Repo
-	err = json.NewDecoder(resp.Body).Decode(&repos)
+
+	err := this.apiRequest(ctx, `api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), &repos)
 	if err != nil {
 		return nil, err
 	}
@@ -65,21 +190,38 @@ func (this *Application) repos() ([]Repo, error) {
 	return repos, nil
 }
 
+// repos gets a list of Git repositories in this organisation. It may have to
+// make multiple network requests.
+func (this *Application) repos(ctx context.Context) ([]Repo, error) {
+	// Seems like gitea-1.13.0-rc1 returns 30 results by default, and supports up to a limit of 100
+	// Make a much larger request
+
+	ret := make([]Repo, 0)
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := this.reposPage(ctx, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(page) == 0 && len(ret) > 0 {
+			break // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+
+	return ret, nil
+}
+
 // repoFile gets a single file from the default branch of the git repository
 // Usually the default branch is `master`.
-func (this *Application) repoFile(repo, filename string) ([]byte, error) {
-	resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/contents/` + url.PathEscape(filename))
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
+func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) {
 	var cr ContentsResponse
-	err = json.NewDecoder(resp.Body).Decode(&cr)
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), &cr)
 	if err != nil {
 		return nil, err
 	}
@@ -87,8 +229,19 @@ func (this *Application) repoFile(repo, filename string) ([]byte, error) {
 	return cr.Content, nil
 }
 
-func (this *Application) filesInDirectory(repo, dir string) ([]ReaddirEntry, error) {
-	resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/contents/` + dir) // n.b. $dir param not escaped
+func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) {
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer this.apiSem.Release(1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -120,14 +273,14 @@ func (this *Application) filesInDirectory(repo, dir string) ([]ReaddirEntry, err
 }
 
 // imageFilesForRepo finds documentation images for the repository.
-// It searches the dist/ and doc/ subdirectories.
-func (this *Application) imageFilesForRepo(repo string) ([]ReaddirEntry, error) {
+// It searches the top-level directory and the dist/ and doc/ subdirectories.
+func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]ReaddirEntry, error) {
 
 	ret := []ReaddirEntry{}
 
-	for _, dirName := range []string{`dist`, `doc`} {
+	for _, dirName := range []string{``, `dist`, `doc`} {
 
-		files, err := this.filesInDirectory(repo, dirName)
+		files, err := this.filesInDirectory(ctx, repo, dirName)
 		if err != nil {
 			return nil, fmt.Errorf("readdir(%s): %w", dirName, err)
 		}
@@ -142,19 +295,10 @@ func (this *Application) imageFilesForRepo(repo string) ([]ReaddirEntry, error)
 	return ret, nil
 }
 
-func (this *Application) topicsForRepo(repo string) ([]string, error) {
-	resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/topics`)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
+func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
 	var tr TopicsResponse
-	err = json.NewDecoder(resp.Body).Decode(&tr)
+
+	err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, &tr)
 	if err != nil {
 		return nil, err
 	}
@@ -163,19 +307,30 @@ func (this *Application) topicsForRepo(repo string) ([]string, error) {
 }
 
 // renderMarkdown calls the remote Gitea server's own markdown renderer.
-func (this *Application) renderMarkdown(repoName string, body string) ([]byte, error) {
-	req := MarkdownRequest{
+func (this *Application) renderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) {
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer this.apiSem.Release(1)
+
+	jb, err := json.Marshal(MarkdownRequest{
 		Context: this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) + `/src/branch/master`,
 		Mode:    "gfm", // magic constant - Github Flavoured Markdown
 		Text:    body,
-	}
-
-	jb, err := json.Marshal(req)
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	resp, err := http.Post(this.cfg.Gitea.URL+`api/v1/markdown/`, `application/json`, bytes.NewReader(jb))
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/`, bytes.NewReader(jb))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set(`Content-Type`, `application/json`)
+	req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(jb)))
+
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -189,8 +344,21 @@ func (this *Application) renderMarkdown(repoName string, body string) ([]byte, e
 }
 
 // renderMarkdownRaw calls the remote Gitea server's own markdown renderer.
-func (this *Application) renderMarkdownRaw(body []byte) ([]byte, error) {
-	resp, err := http.Post(this.cfg.Gitea.URL+`api/v1/markdown/raw`, `text/plain`, bytes.NewReader(body))
+func (this *Application) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) {
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer this.apiSem.Release(1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/raw`, bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set(`Content-Type`, `text/plain`)
+	req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(body)))
+
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}

config.toml.sample

@@ -5,6 +5,7 @@ BindTo="0.0.0.0:5656"
 [Gitea]
 URL="https://gitea.com/"
 Org="gitea"
+MaxConnections=2 # Use zero for unlimited
 
 [Redirect]
 "old-project-name" = "new-project-name"

go.mod

@@ -2,4 +2,7 @@ module teafolio
 
 go 1.13
 
-require github.com/BurntSushi/toml v0.3.1
+require (
+	github.com/BurntSushi/toml v0.3.1
+	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+)

go.sum

@@ -1,2 +1,4 @@
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

main.go

@@ -1,25 +1,23 @@
 package main
 
 import (
-	"bytes"
-	"encoding/base64"
+	"context"
 	"flag"
-	"fmt"
-	"html"
 	"log"
 	"net/http"
-	"net/url"
 	"regexp"
-	"sort"
 	"strings"
+	"sync"
 
 	"github.com/BurntSushi/toml"
+	"golang.org/x/sync/semaphore"
 )
 
 type Config struct {
 	BindTo string
 	Gitea  struct {
-		URL, Org string
+		URL, Org       string
+		MaxConnections int64
 	}
 	Redirect map[string]string
 	Template struct {
@@ -33,336 +31,11 @@ type Application struct {
 	cfg Config
 
 	rxRepoPage, rxRepoImage *regexp.Regexp
-}
-
-func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
-
-	pageTitle := this.cfg.Template.AppName
-	if pageDesc != "" {
-		pageTitle = pageDesc + ` | ` + pageTitle
-	}
-
-	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
-	w.WriteHeader(200)
-	fmt.Fprint(w, `<!DOCTYPE html>
-<html lang="en">
-	<head>
-		<meta http-equiv="X-UA-Compatible" content="IE=edge">
-		<meta charset="UTF-8">
-		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-		<meta name="viewport" content="width=960">
-		<title>`+html.EscapeString(pageTitle)+`</title>
-		`+extraHead+`
-		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
-		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
-		<link type="text/css" rel="stylesheet" href="/static/style.css">
-	</head>
-	<body>
-		<div id="container">
-			<div id="content">
-				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
-	`)
-	cb()
-	fmt.Fprint(w, `
-	</body>
-	<script type="text/javascript" src="/static/site.js"></script>
-</html>
-`)
-
-}
-
-func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
-	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
-	http.Error(w, "An internal error occurred.", 500)
-}
-
-func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
-	repos, err := this.repos()
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing repos: %w", err))
-		return
-	}
-
-	topics := make(map[string][]string)
-	for _, repo := range repos {
-		if t, err := this.topicsForRepo(repo.Name); err == nil {
-			topics[repo.Name] = t
-		}
-	}
-
-	// Sort repos once alphabetically, to get alphabetical indexes...
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].Name < repos[j].Name
-	})
-	alphabeticalOrderIndexes := make(map[string]int, len(repos))
-	for idx, repo := range repos {
-		alphabeticalOrderIndexes[repo.Name] = idx
-	}
-
-	// But then make sure the final sort is by most-recently-created
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].CreatedAt.After(repos[j].CreatedAt)
-	})
-
-	// Ready for template
-
-	this.Templatepage(w, r, "", "", func() {
-		fmt.Fprint(w, `
-		`+this.cfg.Template.HomepageHeaderHTML+`		
-		<select id="sortorder" style="float:right;">
-			<option value="data-sort-al">Alphabetical</option>
-			<option value="data-sort-ls">Lifespan</option>
-			<option value="data-sort-ct" selected>Recent projects</option>
-			<option value="data-sort-mt">Recent updates</option>
-		</select>
-		
-		<h2>Projects</h2>
-		<table id="projtable-main" class="projtable">
-`)
-		for _, repo := range repos {
-			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
-
-			normalisedDesc := repo.Description
-			normalisedDesc = strings.TrimRight(repo.Description, `.`)
-			if len(normalisedDesc) > 0 {
-				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
-				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
-				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
-
-				if !(isAcronymOrCamelCase || firstWord == `Go`) {
-					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
-				}
-
-				// Add leading `<COMMA><SPACE>` to separate from the repo title
-				normalisedDesc = `, ` + normalisedDesc
-			}
-
-			rowClass := ""
-			for _, topic := range topics[repo.Name] {
-				rowClass += `taggedWith-` + topic + ` `
-			}
-
-			fmt.Fprint(w, `
-			<tr
-				class="`+html.EscapeString(rowClass)+`"
-				data-sort-al="`+fmt.Sprintf("-%d", alphabeticalOrderIndexes[repo.Name])+`"
-				data-sort-ls="`+fmt.Sprintf("%.0f", repo.UpdatedAt.Sub(repo.CreatedAt).Seconds())+`"
-				data-sort-ct="`+fmt.Sprintf("%d", repo.CreatedAt.Unix())+`"
-				data-sort-mt="`+fmt.Sprintf("%d", repo.UpdatedAt.Unix())+`"
-			>
-				<td>
-					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
-				</td>
-				<td>
-					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
-					<a href="`+pageHref+`" class="article-read-more">more...</a>
-					<br>
-					<small>
-			`)
-			for _, topic := range topics[repo.Name] {
-				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
-			}
-			fmt.Fprint(w, `
-					</small>
-				</td>
-			</tr>
-			`)
-		}
-		fmt.Fprint(w, `
-		</table>
-		`)
-	})
-}
-
-func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
-
-	images, err := this.imageFilesForRepo(repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	if len(images) == 0 {
-		w.Header().Set(`Location`, `/static/no_image.png`)
-		w.WriteHeader(301)
-		return
-	}
-
-	w.Header().Set(`Location`, images[0].RawURL)
-	w.WriteHeader(301)
-}
-
-func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
-
-	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
-	extraHead := ""
-
-	readme, err := this.repoFile(repoName, `README.md`)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
-		return
-	}
-
-	lines := strings.Split(string(readme), "\n")
-
-	// We add some extra badges based on special text entries
-	extraBadgesMd := ` ![](https://img.shields.io/badge/build-success-brightgreen)`
-	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
-
-	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
-	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
-		lines[2] += ` ` + extraBadgesMd
-	} else {
-		// Push other lines down
-		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
-	}
-
-	readmeHtml, err := this.renderMarkdown(repoName, strings.Join(lines, "\n"))
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
-		return
-	}
-
-	images, err := this.imageFilesForRepo(repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	// If the Git repository contains a top-level go.mod file, allow vanity imports
-	if goMod, err := this.repoFile(repoName, `go.mod`); err == nil {
-
-		// Check the first line should be `module MODULENAME\n`
-		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
-		if bytes.HasPrefix(firstLine, []byte("module ")) {
-			moduleName := firstLine[7:]
-			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
-		}
-	}
-
-	// De-escalate all headers in rendered markdown to match our style
-	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
-		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
-
-	// Ready for template
-
-	this.Templatepage(w, r, repoName, extraHead, func() {
-
-		projBodyclass := `projbody`
-		if len(images) > 0 {
-			projBodyclass += ` projbody_halfw`
-		}
-
-		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
-		repl.WriteString(w, string(readmeHtml))
-		fmt.Fprint(w, `</div>`)
-
-		if len(images) > 0 {
-			fmt.Fprint(w, `<div class="projimg">`)
-			for _, img := range images {
-				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
-			}
-			fmt.Fprint(w, `</div>`)
-		}
-
-		fmt.Fprint(w, `<div style="clear:both;"></div>`)
-		fmt.Fprint(w, `</div>`) // projbody
-	})
-}
-
-func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method == `GET` {
-		if r.URL.Path == `/` {
-			this.Homepage(w, r)
-
-		} else if r.URL.Path == `/favicon.ico` {
-			w.Header().Set(`Location`, `/static/logo.png`)
-			w.WriteHeader(301)
-
-		} else if r.URL.Path == `/robots.txt` {
-			http.Error(w, "not found", 404)
-
-		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
-			this.Bannerpage(w, r, parts[1])
-
-		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
-
-			// Support /repo.html URIs for backward compatibility
-			if strings.HasSuffix(parts[1], `.html`) {
-				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
-				w.WriteHeader(301)
-				return
-			}
-
-			// The regexp supports an optional trailing slash
-			// Redirect to canonical no-trailing-slash
-			if strings.HasSuffix(r.URL.Path, `/`) {
-				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
-				w.WriteHeader(301)
-				return
-			}
-
-			// Proper decoding of special characters in repo path component
-			repoName, err := url.PathUnescape(parts[1])
-			if err != nil {
-				http.Error(w, "malformed url encoding in repository name", 400)
-				return
-			}
-
-			// Maybe it's a redirected project (alternative name)
-			if rename, ok := this.cfg.Redirect[repoName]; ok {
-				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
-				w.WriteHeader(301)
-				return
-			}
-
-			this.Repopage(w, r, repoName)
-
-		} else if r.URL.Path == `/static/logo.png` {
-			if this.cfg.Template.CustomLogoPngBase64 != "" {
-
-				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
-				if err != nil {
-					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
-					return
-				}
-
-				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
-				w.Header().Set(`Content-Type`, `image/png`)
-				w.WriteHeader(200)
-				w.Write(logoPng)
-
-			} else {
-				r.URL.Path = r.URL.Path[8:]
-				http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-			}
-
-		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
-			r.URL.Path = r.URL.Path[8:]
-			http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-		} else if r.URL.Query().Get("go-get") == "1" {
-			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
-			// It must be a subpackage request
-			// We can't serve the proper go-import meta tag immediately because
-			// we haven't looked up the go.mod yet. Just redirect to the root
-			// package - `go get` will follow redirects and the resulting meta tag is correct
-
-			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
-
-			w.Header().Set(`Location`, `/`+slashParts[1])
-			w.WriteHeader(301)
-			return
-
-		} else {
-			http.Error(w, "not found", 404)
-		}
-
-	} else {
-		http.Error(w, "invalid method", 400)
-	}
+	apiSem                  *semaphore.Weighted
 
+	reposMut               sync.RWMutex
+	reposCache             []Repo // Sorted by recently-created-first
+	reposAlphabeticalOrder map[string]int
 }
 
 func main() {
@@ -376,7 +49,7 @@ func main() {
 
 	_, err := toml.DecodeFile(*configFile, &app.cfg)
 	if err != nil {
-		panic(err)
+		log.Fatalf("toml.DecodeFile: %s", err.Error())
 	}
 
 	// Assert Gitea URL always has trailing slash
@@ -384,5 +57,16 @@ func main() {
 		app.cfg.Gitea.URL += `/`
 	}
 
+	// Create semaphore
+	if app.cfg.Gitea.MaxConnections == 0 { // unlimited
+		app.apiSem = semaphore.NewWeighted(99999)
+	} else {
+		app.apiSem = semaphore.NewWeighted(app.cfg.Gitea.MaxConnections)
+	}
+
+	// Sync worker
+	go app.syncWorker(context.Background())
+
+	log.Printf("Starting web server on [%s]...", app.cfg.BindTo)
 	log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app))
 }

pages.go

@@ -0,0 +1,261 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
+
+	pageTitle := this.cfg.Template.AppName
+	if pageDesc != "" {
+		pageTitle = pageDesc + ` | ` + pageTitle
+	}
+
+	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
+	w.WriteHeader(200)
+	fmt.Fprint(w, `<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta charset="UTF-8">
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+		<title>`+html.EscapeString(pageTitle)+`</title>
+		`+extraHead+`
+		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
+		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
+		<link type="text/css" rel="stylesheet" href="/static/style.css">
+	</head>
+	<body>
+		<div id="container">
+			<div id="content">
+				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
+	`)
+	cb()
+	fmt.Fprint(w, `
+	</body>
+	<script type="text/javascript" src="/static/site.js"></script>
+</html>
+`)
+
+}
+
+func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
+	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
+	http.Error(w, "An internal error occurred.", 500)
+}
+
+func (this *Application) Delay(w http.ResponseWriter, r *http.Request) {
+	this.Templatepage(w, r, "Loading...", "", func() {
+		fmt.Fprintf(w, `
+		<h2>Loading, please wait...</h2>
+		
+		<meta http-equiv="refresh" content="5">
+		`)
+	})
+}
+
+func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
+
+	this.reposMut.RLock()
+	defer this.reposMut.RUnlock()
+
+	if len(this.reposCache) == 0 {
+		// We haven't loaded the repositories from Gitea yet
+		this.Delay(w, r)
+		return
+	}
+
+	// Ready for template
+
+	this.Templatepage(w, r, "", "", func() {
+		fmt.Fprint(w, `
+		`+this.cfg.Template.HomepageHeaderHTML+`		
+		<select id="sortorder" style="float:right;">
+			<option value="data-sort-al">Alphabetical</option>
+			<option value="data-sort-ls">Lifespan</option>
+			<option value="data-sort-ct" selected>Recent projects</option>
+			<option value="data-sort-mt">Recent updates</option>
+		</select>
+		
+		<h2>Projects <small>(`+fmt.Sprintf("%d", len(this.reposCache))+`)</small></h2>
+		<table id="projtable-main" class="projtable">
+`)
+		for _, repo := range this.reposCache {
+			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
+
+			normalisedDesc := repo.Description
+			normalisedDesc = strings.TrimRight(repo.Description, `.`)
+			if len(normalisedDesc) > 0 {
+				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
+				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
+				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
+
+				if !(isAcronymOrCamelCase || firstWord == `Go`) {
+					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
+				}
+
+				// Add leading `<COMMA><SPACE>` to separate from the repo title
+				normalisedDesc = `, ` + normalisedDesc
+			}
+
+			rowClass := ""
+			for _, topic := range repo.topics {
+				rowClass += `taggedWith-` + topic + ` `
+			}
+
+			fmt.Fprint(w, `
+			<tr
+				class="`+html.EscapeString(rowClass)+`"
+				data-sort-al="`+fmt.Sprintf("-%d", this.reposAlphabeticalOrder[repo.Name])+`"
+				data-sort-ls="`+fmt.Sprintf("%.0f", repo.newestCommit.Sub(repo.GiteaCreated).Seconds())+`"
+				data-sort-ct="`+fmt.Sprintf("%d", repo.GiteaCreated.Unix())+`"
+				data-sort-mt="`+fmt.Sprintf("%d", repo.newestCommit.Unix())+`"
+			>
+				<td>
+					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
+				</td>
+				<td>
+					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
+					<a href="`+pageHref+`" class="article-read-more">more...</a>
+					<br>
+					<small>
+			`)
+			for _, topic := range repo.topics {
+				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
+			}
+			fmt.Fprint(w, `
+					</small>
+				</td>
+			</tr>
+			`)
+		}
+		fmt.Fprint(w, `
+		</table>
+		`)
+	})
+}
+
+func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
+
+	images, err := this.imageFilesForRepo(ctx, repoName)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
+		return
+	}
+
+	if len(images) == 0 {
+		w.Header().Set(`Location`, `/static/no_image.png`)
+		w.WriteHeader(301)
+		return
+	}
+
+	w.Header().Set(`Location`, images[0].RawURL)
+	w.WriteHeader(301)
+}
+
+func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
+	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
+	extraHead := ""
+
+	readme, err := this.repoFile(ctx, repoName, `README.md`)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
+		return
+	}
+
+	lines := strings.Split(string(readme), "\n")
+
+	// Check if this repo has the 'article' tag
+	hasArticleTag := false
+	this.reposMut.RLock()
+	for _, rr := range this.reposCache {
+		if rr.Name != repoName {
+			continue
+		}
+		for _, topic := range rr.topics {
+			if topic == "article" {
+				hasArticleTag = true
+				break
+			}
+		}
+	}
+	this.reposMut.RUnlock()
+
+	// We add some extra badges based on special text entries
+	extraBadgesMd := ``
+	if !hasArticleTag {
+		extraBadgesMd += ` %%REPLACEME__BADGE%%`
+	}
+	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
+
+	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
+	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
+		lines[2] += ` ` + extraBadgesMd
+	} else {
+		// Push other lines down
+		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
+	}
+
+	readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
+		return
+	}
+
+	readmeHtml = []byte(strings.Replace(string(readmeHtml), `%%REPLACEME__BADGE%%`, `<img src="/static/build_success_brightgreen.svg" style="width:90px;height:20px;">`, 1))
+
+	images, err := this.imageFilesForRepo(ctx, repoName)
+	if err != nil {
+		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
+		return
+	}
+
+	// If the Git repository contains a top-level go.mod file, allow vanity imports
+	if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
+
+		// Check the first line should be `module MODULENAME\n`
+		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
+		if bytes.HasPrefix(firstLine, []byte("module ")) {
+			moduleName := firstLine[7:]
+			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
+		}
+	}
+
+	// De-escalate all headers in rendered markdown to match our style
+	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
+		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
+
+	// Ready for template
+
+	this.Templatepage(w, r, repoName, extraHead, func() {
+
+		projBodyclass := `projbody`
+		if len(images) > 0 {
+			projBodyclass += ` projbody_halfw`
+		}
+
+		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
+		repl.WriteString(w, string(readmeHtml))
+		fmt.Fprint(w, `</div>`)
+
+		if len(images) > 0 {
+			fmt.Fprint(w, `<div class="projimg">`)
+			for _, img := range images {
+				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
+			}
+			fmt.Fprint(w, `</div>`)
+		}
+
+		fmt.Fprint(w, `<div style="clear:both;"></div>`)
+		fmt.Fprint(w, `</div>`) // projbody
+	})
+}

router.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method == `GET` {
+		if r.URL.Path == `/` {
+			this.Homepage(w, r)
+
+		} else if r.URL.Path == `/favicon.ico` {
+			w.Header().Set(`Location`, `/static/logo.png`)
+			w.WriteHeader(301)
+
+		} else if r.URL.Path == `/robots.txt` {
+			http.Error(w, "not found", 404)
+
+		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
+			this.Bannerpage(w, r, parts[1])
+
+		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
+
+			// Support /repo.html URIs for backward compatibility
+			if strings.HasSuffix(parts[1], `.html`) {
+				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
+				w.WriteHeader(301)
+				return
+			}
+
+			// The regexp supports an optional trailing slash
+			// Redirect to canonical no-trailing-slash
+			if strings.HasSuffix(r.URL.Path, `/`) {
+				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
+				w.WriteHeader(301)
+				return
+			}
+
+			// Proper decoding of special characters in repo path component
+			repoName, err := url.PathUnescape(parts[1])
+			if err != nil {
+				http.Error(w, "malformed url encoding in repository name", 400)
+				return
+			}
+
+			// Maybe it's a redirected project (alternative name)
+			if rename, ok := this.cfg.Redirect[repoName]; ok {
+				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
+				w.WriteHeader(301)
+				return
+			}
+
+			this.Repopage(w, r, repoName)
+
+		} else if r.URL.Path == `/static/logo.png` {
+			if this.cfg.Template.CustomLogoPngBase64 != "" {
+
+				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
+				if err != nil {
+					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
+					return
+				}
+
+				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
+				w.Header().Set(`Content-Type`, `image/png`)
+				w.WriteHeader(200)
+				w.Write(logoPng)
+
+			} else {
+				r.URL.Path = r.URL.Path[8:]
+				http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
+
+			}
+
+		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
+			r.URL.Path = r.URL.Path[8:]
+			http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
+
+		} else if r.URL.Query().Get("go-get") == "1" {
+			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
+			// It must be a subpackage request
+			// We can't serve the proper go-import meta tag immediately because
+			// we haven't looked up the go.mod yet. Just redirect to the root
+			// package - `go get` will follow redirects and the resulting meta tag is correct
+
+			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
+
+			w.Header().Set(`Location`, `/`+slashParts[1])
+			w.WriteHeader(301)
+			return
+
+		} else {
+			http.Error(w, "not found", 404)
+		}
+
+	} else {
+		http.Error(w, "invalid method", 400)
+	}
+
+}

static/build_success_brightgreen.svg

@@ -0,0 +1,2 @@
+
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="90" height="20" role="img" aria-label="build: success"><title>build: success</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="90" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="37" height="20" fill="#555"/><rect x="37" width="53" height="20" fill="#4c1"/><rect width="90" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="195" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="270">build</text><text x="195" y="140" transform="scale(.1)" fill="#fff" textLength="270">build</text><text aria-hidden="true" x="625" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">success</text><text x="625" y="140" transform="scale(.1)" fill="#fff" textLength="430">success</text></g></svg>

static/style.css

@@ -25,7 +25,7 @@ h1 a {
 h1 a:hover {
 	color:black;
 }
-h1,h2,h3 {
+h1 {
 	margin-top:0;
 }
 .code {
@@ -48,8 +48,7 @@ h1,h2,h3 {
 
 html, body {
 	/* structural */
-	height:100%;
-	min-height:100%;
+	min-height:100vh;
 	margin:0;
 	border:0;
 	padding:0;
@@ -58,18 +57,23 @@ html, body {
 	font-family:"Helvetica Neue","Segoe UI",Arial,sans-serif;
 	font-size:13px;
 	line-height:1.4;
-	background:#DDD url('greyzz.png');	/* thanks subtlepatterns.com ! */
 	color:#333;
 }
 
+/* Create background pattern by layering two gradients */
+html {
+	background: repeating-linear-gradient(45deg, #FFF, #f0f0f0 3px, #fff 6px);
+}
+body {
+	background: repeating-linear-gradient(135deg, rgba(255,255,255,0), rgba(255,255,255,255) 3px, rgba(255,255,255,0) 6px);
+}
+
 #container {
 	margin:0 auto;
 	width:960px;
 	position:relative;
 	
-	height:auto !important;
-	height:100%; /* oldIE */
-	min-height:100%;
+	min-height:100vh;
 	
 	/* cosmetic */
 	background:white;
@@ -137,6 +141,9 @@ html, body {
 	float:left;
 	width: 860px; /* 740px full - 60px rhs column - 2px border */
 }
+.projbody_halfw h2:first-child {
+	margin-top: 0;
+}
 .projbody_fullw {
 	
 }
@@ -147,6 +154,32 @@ html, body {
 
 /* */
 
+img[src*="shields.io"] {
+	width: auto;
+	height: 20px; /* Set default height to avoid reflow flashing */
+}
+
+/* */
+
+.projbody table {
+	width: 100%;
+}
+
+.projbody table {
+    border-collapse: collapse;
+}
+
+.projbody table td, .projbody table th {
+	border: 1px solid #eee;
+    padding: 4px;
+}
+
+.projbody tr:hover td {
+    background: #f8f8f8;
+}
+
+/* */
+
 @media screen and (max-width:960px) {
 	
 	#container {

sync.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"context"
+	"log"
+	"sort"
+	"time"
+)
+
+func (this *Application) sync(ctx context.Context) (bool, error) {
+
+	// List repositories on Gitea
+	repos, err := this.repos(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	// Compare this list of repositories to our existing one
+	// If the repository is new, or if it's update-time has changed since we last
+	// saw it, then re-refresh its real git commit timestamps
+	// Otherwise copy them from the previous version
+
+	this.reposMut.RLock() // readonly
+
+	anyChanges := false
+	if len(repos) != len(this.reposCache) {
+		anyChanges = true
+	}
+
+	for i, rr := range repos {
+		if idx, ok := this.reposAlphabeticalOrder[rr.Name]; ok && this.reposCache[idx].GiteaUpdated == rr.GiteaUpdated {
+			// Already exists in cache with same Gitea update time
+			// Copy timestamps
+			repos[i] = this.reposCache[idx]
+
+		} else {
+
+			// New repo, or Gitea has updated timestamp
+			anyChanges = true
+
+			// Refresh timestamps
+			this.populateCommitInfo(ctx, &rr)
+
+			// Refresh topics
+			if t, err := this.topicsForRepo(ctx, rr.Name); err == nil {
+				rr.topics = t
+			}
+
+			// Save
+			repos[i] = rr
+		}
+	}
+
+	this.reposMut.RUnlock()
+
+	//
+	if !anyChanges {
+		return false, nil // nothing to do
+	}
+
+	// We have a final updated repos array
+
+	// Sort repos once alphabetically, to get alphabetical indexes...
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	alphabeticalOrderIndexes := make(map[string]int, len(repos))
+	for idx, repo := range repos {
+		alphabeticalOrderIndexes[repo.Name] = idx
+	}
+
+	// But then make sure the final sort is by most-recently-created
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GiteaCreated.After(repos[j].GiteaCreated)
+	})
+
+	// Commit our changes for the other threads to look at
+	this.reposMut.Lock()
+	this.reposCache = repos
+	this.reposAlphabeticalOrder = alphabeticalOrderIndexes
+	this.reposMut.Unlock()
+
+	// Done
+	return true, nil
+}
+
+func (this *Application) syncWorker(ctx context.Context) {
+
+	t := time.NewTicker(30 * time.Minute)
+	defer t.Stop()
+
+	for {
+		anyChanges, err := this.sync(ctx)
+		if err != nil {
+			// log and continue
+			log.Printf("Refreshing repositories: %s", err.Error())
+		}
+		if anyChanges {
+			log.Printf("Repositories updated")
+		}
+
+		select {
+		case <-t.C:
+			continue
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}