doc/README: update for v1.4.1

For codesite-migrated repositories, looking at the oldest commit is
preferable to determine the "create date". But for forked projects,
looking at the oldest commit is incorrect for when we started the project
If inferring the real create date has to be manual, then let's rely on
the Gitea metadata - it's faster and can be modified by hand if needed

Dockerfile

@@ -1,6 +1,6 @@
 # Dockerfile for production Teafolio deployments
 
-FROM golang:1.14-alpine AS builder
+FROM golang:1.22-alpine AS builder
 
 WORKDIR /app
 COPY . .
@@ -10,6 +10,5 @@ FROM alpine:latest
 
 WORKDIR /app
 COPY --from=builder /app/teafolio /app/teafolio
-COPY /static /app/static
 
 ENTRYPOINT [ "/app/teafolio" ]

README.md

@@ -31,6 +31,34 @@ dokku storage:mount teafolio /srv/teafolio-dokku/config.toml:/app/config.toml
 
 ## CHANGELOG
 
+2024-03-19 v1.4.1
+- Add cooldown for failed markdown API calls
+- Fix missing extra html elements when using fallback markdown renderer
+
+2024-03-19 v1.4.0
+- Support using application token for Gitea API
+- Add fallback to internal markdown renderer if Gitea's API is unavailable
+- Add fallback to Gitea page redirect if there are any errors building repo page
+- Embed static web assets in binary (requires Go 1.16+)
+
+2022-12-31 v1.3.1
+- Fix missing images on homepage
+- Fix an issue with wrong mtime comparisons for updated repositories
+
+2022-12-31 v1.3.0
+- Add `OverrideOrder` configuration option to insert repos at specific positions
+- Cache target image URLs to improve homepage load performance
+
+2021-04-12 v1.2.1
+- Exclude 'build:success' tag from article-only repositories, and re-host the image locally
+- Also search root repo directory for images
+- Enhance mobile viewport and table styling
+
+2020-11-19 v1.2.0
+- Cache homepage repositories, sync changes in the background
+- Consider the updated time to be the most recent commit, not the Gitea repository metadata update field
+- Add extra logging to startup
+
 2020-11-08 v1.1.1
 - Fix an issue with newer versions of Gitea that paginate repoistory list responses
 - Fix an issue with blocking semaphores for a cancelled network request

api.go

@@ -1,297 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strings"
-	"time"
-)
-
-type Repo struct {
-	Name        string    `json:"name"`
-	Description string    `json:"description"`
-	CreatedAt   time.Time `json:"created_at"`
-	UpdatedAt   time.Time `json:"updated_at"`
-}
-
-type ContentsResponse struct {
-	Content []byte `json:"content"` // Assume base64 "encoding" parameter in Gitea response, and use Go's auto decode
-}
-
-type TopicsResponse struct {
-	Topics []string `json:"topics"`
-}
-
-type ReaddirEntry struct {
-	Name   string `json:"name"`
-	Path   string `json:"path"`
-	Size   int    `json:"size"`
-	RawURL string `json:"download_url"`
-}
-
-func (rde ReaddirEntry) isImage() bool {
-	return strings.HasSuffix(rde.Name, `.png`) || strings.HasSuffix(rde.Name, `.jpg`) || strings.HasSuffix(rde.Name, `.jpeg`)
-}
-
-type MarkdownRequest struct {
-	Context string
-	Mode    string
-	Text    string
-	Wiki    bool
-}
-
-// reposPage gets a single page of the list of Git repositories in this organisation.
-func (this *Application) reposPage(ctx context.Context, page, limit int) ([]Repo, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
-	var repos []Repo
-	err = json.NewDecoder(resp.Body).Decode(&repos)
-	if err != nil {
-		return nil, err
-	}
-
-	return repos, nil
-}
-
-// repos gets a list of Git repositories in this organisation. It may have to
-// make multiple network requests.
-func (this *Application) repos(ctx context.Context) ([]Repo, error) {
-	// Seems like gitea-1.13.0-rc1 returns 30 results by default, and supports up to a limit of 100
-	// Make a much larger request
-
-	ret := make([]Repo, 0)
-
-	nextPage := 1 // Counting starts at 1
-	for {
-		page, err := this.reposPage(ctx, nextPage, 300)
-		if err != nil {
-			return nil, err
-		}
-
-		if len(page) == 0 && len(ret) > 0 {
-			return ret, nil // Found enough already
-		}
-
-		ret = append(ret, page...)
-		nextPage += 1
-	}
-}
-
-// repoFile gets a single file from the default branch of the git repository
-// Usually the default branch is `master`.
-func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
-	var cr ContentsResponse
-	err = json.NewDecoder(resp.Body).Decode(&cr)
-	if err != nil {
-		return nil, err
-	}
-
-	return cr.Content, nil
-}
-
-func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-
-		// "No files found" happens with a HTTP 500/404 error depending on Gitea version. Catch this special case
-		if resp.StatusCode == 500 || resp.StatusCode == 404 {
-			b, err := ioutil.ReadAll(resp.Body)
-			if err != nil {
-				return nil, err
-			}
-			if strings.Contains(string(b), `does not exist`) {
-				return []ReaddirEntry{}, nil // no files found
-			}
-		}
-
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
-	var ret []ReaddirEntry
-	err = json.NewDecoder(resp.Body).Decode(&ret)
-	if err != nil {
-		return nil, err
-	}
-
-	return ret, nil
-}
-
-// imageFilesForRepo finds documentation images for the repository.
-// It searches the dist/ and doc/ subdirectories.
-func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]ReaddirEntry, error) {
-
-	ret := []ReaddirEntry{}
-
-	for _, dirName := range []string{`dist`, `doc`} {
-
-		files, err := this.filesInDirectory(ctx, repo, dirName)
-		if err != nil {
-			return nil, fmt.Errorf("readdir(%s): %w", dirName, err)
-		}
-
-		for _, f := range files {
-			if f.isImage() {
-				ret = append(ret, f)
-			}
-		}
-	}
-
-	return ret, nil
-}
-
-func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
-	var tr TopicsResponse
-	err = json.NewDecoder(resp.Body).Decode(&tr)
-	if err != nil {
-		return nil, err
-	}
-
-	return tr.Topics, nil
-}
-
-// renderMarkdown calls the remote Gitea server's own markdown renderer.
-func (this *Application) renderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
-
-	jb, err := json.Marshal(MarkdownRequest{
-		Context: this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) + `/src/branch/master`,
-		Mode:    "gfm", // magic constant - Github Flavoured Markdown
-		Text:    body,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/`, bytes.NewReader(jb))
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set(`Content-Type`, `application/json`)
-	req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(jb)))
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
-	return ioutil.ReadAll(resp.Body)
-}
-
-// renderMarkdownRaw calls the remote Gitea server's own markdown renderer.
-func (this *Application) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) {
-	err := this.apiSem.Acquire(ctx, 1)
-	if err != nil {
-		return nil, err // e.g. ctx closed
-	}
-	defer this.apiSem.Release(1)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/raw`, bytes.NewReader(body))
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set(`Content-Type`, `text/plain`)
-	req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(body)))
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
-	return ioutil.ReadAll(resp.Body)
-}

config.toml.sample

@@ -5,6 +5,7 @@ BindTo="0.0.0.0:5656"
 [Gitea]
 URL="https://gitea.com/"
 Org="gitea"
+Token="" # Must have misc:write, org:read, repo:read permission
 MaxConnections=2 # Use zero for unlimited
 
 [Redirect]
@@ -18,3 +19,8 @@ HomepageHeaderHTML="""
 	Teafolio is a web-based portfolio frontend for a Gitea server.
 </p>
 """
+
+# OverrideOrder allows reordering target repositories (0 for oldest)
+# This affects the default "Recent Projects" sort ordering only.
+#[OverrideOrder]
+#tea=0

gitea/apiclient.go

@@ -0,0 +1,389 @@
+package gitea
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"golang.org/x/sync/semaphore"
+)
+
+const MarkdownFailureCooldownSeconds = 3600 // 1 hour
+
+type APIClient struct {
+	urlBase string
+	orgName string
+	token   string
+	apiSem  *semaphore.Weighted
+
+	lastMarkdownFailure atomic.Int64
+}
+
+// NewAPIClient creates a new Gitea API client for a single Gitea organization.
+// Set maxConnections to 0 for unlimited concurrent API calls.
+func NewAPIClient(urlBase string, orgName string, token string, maxConnections int64) *APIClient {
+
+	if !strings.HasSuffix(urlBase, `/`) {
+		urlBase += `/`
+	}
+
+	ret := &APIClient{
+		urlBase: urlBase,
+		orgName: orgName,
+		token:   token,
+	}
+
+	if maxConnections == 0 { // unlimited
+		ret.apiSem = semaphore.NewWeighted(99999)
+	} else {
+		ret.apiSem = semaphore.NewWeighted(maxConnections)
+	}
+
+	return ret
+}
+
+func (ac *APIClient) PopulateCommitInfo(ctx context.Context, rr *Repo) error {
+
+	// The most recent commit will be the head of one of the branches (easy to find)
+
+	brs, err := ac.Branches(ctx, rr.Name)
+	if err != nil {
+		return err
+
+	} else {
+
+		NewestCommit := time.Unix(0, 0) // sentinel
+		for _, br := range brs {
+			if br.Commit.Timestamp.After(NewestCommit) {
+				NewestCommit = br.Commit.Timestamp
+			}
+		}
+		if !NewestCommit.Equal(time.Unix(0, 0)) {
+			rr.NewestCommit = NewestCommit // replace it
+		}
+	}
+
+	return nil
+}
+
+func (ac *APIClient) PopulateTopics(ctx context.Context, rr *Repo) error {
+	t, err := ac.topicsForRepo(ctx, rr.Name)
+	if err != nil {
+		return err
+	}
+
+	rr.Topics = t
+	return nil
+}
+
+func (ac *APIClient) PopulateImages(ctx context.Context, rr *Repo) error {
+	img, err := ac.ImageFilesForRepo(ctx, rr.Name)
+	if err != nil {
+		return err
+	}
+
+	rr.Images = img
+	return nil
+}
+
+func (ac *APIClient) apiRequest(ctx context.Context, endpoint string, target interface{}) error {
+	err := ac.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return err // e.g. ctx closed
+	}
+	defer ac.apiSem.Release(1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ac.urlBase+endpoint, nil)
+	if err != nil {
+		return err
+	}
+
+	if ac.token != "" {
+		req.Header.Set(`Authorization`, `token `+ac.token)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+
+	err = json.NewDecoder(resp.Body).Decode(target)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ac *APIClient) Branches(ctx context.Context, repo string) ([]Branch, error) {
+	var branches []Branch
+
+	err := ac.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(ac.orgName)+`/`+url.PathEscape(repo)+`/branches`, &branches)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return branches, nil
+}
+
+func (ac *APIClient) commitsPage(ctx context.Context, repo, ref string, page, limit int) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	err := ac.apiRequest(ctx, fmt.Sprintf(`api/v1/repos/%s/%s/commits?page=%d&limit=%d`, url.PathEscape(ac.orgName), url.PathEscape(repo), page, limit), &ret)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+
+	return ret, nil
+}
+
+func (ac *APIClient) Commits(ctx context.Context, repo, ref string) ([]CommitListEntry, error) {
+	var ret []CommitListEntry
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := ac.commitsPage(ctx, repo, ref, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(page) == 0 && len(ret) > 0 {
+			break // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+
+	if len(ret) == 0 {
+		return nil, fmt.Errorf("no commits found")
+	}
+
+	return ret, nil
+}
+
+// reposPage gets a single page of the list of Git repositories in this organisation.
+func (ac *APIClient) reposPage(ctx context.Context, page, limit int) ([]Repo, error) {
+	var repos []Repo
+
+	err := ac.apiRequest(ctx, `api/v1/orgs/`+url.PathEscape(ac.orgName)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), &repos)
+	if err != nil {
+		return nil, err
+	}
+
+	return repos, nil
+}
+
+// repos gets a list of Git repositories in this organisation. It may have to
+// make multiple network requests.
+func (ac *APIClient) Repos(ctx context.Context) ([]Repo, error) {
+	// Seems like gitea-1.13.0-rc1 returns 30 results by default, and supports up to a limit of 100
+	// Make a much larger request
+
+	ret := make([]Repo, 0)
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := ac.reposPage(ctx, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(page) == 0 && len(ret) > 0 {
+			break // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+
+	return ret, nil
+}
+
+// repoFile gets a single file from the default branch of the git repository
+// Usually the default branch is `master`.
+func (ac *APIClient) RepoFile(ctx context.Context, repo, filename string) ([]byte, error) {
+	var cr ContentsResponse
+
+	err := ac.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(ac.orgName)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), &cr)
+	if err != nil {
+		return nil, err
+	}
+
+	return cr.Content, nil
+}
+
+func (ac *APIClient) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) {
+	err := ac.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer ac.apiSem.Release(1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ac.urlBase+`api/v1/repos/`+url.PathEscape(ac.orgName)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+
+		// "No files found" happens with a HTTP 500/404 error depending on Gitea version. Catch this special case
+		if resp.StatusCode == 500 || resp.StatusCode == 404 {
+			b, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return nil, err
+			}
+			if strings.Contains(string(b), `does not exist`) {
+				return []ReaddirEntry{}, nil // no files found
+			}
+		}
+
+		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+
+	var ret []ReaddirEntry
+	err = json.NewDecoder(resp.Body).Decode(&ret)
+	if err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// ImageFilesForRepo finds documentation images for the repository.
+// It searches the top-level directory and the dist/ and doc/ subdirectories.
+func (ac *APIClient) ImageFilesForRepo(ctx context.Context, repo string) ([]ReaddirEntry, error) {
+
+	ret := []ReaddirEntry{}
+
+	for _, dirName := range []string{``, `dist`, `doc`} {
+
+		files, err := ac.filesInDirectory(ctx, repo, dirName)
+		if err != nil {
+			return nil, fmt.Errorf("readdir(%s): %w", dirName, err)
+		}
+
+		for _, f := range files {
+			if f.isImage() {
+				ret = append(ret, f)
+			}
+		}
+	}
+
+	return ret, nil
+}
+
+func (ac *APIClient) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
+	var tr TopicsResponse
+
+	err := ac.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(ac.orgName)+`/`+url.PathEscape(repo)+`/topics`, &tr)
+	if err != nil {
+		return nil, err
+	}
+
+	return tr.Topics, nil
+}
+
+var ErrMarkdownCooldown = errors.New("Markdown API was recently not functional")
+
+func (ac *APIClient) checkMarkdownCooldown() error {
+	if ac.lastMarkdownFailure.Load()+MarkdownFailureCooldownSeconds > time.Now().Unix() {
+		return ErrMarkdownCooldown
+	}
+
+	return nil
+}
+
+// renderMarkdown calls the remote Gitea server's own markdown renderer.
+func (ac *APIClient) RenderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) {
+	if err := ac.checkMarkdownCooldown(); err != nil {
+		return nil, err
+	}
+
+	err := ac.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer ac.apiSem.Release(1)
+
+	jb, err := json.Marshal(MarkdownRequest{
+		Context: ac.urlBase + url.PathEscape(ac.orgName) + `/` + url.PathEscape(repoName) + `/src/branch/master`,
+		Mode:    "gfm", // magic constant - Github Flavoured Markdown
+		Text:    body,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, ac.urlBase+`api/v1/markdown/`, bytes.NewReader(jb))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set(`Content-Type`, `application/json`)
+	req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(jb)))
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		ac.lastMarkdownFailure.Store(time.Now().Unix())
+		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+
+	return ioutil.ReadAll(resp.Body)
+}
+
+// renderMarkdownRaw calls the remote Gitea server's own markdown renderer.
+func (ac *APIClient) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) {
+	if err := ac.checkMarkdownCooldown(); err != nil {
+		return nil, err
+	}
+
+	err := ac.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer ac.apiSem.Release(1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, ac.urlBase+`api/v1/markdown/raw`, bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set(`Content-Type`, `text/plain`)
+	req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(body)))
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		ac.lastMarkdownFailure.Store(time.Now().Unix())
+		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+
+	return ioutil.ReadAll(resp.Body)
+}

gitea/types.go

@@ -0,0 +1,74 @@
+package gitea
+
+import (
+	"strings"
+	"time"
+)
+
+type Repo struct {
+	Name         string    `json:"name"`
+	Description  string    `json:"description"`
+	GiteaCreated time.Time `json:"created_at"`
+	GiteaUpdated time.Time `json:"updated_at"`
+
+	// NewestCommit is populated via PopulateCommitInfo().
+	NewestCommit time.Time
+	// Topics is populated via PopulateTopics().
+	Topics []string
+	// Images are populated via PopulateImages().
+	Images []ReaddirEntry
+}
+
+type ContentsResponse struct {
+	Content []byte `json:"content"` // Assume base64 "encoding" parameter in Gitea response, and use Go's auto decode
+}
+
+type TopicsResponse struct {
+	Topics []string `json:"topics"`
+}
+
+type ReaddirEntry struct {
+	Name   string `json:"name"`
+	Path   string `json:"path"`
+	Size   int    `json:"size"`
+	RawURL string `json:"download_url"`
+}
+
+func (rde ReaddirEntry) isImage() bool {
+	return strings.HasSuffix(rde.Name, `.png`) || strings.HasSuffix(rde.Name, `.jpg`) || strings.HasSuffix(rde.Name, `.jpeg`)
+}
+
+type MarkdownRequest struct {
+	Context string
+	Mode    string
+	Text    string
+	Wiki    bool
+}
+
+type BranchCommit struct {
+	ID        string    `json:"id"`
+	Message   string    `json:"message"`
+	Timestamp time.Time `json:"timestamp"`
+}
+
+type Branch struct {
+	Name   string       `json:"name"`
+	Commit BranchCommit `json:"commit"`
+}
+
+type AuthorInfo struct {
+	Name  string    `json:"name"`
+	Email string    `json:"email"`
+	Date  time.Time `json:"date"`
+}
+
+type CommitListEntryCommit struct {
+	Message   string     `json:"message"`
+	Author    AuthorInfo `json:"author"`
+	Committer AuthorInfo `json:"committer"`
+}
+
+type CommitListEntry struct {
+	ID     string                `json:"sha"`
+	Commit CommitListEntryCommit `json:"commit"`
+}

go.mod

@@ -1,8 +1,10 @@
 module teafolio
 
-go 1.13
+go 1.19
 
 require (
 	github.com/BurntSushi/toml v0.3.1
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 )
+
+require github.com/yuin/goldmark v1.7.0 // indirect

go.sum

@@ -1,4 +1,6 @@
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA=
+github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

main.go

@@ -1,27 +1,22 @@
 package main
 
 import (
-	"bytes"
-	"encoding/base64"
+	"context"
 	"flag"
-	"fmt"
-	"html"
 	"log"
 	"net/http"
-	"net/url"
 	"regexp"
-	"sort"
-	"strings"
+	"sync"
+	"teafolio/gitea"
 
 	"github.com/BurntSushi/toml"
-	"golang.org/x/sync/semaphore"
 )
 
 type Config struct {
 	BindTo string
 	Gitea  struct {
-		URL, Org       string
-		MaxConnections int64
+		URL, Org, Token string
+		MaxConnections  int64
 	}
 	Redirect map[string]string
 	Template struct {
@@ -29,346 +24,20 @@ type Config struct {
 		HomepageHeaderHTML  string
 		CustomLogoPngBase64 string
 	}
+	OverrideOrder map[string]int
 }
 
 type Application struct {
 	cfg Config
 
 	rxRepoPage, rxRepoImage *regexp.Regexp
-	apiSem                  *semaphore.Weighted
-}
 
-func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
-
-	pageTitle := this.cfg.Template.AppName
-	if pageDesc != "" {
-		pageTitle = pageDesc + ` | ` + pageTitle
-	}
-
-	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
-	w.WriteHeader(200)
-	fmt.Fprint(w, `<!DOCTYPE html>
-<html lang="en">
-	<head>
-		<meta http-equiv="X-UA-Compatible" content="IE=edge">
-		<meta charset="UTF-8">
-		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-		<meta name="viewport" content="width=960">
-		<title>`+html.EscapeString(pageTitle)+`</title>
-		`+extraHead+`
-		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
-		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
-		<link type="text/css" rel="stylesheet" href="/static/style.css">
-	</head>
-	<body>
-		<div id="container">
-			<div id="content">
-				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
-	`)
-	cb()
-	fmt.Fprint(w, `
-	</body>
-	<script type="text/javascript" src="/static/site.js"></script>
-</html>
-`)
-
-}
-
-func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
-	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
-	http.Error(w, "An internal error occurred.", 500)
-}
-
-func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-
-	repos, err := this.repos(ctx)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing repos: %w", err))
-		return
-	}
-
-	topics := make(map[string][]string)
-	for _, repo := range repos {
-		if t, err := this.topicsForRepo(ctx, repo.Name); err == nil {
-			topics[repo.Name] = t
-		}
-	}
-
-	// Sort repos once alphabetically, to get alphabetical indexes...
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].Name < repos[j].Name
-	})
-	alphabeticalOrderIndexes := make(map[string]int, len(repos))
-	for idx, repo := range repos {
-		alphabeticalOrderIndexes[repo.Name] = idx
-	}
-
-	// But then make sure the final sort is by most-recently-created
-	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].CreatedAt.After(repos[j].CreatedAt)
-	})
-
-	// Ready for template
-
-	this.Templatepage(w, r, "", "", func() {
-		fmt.Fprint(w, `
-		`+this.cfg.Template.HomepageHeaderHTML+`		
-		<select id="sortorder" style="float:right;">
-			<option value="data-sort-al">Alphabetical</option>
-			<option value="data-sort-ls">Lifespan</option>
-			<option value="data-sort-ct" selected>Recent projects</option>
-			<option value="data-sort-mt">Recent updates</option>
-		</select>
-		
-		<h2>Projects <small>(`+fmt.Sprintf("%d", len(repos))+`)</small></h2>
-		<table id="projtable-main" class="projtable">
-`)
-		for _, repo := range repos {
-			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
-
-			normalisedDesc := repo.Description
-			normalisedDesc = strings.TrimRight(repo.Description, `.`)
-			if len(normalisedDesc) > 0 {
-				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
-				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
-				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
-
-				if !(isAcronymOrCamelCase || firstWord == `Go`) {
-					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
-				}
-
-				// Add leading `<COMMA><SPACE>` to separate from the repo title
-				normalisedDesc = `, ` + normalisedDesc
-			}
-
-			rowClass := ""
-			for _, topic := range topics[repo.Name] {
-				rowClass += `taggedWith-` + topic + ` `
-			}
-
-			fmt.Fprint(w, `
-			<tr
-				class="`+html.EscapeString(rowClass)+`"
-				data-sort-al="`+fmt.Sprintf("-%d", alphabeticalOrderIndexes[repo.Name])+`"
-				data-sort-ls="`+fmt.Sprintf("%.0f", repo.UpdatedAt.Sub(repo.CreatedAt).Seconds())+`"
-				data-sort-ct="`+fmt.Sprintf("%d", repo.CreatedAt.Unix())+`"
-				data-sort-mt="`+fmt.Sprintf("%d", repo.UpdatedAt.Unix())+`"
-			>
-				<td>
-					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
-				</td>
-				<td>
-					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
-					<a href="`+pageHref+`" class="article-read-more">more...</a>
-					<br>
-					<small>
-			`)
-			for _, topic := range topics[repo.Name] {
-				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
-			}
-			fmt.Fprint(w, `
-					</small>
-				</td>
-			</tr>
-			`)
-		}
-		fmt.Fprint(w, `
-		</table>
-		`)
-	})
-}
-
-func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
-	ctx := r.Context()
-
-	images, err := this.imageFilesForRepo(ctx, repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	if len(images) == 0 {
-		w.Header().Set(`Location`, `/static/no_image.png`)
-		w.WriteHeader(301)
-		return
-	}
-
-	w.Header().Set(`Location`, images[0].RawURL)
-	w.WriteHeader(301)
-}
-
-func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
-	ctx := r.Context()
-	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
-	extraHead := ""
-
-	readme, err := this.repoFile(ctx, repoName, `README.md`)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
-		return
-	}
-
-	lines := strings.Split(string(readme), "\n")
-
-	// We add some extra badges based on special text entries
-	extraBadgesMd := ` ![](https://img.shields.io/badge/build-success-brightgreen)`
-	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
-
-	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
-	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
-		lines[2] += ` ` + extraBadgesMd
-	} else {
-		// Push other lines down
-		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
-	}
-
-	readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
-		return
-	}
-
-	images, err := this.imageFilesForRepo(ctx, repoName)
-	if err != nil {
-		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
-		return
-	}
-
-	// If the Git repository contains a top-level go.mod file, allow vanity imports
-	if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
-
-		// Check the first line should be `module MODULENAME\n`
-		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
-		if bytes.HasPrefix(firstLine, []byte("module ")) {
-			moduleName := firstLine[7:]
-			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
-		}
-	}
-
-	// De-escalate all headers in rendered markdown to match our style
-	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
-		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
-
-	// Ready for template
-
-	this.Templatepage(w, r, repoName, extraHead, func() {
-
-		projBodyclass := `projbody`
-		if len(images) > 0 {
-			projBodyclass += ` projbody_halfw`
-		}
-
-		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
-		repl.WriteString(w, string(readmeHtml))
-		fmt.Fprint(w, `</div>`)
-
-		if len(images) > 0 {
-			fmt.Fprint(w, `<div class="projimg">`)
-			for _, img := range images {
-				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
-			}
-			fmt.Fprint(w, `</div>`)
-		}
-
-		fmt.Fprint(w, `<div style="clear:both;"></div>`)
-		fmt.Fprint(w, `</div>`) // projbody
-	})
-}
-
-func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method == `GET` {
-		if r.URL.Path == `/` {
-			this.Homepage(w, r)
-
-		} else if r.URL.Path == `/favicon.ico` {
-			w.Header().Set(`Location`, `/static/logo.png`)
-			w.WriteHeader(301)
-
-		} else if r.URL.Path == `/robots.txt` {
-			http.Error(w, "not found", 404)
-
-		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
-			this.Bannerpage(w, r, parts[1])
-
-		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
-
-			// Support /repo.html URIs for backward compatibility
-			if strings.HasSuffix(parts[1], `.html`) {
-				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
-				w.WriteHeader(301)
-				return
-			}
-
-			// The regexp supports an optional trailing slash
-			// Redirect to canonical no-trailing-slash
-			if strings.HasSuffix(r.URL.Path, `/`) {
-				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
-				w.WriteHeader(301)
-				return
-			}
-
-			// Proper decoding of special characters in repo path component
-			repoName, err := url.PathUnescape(parts[1])
-			if err != nil {
-				http.Error(w, "malformed url encoding in repository name", 400)
-				return
-			}
-
-			// Maybe it's a redirected project (alternative name)
-			if rename, ok := this.cfg.Redirect[repoName]; ok {
-				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
-				w.WriteHeader(301)
-				return
-			}
-
-			this.Repopage(w, r, repoName)
-
-		} else if r.URL.Path == `/static/logo.png` {
-			if this.cfg.Template.CustomLogoPngBase64 != "" {
-
-				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
-				if err != nil {
-					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
-					return
-				}
-
-				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
-				w.Header().Set(`Content-Type`, `image/png`)
-				w.WriteHeader(200)
-				w.Write(logoPng)
-
-			} else {
-				r.URL.Path = r.URL.Path[8:]
-				http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-			}
-
-		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
-			r.URL.Path = r.URL.Path[8:]
-			http.FileServer(http.Dir(`static`)).ServeHTTP(w, r)
-
-		} else if r.URL.Query().Get("go-get") == "1" {
-			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
-			// It must be a subpackage request
-			// We can't serve the proper go-import meta tag immediately because
-			// we haven't looked up the go.mod yet. Just redirect to the root
-			// package - `go get` will follow redirects and the resulting meta tag is correct
-
-			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
-
-			w.Header().Set(`Location`, `/`+slashParts[1])
-			w.WriteHeader(301)
-			return
-
-		} else {
-			http.Error(w, "not found", 404)
-		}
-
-	} else {
-		http.Error(w, "invalid method", 400)
-	}
+	gitea *gitea.APIClient
 
+	reposMut               sync.RWMutex
+	reposCache             []gitea.Repo // Sorted by recently-created-first
+	reposCacheByName       map[string]int
+	reposAlphabeticalOrder map[string]int
 }
 
 func main() {
@@ -385,17 +54,12 @@ func main() {
 		log.Fatalf("toml.DecodeFile: %s", err.Error())
 	}
 
-	// Assert Gitea URL always has trailing slash
-	if !strings.HasSuffix(app.cfg.Gitea.URL, `/`) {
-		app.cfg.Gitea.URL += `/`
-	}
+	// Create Gitea API client
+	app.gitea = gitea.NewAPIClient(app.cfg.Gitea.URL, app.cfg.Gitea.Org, app.cfg.Gitea.Token, app.cfg.Gitea.MaxConnections)
 
-	// Create semaphore
-	if app.cfg.Gitea.MaxConnections == 0 { // unlimited
-		app.apiSem = semaphore.NewWeighted(99999)
-	} else {
-		app.apiSem = semaphore.NewWeighted(app.cfg.Gitea.MaxConnections)
-	}
+	// Sync worker
+	go app.syncWorker(context.Background())
 
+	log.Printf("Starting web server on [%s]...", app.cfg.BindTo)
 	log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app))
 }

page_home.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"fmt"
+	"html"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
+
+	this.reposMut.RLock()
+	defer this.reposMut.RUnlock()
+
+	if len(this.reposCache) == 0 {
+		// We haven't loaded the repositories from Gitea yet
+		this.Delay(w, r)
+		return
+	}
+
+	// Ready for template
+
+	this.Templatepage(w, r, "", "", func() {
+		fmt.Fprint(w, `
+		`+this.cfg.Template.HomepageHeaderHTML+`		
+		<select id="sortorder" style="float:right;">
+			<option value="data-sort-al">Alphabetical</option>
+			<option value="data-sort-ls">Lifespan</option>
+			<option value="data-sort-ct" selected>Recent projects</option>
+			<option value="data-sort-mt">Recent updates</option>
+		</select>
+		
+		<h2>Projects <small>(`+fmt.Sprintf("%d", len(this.reposCache))+`)</small></h2>
+		<table id="projtable-main" class="projtable">
+`)
+		for repoIdx, repo := range this.reposCache {
+			pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name))
+
+			normalisedDesc := repo.Description
+			normalisedDesc = strings.TrimRight(repo.Description, `.`)
+			if len(normalisedDesc) > 0 {
+				// Lowercase the first letter of the description, unless it starts with an acronym (all letters uppercase first word) or CamelCase word
+				firstWord := strings.SplitN(normalisedDesc, " ", 2)[0]
+				isAcronymOrCamelCase := len(firstWord) > 1 && (firstWord[1:] != strings.ToLower(firstWord[1:]))
+
+				if !(isAcronymOrCamelCase || firstWord == `Go`) {
+					normalisedDesc = strings.ToLower(normalisedDesc[0:1]) + normalisedDesc[1:]
+				}
+
+				// Add leading `<COMMA><SPACE>` to separate from the repo title
+				normalisedDesc = `, ` + normalisedDesc
+			}
+
+			rowClass := ""
+			for _, topic := range repo.Topics {
+				rowClass += `taggedWith-` + topic + ` `
+			}
+
+			fmt.Fprint(w, `
+			<tr
+				class="`+html.EscapeString(rowClass)+`"
+				data-sort-al="`+fmt.Sprintf("-%d", this.reposAlphabeticalOrder[repo.Name])+`"
+				data-sort-ls="`+fmt.Sprintf("%.0f", repo.NewestCommit.Sub(repo.GiteaCreated).Seconds())+`"
+				data-sort-ct="`+fmt.Sprintf("-%d", repoIdx)+`"
+				data-sort-mt="`+fmt.Sprintf("%d", repo.NewestCommit.Unix())+`"
+			>
+				<td>
+					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
+				</td>
+				<td>
+					<strong>`+html.EscapeString(repo.Name)+`</strong>`+html.EscapeString(normalisedDesc)+`
+					<a href="`+pageHref+`" class="article-read-more">more...</a>
+					<br>
+					<small>
+			`)
+			for _, topic := range repo.Topics {
+				fmt.Fprint(w, `<a class="tag tag-link" data-tag="`+html.EscapeString(topic)+`">`+html.EscapeString(topic)+`</a> `)
+			}
+			fmt.Fprint(w, `
+					</small>
+				</td>
+			</tr>
+			`)
+		}
+		fmt.Fprint(w, `
+		</table>
+		`)
+	})
+}

page_imageredir.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"net/http"
+)
+
+func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
+
+	this.reposMut.RLock()
+	defer this.reposMut.RUnlock()
+
+	ridx, ok := this.reposCacheByName[repoName]
+	if !ok {
+		w.Header().Set(`Location`, `/static/no_image.png`)
+		w.WriteHeader(301)
+		return
+	}
+
+	images := this.reposCache[ridx].Images
+	if len(images) == 0 {
+		w.Header().Set(`Location`, `/static/no_image.png`)
+		w.WriteHeader(301)
+		return
+	}
+
+	w.Header().Set(`Location`, images[0].RawURL)
+	w.WriteHeader(301)
+}

page_repository.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"html"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"teafolio/gitea"
+
+	"github.com/yuin/goldmark"
+)
+
+func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
+	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
+	extraHead := ""
+
+	readme, err := this.gitea.RepoFile(ctx, repoName, `README.md`)
+	if err != nil {
+		log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
+		http.Redirect(w, r, repoURL, http.StatusTemporaryRedirect)
+		return
+	}
+
+	lines := strings.Split(string(readme), "\n")
+
+	// Check if this repo has the 'article' tag
+	hasArticleTag := false
+	this.reposMut.RLock()
+	for _, rr := range this.reposCache {
+		if rr.Name != repoName {
+			continue
+		}
+		for _, topic := range rr.Topics {
+			if topic == "article" {
+				hasArticleTag = true
+				break
+			}
+		}
+	}
+	this.reposMut.RUnlock()
+
+	// We add some extra badges based on special text entries
+	extraBadgesMd := ``
+	if !hasArticleTag {
+		extraBadgesMd += ` %%REPLACEME__BADGE%%`
+	}
+	extraBadgesMd += ` [![](https://img.shields.io/badge/vcs-git-green?logo=git)](` + repoURL + `)`
+
+	// Inject more badges to 3rd line; or, create badges on 3rd line if there are none already
+	if len(lines) >= 3 && strings.Contains(lines[2], `shields.io`) {
+		lines[2] += ` ` + extraBadgesMd
+	} else {
+		// Push other lines down
+		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
+	}
+
+	readmeHtml, err := this.gitea.RenderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
+	if err != nil {
+		if !errors.Is(err, gitea.ErrMarkdownCooldown) {
+			log.Printf("%s %s: %s", r.Method, r.URL.Path, fmt.Errorf("rendering markdown: %w", err))
+		}
+
+		// Failed to use Gitea's markdown renderer
+		// HTTP 401 (Unauthorized) started happening with this API sometime
+		// between Gitea 1.18 -> 1.21, and no authorization token seems
+		// sufficient to make it work again
+		// Use our own one instead as a fallback
+		buff := bytes.Buffer{}
+		err = goldmark.Convert([]byte(strings.Join(lines, "\n")), &buff)
+
+		if err != nil {
+			// Built-in markdown renderer didn't work either
+			log.Printf("%s %s: %s", r.Method, r.URL.Path, fmt.Errorf("rendering markdown: %w", err))
+			http.Redirect(w, r, repoURL, http.StatusTemporaryRedirect)
+			return
+		}
+
+		// OK
+		readmeHtml = buff.Bytes()
+		err = nil
+	}
+
+	readmeHtml = []byte(strings.Replace(string(readmeHtml), `%%REPLACEME__BADGE%%`, `<img src="/static/build_success_brightgreen.svg" style="width:90px;height:20px;">`, 1))
+
+	images, err := this.gitea.ImageFilesForRepo(ctx, repoName)
+	if err != nil {
+		log.Printf("%s %s: %s", r.Method, r.URL.Path, fmt.Errorf("listing images: %w", err))
+		http.Redirect(w, r, repoURL, http.StatusTemporaryRedirect)
+		return
+	}
+
+	// If the Git repository contains a top-level go.mod file, allow vanity imports
+	if goMod, err := this.gitea.RepoFile(ctx, repoName, `go.mod`); err == nil {
+
+		// Check the first line should be `module MODULENAME\n`
+		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
+		if bytes.HasPrefix(firstLine, []byte("module ")) {
+			moduleName := firstLine[7:]
+			extraHead = `<meta name="go-import" content="` + html.EscapeString(string(moduleName)) + ` git ` + repoURL + `.git">`
+		}
+	}
+
+	// De-escalate all headers in rendered markdown to match our style
+	repl := strings.NewReplacer(`<h1`, `<h2`, `<h2`, `<h3`, `<h3`, `<h4`,
+		`</h1>`, `</h2>`, `</h2>`, `</h3>`, `</h3>`, `</h4>`)
+
+	// Ready for template
+
+	this.Templatepage(w, r, repoName, extraHead, func() {
+
+		projBodyclass := `projbody`
+		if len(images) > 0 {
+			projBodyclass += ` projbody_halfw`
+		}
+
+		fmt.Fprint(w, `<div class="projinfo"><div class="`+projBodyclass+`">`)
+		repl.WriteString(w, string(readmeHtml))
+		fmt.Fprint(w, `</div>`)
+
+		if len(images) > 0 {
+			fmt.Fprint(w, `<div class="projimg">`)
+			for _, img := range images {
+				fmt.Fprint(w, `<a href="`+html.EscapeString(img.RawURL)+`"><img alt="" class="thumbimage" src="`+html.EscapeString(img.RawURL)+`" /></a>`)
+			}
+			fmt.Fprint(w, `</div>`)
+		}
+
+		fmt.Fprint(w, `<div style="clear:both;"></div>`)
+		fmt.Fprint(w, `</div>`) // projbody
+	})
+}

page_util.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"html"
+	"log"
+	"net/http"
+)
+
+func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
+
+	pageTitle := this.cfg.Template.AppName
+	if pageDesc != "" {
+		pageTitle = pageDesc + ` | ` + pageTitle
+	}
+
+	w.Header().Set(`Content-Type`, `text/html; charset=UTF-8`)
+	w.WriteHeader(200)
+	fmt.Fprint(w, `<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta charset="UTF-8">
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+		<title>`+html.EscapeString(pageTitle)+`</title>
+		`+extraHead+`
+		<link rel="shortcut icon" href="/static/logo.png" type="image/png">
+		<link rel="apple-touch-icon" href="/static/logo.png" type="image/png">
+		<link type="text/css" rel="stylesheet" href="/static/style.css">
+	</head>
+	<body>
+		<div id="container">
+			<div id="content">
+				<h1><a href="/"><div id="logo"></div>`+html.EscapeString(this.cfg.Template.AppName)+`</a></h1>
+	`)
+	cb()
+	fmt.Fprint(w, `
+	</body>
+	<script type="text/javascript" src="/static/site.js"></script>
+</html>
+`)
+
+}
+
+func (this *Application) internalError(w http.ResponseWriter, r *http.Request, err error) {
+	log.Printf("%s %s: %s", r.Method, r.URL.Path, err)
+	http.Error(w, "An internal error occurred.", 500)
+}
+
+func (this *Application) Delay(w http.ResponseWriter, r *http.Request) {
+	this.Templatepage(w, r, "Loading...", "", func() {
+		fmt.Fprintf(w, `
+		<h2>Loading, please wait...</h2>
+		
+		<meta http-equiv="refresh" content="5">
+		`)
+	})
+}

router.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"embed"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+//go:embed static/*
+var StaticFiles embed.FS
+
+func (this *Application) ServeStatic(w http.ResponseWriter, r *http.Request) {
+	http.FileServer(http.FS(StaticFiles)).ServeHTTP(w, r)
+	// http.StripPrefix(`/static/`, http.FileServer(http.FS(StaticFiles))).ServeHTTP(w, r)
+}
+
+func (this *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method == `GET` {
+		if r.URL.Path == `/` {
+			this.Homepage(w, r)
+
+		} else if r.URL.Path == `/favicon.ico` {
+			w.Header().Set(`Location`, `/static/logo.png`)
+			w.WriteHeader(301)
+
+		} else if r.URL.Path == `/robots.txt` {
+			http.Error(w, "not found", 404)
+
+		} else if parts := this.rxRepoImage.FindStringSubmatch(r.URL.Path); parts != nil {
+			this.Bannerpage(w, r, parts[1])
+
+		} else if parts := this.rxRepoPage.FindStringSubmatch(r.URL.Path); parts != nil {
+
+			// Support /repo.html URIs for backward compatibility
+			if strings.HasSuffix(parts[1], `.html`) {
+				w.Header().Set(`Location`, r.URL.Path[0:len(r.URL.Path)-5])
+				w.WriteHeader(301)
+				return
+			}
+
+			// The regexp supports an optional trailing slash
+			// Redirect to canonical no-trailing-slash
+			if strings.HasSuffix(r.URL.Path, `/`) {
+				w.Header().Set(`Location`, `/`+parts[1]) // n.b. parts[1] isn't urldecoded yet
+				w.WriteHeader(301)
+				return
+			}
+
+			// Proper decoding of special characters in repo path component
+			repoName, err := url.PathUnescape(parts[1])
+			if err != nil {
+				http.Error(w, "malformed url encoding in repository name", 400)
+				return
+			}
+
+			// Maybe it's a redirected project (alternative name)
+			if rename, ok := this.cfg.Redirect[repoName]; ok {
+				w.Header().Set(`Location`, `/`+url.PathEscape(rename))
+				w.WriteHeader(301)
+				return
+			}
+
+			this.Repopage(w, r, repoName)
+
+		} else if strings.HasPrefix(r.URL.Path, `/static/`) {
+
+			if r.URL.Path == `/static/logo.png` && this.cfg.Template.CustomLogoPngBase64 != "" {
+
+				logoPng, err := base64.StdEncoding.DecodeString(this.cfg.Template.CustomLogoPngBase64)
+				if err != nil {
+					this.internalError(w, r, fmt.Errorf("parsing base64 logo: %w", err))
+					return
+				}
+
+				w.Header().Set(`Content-Length`, fmt.Sprintf("%d", len(logoPng)))
+				w.Header().Set(`Content-Type`, `image/png`)
+				w.WriteHeader(200)
+				w.Write(logoPng)
+				return
+			}
+
+			// Embedded resource
+			// r.URL.Path = r.URL.Path[8:]
+			this.ServeStatic(w, r)
+
+		} else if r.URL.Query().Get("go-get") == "1" {
+			// This wasn't one of our standard `/repo` paths, but there is the ?go-get=1 parameter
+			// It must be a subpackage request
+			// We can't serve the proper go-import meta tag immediately because
+			// we haven't looked up the go.mod yet. Just redirect to the root
+			// package - `go get` will follow redirects and the resulting meta tag is correct
+
+			slashParts := strings.SplitN(r.URL.Path, `/`, 3) // len === 3 is guaranteed from earlier if cases
+
+			w.Header().Set(`Location`, `/`+slashParts[1])
+			w.WriteHeader(301)
+			return
+
+		} else {
+			http.Error(w, "not found", 404)
+		}
+
+	} else {
+		http.Error(w, "invalid method", 400)
+	}
+
+}

static/build_success_brightgreen.svg

@@ -0,0 +1,2 @@
+
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="90" height="20" role="img" aria-label="build: success"><title>build: success</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="90" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="37" height="20" fill="#555"/><rect x="37" width="53" height="20" fill="#4c1"/><rect width="90" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="195" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="270">build</text><text x="195" y="140" transform="scale(.1)" fill="#fff" textLength="270">build</text><text aria-hidden="true" x="625" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">success</text><text x="625" y="140" transform="scale(.1)" fill="#fff" textLength="430">success</text></g></svg>

static/style.css

@@ -161,6 +161,25 @@ img[src*="shields.io"] {
 
 /* */
 
+.projbody table {
+	width: 100%;
+}
+
+.projbody table {
+    border-collapse: collapse;
+}
+
+.projbody table td, .projbody table th {
+	border: 1px solid #eee;
+    padding: 4px;
+}
+
+.projbody tr:hover td {
+    background: #f8f8f8;
+}
+
+/* */
+
 @media screen and (max-width:960px) {
 	
 	#container {

sync.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"sort"
+	"teafolio/gitea"
+	"time"
+)
+
+// rearrangeOrder applies rearrangements from the OverrideOrder configuration.
+func (this *Application) rearrangeOrder(repos []gitea.Repo) ([]gitea.Repo, error) {
+
+	if len(this.cfg.OverrideOrder) == 0 {
+		return repos, nil // nothing to do
+	}
+
+	// Collect pre-existing positions for repos
+	reposByName := make(map[string]gitea.Repo, len(repos))
+	for _, repo := range repos {
+		reposByName[repo.Name] = repo
+	}
+
+	// Sort target insertion positions by lowest-first
+	type insertionPosition struct {
+		pos      int
+		repoName string
+	}
+	insertAt := make([]insertionPosition, 0, len(this.cfg.OverrideOrder))
+
+	for rn, rpos := range this.cfg.OverrideOrder {
+		insertAt = append(insertAt, insertionPosition{len(repos) - rpos - 1, rn})
+	}
+	sort.Slice(insertAt, func(i, j int) bool {
+		return insertAt[i].pos < insertAt[j].pos
+	})
+
+	// Walking-insertion loop
+	ret := make([]gitea.Repo, 0, len(repos))
+	nextRepo := 0
+	nextOverride := 0
+	for {
+		if nextOverride < len(insertAt) && insertAt[nextOverride].pos == len(ret) {
+			// We have an override to insert at this position
+			ret = append(ret, reposByName[insertAt[nextOverride].repoName])
+			nextOverride++
+
+		} else if nextRepo < len(repos) {
+			// There are still other repos to insert generally
+			if _, ok := this.cfg.OverrideOrder[repos[nextRepo].Name]; ok {
+				// This repo has an overridden position. Don't insert it generally here
+				nextRepo++
+
+			} else {
+				// This repo does not have an overriden position. Here is fine
+				ret = append(ret, repos[nextRepo])
+				nextRepo++
+			}
+
+		} else {
+			// There are no more overrides to insert and there are no remaining
+			// non-override repos
+			// That means we're done
+			break
+		}
+	}
+
+	if len(ret) != len(repos) {
+		return nil, fmt.Errorf("Failed to apply OverrideOrder (got %d of %d repositories)", len(ret), len(repos))
+	}
+
+	return ret, nil
+}
+
+func (this *Application) sync(ctx context.Context) (bool, error) {
+
+	// List repositories on Gitea
+	repos, err := this.gitea.Repos(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	// Compare this list of repositories to our existing one
+	// If the repository is new, or if it's update-time has changed since we last
+	// saw it, then re-refresh its real git commit timestamps
+	// Otherwise copy them from the previous version
+
+	this.reposMut.RLock() // readonly
+
+	anyChanges := false
+	if len(repos) != len(this.reposCache) {
+		anyChanges = true
+	}
+
+	for i, rr := range repos {
+		if idx, ok := this.reposCacheByName[rr.Name]; ok && this.reposCache[idx].GiteaUpdated == rr.GiteaUpdated {
+			// Already exists in cache with same Gitea update time
+			// Copy timestamps
+			repos[i] = this.reposCache[idx]
+
+		} else {
+
+			// New repo, or Gitea has updated timestamp
+			anyChanges = true
+
+			// Refresh timestamps
+			err := this.gitea.PopulateCommitInfo(ctx, &rr)
+			if err != nil {
+				log.Printf("loading branches for '%s': %s", rr.Name, err)
+				rr.NewestCommit = rr.GiteaUpdated // best guess
+			}
+
+			// Refresh topics
+			err = this.gitea.PopulateTopics(ctx, &rr)
+			if err != nil {
+				log.Printf("loading topics for '%s': %s", rr.Name, err)
+			}
+
+			err = this.gitea.PopulateImages(ctx, &rr)
+			if err != nil {
+				log.Printf("loading images for '%s': %s", rr.Name, err)
+			}
+
+			// Save
+			repos[i] = rr
+		}
+	}
+
+	this.reposMut.RUnlock()
+
+	//
+	if !anyChanges {
+		return false, nil // nothing to do
+	}
+
+	// We have a final updated repos array
+
+	// Sort repos once alphabetically, to get alphabetical indexes...
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	alphabeticalOrderIndexes := make(map[string]int, len(repos))
+	for idx, repo := range repos {
+		alphabeticalOrderIndexes[repo.Name] = idx
+	}
+
+	// But then make sure the final sort is by most-recently-created
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GiteaCreated.After(repos[j].GiteaCreated)
+	})
+	reordered, err := this.rearrangeOrder(repos)
+	if err != nil {
+		log.Printf("Reorder failure: %s", err.Error())
+	} else {
+		repos = reordered
+	}
+
+	reposCacheByName := make(map[string]int, len(repos))
+	for idx, repo := range repos {
+		reposCacheByName[repo.Name] = idx
+	}
+
+	// Commit our changes for the other threads to look at
+	this.reposMut.Lock()
+	this.reposCache = repos
+	this.reposCacheByName = reposCacheByName
+	this.reposAlphabeticalOrder = alphabeticalOrderIndexes
+	this.reposMut.Unlock()
+
+	// Done
+	return true, nil
+}
+
+func (this *Application) syncWorker(ctx context.Context) {
+
+	t := time.NewTicker(30 * time.Minute)
+	defer t.Stop()
+
+	for {
+		anyChanges, err := this.sync(ctx)
+		if err != nil {
+			// log and continue
+			log.Printf("Refreshing repositories: %s", err.Error())
+		}
+		if anyChanges {
+			log.Printf("Repositories updated")
+		}
+
+		select {
+		case <-t.C:
+			continue
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}