api: begin conversion to gitea SDK

README.md

@@ -31,6 +31,15 @@ dokku storage:mount teafolio /srv/teafolio-dokku/config.toml:/app/config.toml
 
 ## CHANGELOG
 
+2020-11-08 v1.1.1
+- Fix an issue with newer versions of Gitea that paginate repoistory list responses
+- Fix an issue with blocking semaphores for a cancelled network request
+
+2020-05-24 v1.1.0
+- Support limiting the number of concurrent API requests to Gitea
+- Display total number of projects
+- Fix cosmetic issues with page background image, page height, and margins around thumbnails
+
 2020-05-24 v1.0.1
 - Remove image dependency from static files
 - Fix a cosmetic issue with `h2`/`h3` margins

api.go

@@ -2,21 +2,16 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
-	"time"
-)
 
-type Repo struct {
+	"code.gitea.io/sdk/gitea"
-	Name        string    `json:"name"`
+)
-	Description string    `json:"description"`
-	CreatedAt   time.Time `json:"created_at"`
-	UpdatedAt   time.Time `json:"updated_at"`
-}
 
 type ContentsResponse struct {
 	Content []byte `json:"content"` // Assume base64 "encoding" parameter in Gitea response, and use Go's auto decode
@@ -44,20 +39,15 @@ type MarkdownRequest struct {
 	Wiki    bool
 }
 
-// repos gets a list of Git repositories in this organisation.
+// reposPage gets a single page of the list of Git repositories in this organisation.
-func (this *Application) repos() ([]Repo, error) {
+func (this *Application) reposPage(ctx context.Context, page, limit int) ([]*gitea.Repository, error) {
-	resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/orgs/` + url.PathEscape(this.cfg.Gitea.Org) + `/repos`)
+	err := this.apiSem.Acquire(ctx, 1)
 	if err != nil {
-		return nil, err
+		return nil, err // e.g. ctx closed
 	}
-	defer resp.Body.Close()
+	defer this.apiSem.Release(1)
 
-	if resp.StatusCode != 200 {
+	repos, _, err := this.gc.ListOrgRepos(this.cfg.Gitea.Org, gitea.ListOrgReposOptions{ListOptions: gitea.ListOptions{Page: page, PageSize: limit}})
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
-	var repos []Repo
-	err = json.NewDecoder(resp.Body).Decode(&repos)
 	if err != nil {
 		return nil, err
 	}
@@ -65,30 +55,60 @@ func (this *Application) repos() ([]Repo, error) {
 	return repos, nil
 }
 
+// repos gets a list of Git repositories in this organisation. It may have to
+// make multiple network requests.
+func (this *Application) repos(ctx context.Context) ([]*gitea.Repository, error) {
+	// Seems like gitea-1.13.0-rc1 returns 30 results by default, and supports up to a limit of 100
+	// Make a much larger request
+
+	ret := make([]*gitea.Repository, 0)
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := this.reposPage(ctx, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(page) == 0 && len(ret) > 0 {
+			return ret, nil // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+}
+
 // repoFile gets a single file from the default branch of the git repository
 // Usually the default branch is `master`.
-func (this *Application) repoFile(repo, filename string) ([]byte, error) {
+func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) {
-	resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/contents/` + url.PathEscape(filename))
+	err := this.apiSem.Acquire(ctx, 1)
 	if err != nil {
-		return nil, err
+		return nil, err // e.g. ctx closed
 	}
-	defer resp.Body.Close()
+	defer this.apiSem.Release(1)
 
-	if resp.StatusCode != 200 {
+	resp, _, err := this.gc.GetContents(this.cfg.Gitea.Org, repo, "", filename)
-		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
-	}
-
-	var cr ContentsResponse
-	err = json.NewDecoder(resp.Body).Decode(&cr)
 	if err != nil {
 		return nil, err
 	}
 
-	return cr.Content, nil
+	return []byte(*resp.Content), nil
 }
 
-func (this *Application) filesInDirectory(repo, dir string) ([]ReaddirEntry, error) {
+func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) {
-	resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/contents/` + dir) // n.b. $dir param not escaped
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer this.apiSem.Release(1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -121,13 +141,13 @@ func (this *Application) filesInDirectory(repo, dir string) ([]ReaddirEntry, err
 
 // imageFilesForRepo finds documentation images for the repository.
 // It searches the dist/ and doc/ subdirectories.
-func (this *Application) imageFilesForRepo(repo string) ([]ReaddirEntry, error) {
+func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]ReaddirEntry, error) {
 
 	ret := []ReaddirEntry{}
 
 	for _, dirName := range []string{`dist`, `doc`} {
 
-		files, err := this.filesInDirectory(repo, dirName)
+		files, err := this.filesInDirectory(ctx, repo, dirName)
 		if err != nil {
 			return nil, fmt.Errorf("readdir(%s): %w", dirName, err)
 		}
@@ -142,8 +162,19 @@ func (this *Application) imageFilesForRepo(repo string) ([]ReaddirEntry, error)
 	return ret, nil
 }
 
-func (this *Application) topicsForRepo(repo string) ([]string, error) {
+func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
-	resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/topics`)
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer this.apiSem.Release(1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -163,19 +194,30 @@ func (this *Application) topicsForRepo(repo string) ([]string, error) {
 }
 
 // renderMarkdown calls the remote Gitea server's own markdown renderer.
-func (this *Application) renderMarkdown(repoName string, body string) ([]byte, error) {
+func (this *Application) renderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) {
-	req := MarkdownRequest{
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer this.apiSem.Release(1)
+
+	jb, err := json.Marshal(MarkdownRequest{
 		Context: this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) + `/src/branch/master`,
 		Mode:    "gfm", // magic constant - Github Flavoured Markdown
 		Text:    body,
-	}
+	})
-
-	jb, err := json.Marshal(req)
 	if err != nil {
 		return nil, err
 	}
 
-	resp, err := http.Post(this.cfg.Gitea.URL+`api/v1/markdown/`, `application/json`, bytes.NewReader(jb))
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/`, bytes.NewReader(jb))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set(`Content-Type`, `application/json`)
+	req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(jb)))
+
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -189,8 +231,21 @@ func (this *Application) renderMarkdown(repoName string, body string) ([]byte, e
 }
 
 // renderMarkdownRaw calls the remote Gitea server's own markdown renderer.
-func (this *Application) renderMarkdownRaw(body []byte) ([]byte, error) {
+func (this *Application) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) {
-	resp, err := http.Post(this.cfg.Gitea.URL+`api/v1/markdown/raw`, `text/plain`, bytes.NewReader(body))
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
+	defer this.apiSem.Release(1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/raw`, bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set(`Content-Type`, `text/plain`)
+	req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(body)))
+
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}

config.toml.sample

@@ -5,6 +5,7 @@ BindTo="0.0.0.0:5656"
 [Gitea]
 URL="https://gitea.com/"
 Org="gitea"
+MaxConnections=2 # Use zero for unlimited
 
 [Redirect]
 "old-project-name" = "new-project-name"

go.mod

@@ -2,4 +2,8 @@ module teafolio
 
 go 1.13
 
-require github.com/BurntSushi/toml v0.3.1
+require (
+	code.gitea.io/sdk/gitea v0.13.1
+	github.com/BurntSushi/toml v0.3.1
+	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+)

go.sum

@@ -1,2 +1,15 @@
+code.gitea.io/sdk v0.11.0 h1:R3VdjBCxObyLKnv4Svd/TM6oGsXzN8JORbzgkEFb83w=
+code.gitea.io/sdk/gitea v0.13.1 h1:Y7bpH2iO6Q0KhhMJfjP/LZ0AmiYITeRQlCD8b0oYqhk=
+code.gitea.io/sdk/gitea v0.13.1/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -13,13 +13,16 @@ import (
 	"sort"
 	"strings"
 
+	"code.gitea.io/sdk/gitea"
 	"github.com/BurntSushi/toml"
+	"golang.org/x/sync/semaphore"
 )
 
 type Config struct {
 	BindTo string
 	Gitea  struct {
 		URL, Org       string
+		MaxConnections int64
 	}
 	Redirect map[string]string
 	Template struct {
@@ -33,6 +36,8 @@ type Application struct {
 	cfg Config
 
 	rxRepoPage, rxRepoImage *regexp.Regexp
+	apiSem                  *semaphore.Weighted
+	gc                      *gitea.Client
 }
 
 func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
@@ -77,7 +82,9 @@ func (this *Application) internalError(w http.ResponseWriter, r *http.Request, e
 }
 
 func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
-	repos, err := this.repos()
+	ctx := r.Context()
+
+	repos, err := this.repos(ctx)
 	if err != nil {
 		this.internalError(w, r, fmt.Errorf("listing repos: %w", err))
 		return
@@ -85,7 +92,7 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
 
 	topics := make(map[string][]string)
 	for _, repo := range repos {
-		if t, err := this.topicsForRepo(repo.Name); err == nil {
+		if t, err := this.topicsForRepo(ctx, repo.Name); err == nil {
 			topics[repo.Name] = t
 		}
 	}
@@ -101,7 +108,7 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
 
 	// But then make sure the final sort is by most-recently-created
 	sort.Slice(repos, func(i, j int) bool {
-		return repos[i].CreatedAt.After(repos[j].CreatedAt)
+		return repos[i].Created.After(repos[j].Created)
 	})
 
 	// Ready for template
@@ -116,7 +123,7 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
 			<option value="data-sort-mt">Recent updates</option>
 		</select>
 		
-		<h2>Projects</h2>
+		<h2>Projects <small>(`+fmt.Sprintf("%d", len(repos))+`)</small></h2>
 		<table id="projtable-main" class="projtable">
 `)
 		for _, repo := range repos {
@@ -146,9 +153,9 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
 			<tr
 				class="`+html.EscapeString(rowClass)+`"
 				data-sort-al="`+fmt.Sprintf("-%d", alphabeticalOrderIndexes[repo.Name])+`"
-				data-sort-ls="`+fmt.Sprintf("%.0f", repo.UpdatedAt.Sub(repo.CreatedAt).Seconds())+`"
+				data-sort-ls="`+fmt.Sprintf("%.0f", repo.Updated.Sub(repo.Created).Seconds())+`"
-				data-sort-ct="`+fmt.Sprintf("%d", repo.CreatedAt.Unix())+`"
+				data-sort-ct="`+fmt.Sprintf("%d", repo.Created.Unix())+`"
-				data-sort-mt="`+fmt.Sprintf("%d", repo.UpdatedAt.Unix())+`"
+				data-sort-mt="`+fmt.Sprintf("%d", repo.Updated.Unix())+`"
 			>
 				<td>
 					<a href="`+pageHref+`"><img class="homeimage" loading="lazy" src="`+html.EscapeString(`/:banner/`+url.PathEscape(repo.Name))+`"></div></a>
@@ -175,8 +182,9 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
 }
 
 func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
+	ctx := r.Context()
 
-	images, err := this.imageFilesForRepo(repoName)
+	images, err := this.imageFilesForRepo(ctx, repoName)
 	if err != nil {
 		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
 		return
@@ -193,11 +201,11 @@ func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repo
 }
 
 func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
-
+	ctx := r.Context()
 	repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
 	extraHead := ""
 
-	readme, err := this.repoFile(repoName, `README.md`)
+	readme, err := this.repoFile(ctx, repoName, `README.md`)
 	if err != nil {
 		this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
 		return
@@ -217,20 +225,20 @@ func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoNa
 		lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
 	}
 
-	readmeHtml, err := this.renderMarkdown(repoName, strings.Join(lines, "\n"))
+	readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
 	if err != nil {
 		this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
 		return
 	}
 
-	images, err := this.imageFilesForRepo(repoName)
+	images, err := this.imageFilesForRepo(ctx, repoName)
 	if err != nil {
 		this.internalError(w, r, fmt.Errorf("listing images: %w", err))
 		return
 	}
 
 	// If the Git repository contains a top-level go.mod file, allow vanity imports
-	if goMod, err := this.repoFile(repoName, `go.mod`); err == nil {
+	if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
 
 		// Check the first line should be `module MODULENAME\n`
 		firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
@@ -376,7 +384,7 @@ func main() {
 
 	_, err := toml.DecodeFile(*configFile, &app.cfg)
 	if err != nil {
-		panic(err)
+		log.Fatalf("toml.DecodeFile: %s", err.Error())
 	}
 
 	// Assert Gitea URL always has trailing slash
@@ -384,5 +392,18 @@ func main() {
 		app.cfg.Gitea.URL += `/`
 	}
 
+	// Create semaphore
+	if app.cfg.Gitea.MaxConnections == 0 { // unlimited
+		app.apiSem = semaphore.NewWeighted(99999)
+	} else {
+		app.apiSem = semaphore.NewWeighted(app.cfg.Gitea.MaxConnections)
+	}
+
+	gc, err := gitea.NewClient(app.cfg.Gitea.URL)
+	if err != nil {
+		log.Fatalf("gitea.NewClient: %s", err.Error())
+	}
+	app.gc = gc
+
 	log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app))
 }

static/style.css

@@ -48,7 +48,7 @@ h1 {
 
 html, body {
 	/* structural */
-	min-height:100%;
+	min-height:100vh;
 	margin:0;
 	border:0;
 	padding:0;
@@ -62,10 +62,10 @@ html, body {
 
 /* Create background pattern by layering two gradients */
 html {
-	background: repeating-linear-gradient(45deg, #FFF, #f8f8f8 5px, #fff 10px);
+	background: repeating-linear-gradient(45deg, #FFF, #f0f0f0 3px, #fff 6px);
 }
 body {
-	background: repeating-linear-gradient(135deg, rgba(255,255,255,0), rgba(255,255,255,255) 5px, rgba(255,255,255,0) 10px);
+	background: repeating-linear-gradient(135deg, rgba(255,255,255,0), rgba(255,255,255,255) 3px, rgba(255,255,255,0) 6px);
 }
 
 #container {
@@ -73,9 +73,7 @@ body {
 	width:960px;
 	position:relative;
 	
-	height:auto !important;
+	min-height:100vh;
-	height:100%; /* oldIE */
-	min-height:100%;
 	
 	/* cosmetic */
 	background:white;
@@ -143,6 +141,9 @@ body {
 	float:left;
 	width: 860px; /* 740px full - 60px rhs column - 2px border */
 }
+.projbody_halfw h2:first-child {
+	margin-top: 0;
+}
 .projbody_fullw {
 	
 }
@@ -153,6 +154,13 @@ body {
 
 /* */
 
+img[src*="shields.io"] {
+	width: auto;
+	height: 20px; /* Set default height to avoid reflow flashing */
+}
+
+/* */
+
 @media screen and (max-width:960px) {
 	
 	#container {