doc/changelog: v1.1.1

README.md

@@ -31,6 +31,10 @@ dokku storage:mount teafolio /srv/teafolio-dokku/config.toml:/app/config.toml
 
 ## CHANGELOG
 
+2020-11-08 v1.1.1
+- Fix an issue with newer versions of Gitea that paginate repoistory list responses
+- Fix an issue with blocking semaphores for a cancelled network request
+
 2020-05-24 v1.1.0
 - Support limiting the number of concurrent API requests to Gitea
 - Display total number of projects

api.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"strings"
@@ -45,12 +46,15 @@ type MarkdownRequest struct {
 	Wiki    bool
 }
 
-// repos gets a list of Git repositories in this organisation.
-func (this *Application) repos(ctx context.Context) ([]Repo, error) {
-	this.apiSem.Acquire(ctx, 1)
+// reposPage gets a single page of the list of Git repositories in this organisation.
+func (this *Application) reposPage(ctx context.Context, page, limit int) ([]Repo, error) {
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+`/repos`, nil)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -74,10 +78,39 @@ func (this *Application) repos(ctx context.Context) ([]Repo, error) {
 	return repos, nil
 }
 
+// repos gets a list of Git repositories in this organisation. It may have to
+// make multiple network requests.
+func (this *Application) repos(ctx context.Context) ([]Repo, error) {
+	// Seems like gitea-1.13.0-rc1 returns 30 results by default, and supports up to a limit of 100
+	// Make a much larger request
+
+	ret := make([]Repo, 0)
+
+	nextPage := 1 // Counting starts at 1
+	for {
+		page, err := this.reposPage(ctx, nextPage, 300)
+		if err != nil {
+			return nil, err
+		}
+
+		log.Printf("Page %d with %d results", nextPage, len(page))
+
+		if len(page) == 0 && len(ret) > 0 {
+			return ret, nil // Found enough already
+		}
+
+		ret = append(ret, page...)
+		nextPage += 1
+	}
+}
+
 // repoFile gets a single file from the default branch of the git repository
 // Usually the default branch is `master`.
 func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) {
-	this.apiSem.Acquire(ctx, 1)
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), nil)
@@ -105,7 +138,10 @@ func (this *Application) repoFile(ctx context.Context, repo, filename string) ([
 }
 
 func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) {
-	this.apiSem.Acquire(ctx, 1)
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped
@@ -168,7 +204,10 @@ func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]
 }
 
 func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
-	this.apiSem.Acquire(ctx, 1)
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, nil)
@@ -197,7 +236,10 @@ func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]stri
 
 // renderMarkdown calls the remote Gitea server's own markdown renderer.
 func (this *Application) renderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) {
-	this.apiSem.Acquire(ctx, 1)
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
 	jb, err := json.Marshal(MarkdownRequest{
@@ -231,7 +273,10 @@ func (this *Application) renderMarkdown(ctx context.Context, repoName string, bo
 
 // renderMarkdownRaw calls the remote Gitea server's own markdown renderer.
 func (this *Application) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) {
-	this.apiSem.Acquire(ctx, 1)
+	err := this.apiSem.Acquire(ctx, 1)
+	if err != nil {
+		return nil, err // e.g. ctx closed
+	}
 	defer this.apiSem.Release(1)
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/raw`, bytes.NewReader(body))