From 2eb39cbeac37a47d0073d89c94a69428c7b88fb4 Mon Sep 17 00:00:00 2001 From: mappu Date: Sun, 24 May 2020 18:39:24 +1200 Subject: [PATCH] support semaphore limiting + context-cancellation of Gitea API requests --- api.go | 87 ++++++++++++++++++++++++++++++++++++---------- config.toml.sample | 1 + go.mod | 5 ++- go.sum | 2 ++ main.go | 31 ++++++++++++----- 5 files changed, 98 insertions(+), 28 deletions(-) diff --git a/api.go b/api.go index e6e35cd..50ca187 100644 --- a/api.go +++ b/api.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/json" "fmt" "io/ioutil" @@ -45,8 +46,16 @@ type MarkdownRequest struct { } // repos gets a list of Git repositories in this organisation. -func (this *Application) repos() ([]Repo, error) { - resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/orgs/` + url.PathEscape(this.cfg.Gitea.Org) + `/repos`) +func (this *Application) repos(ctx context.Context) ([]Repo, error) { + this.apiSem.Acquire(ctx, 1) + defer this.apiSem.Release(1) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+`/repos`, nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } @@ -67,8 +76,16 @@ func (this *Application) repos() ([]Repo, error) { // repoFile gets a single file from the default branch of the git repository // Usually the default branch is `master`. -func (this *Application) repoFile(repo, filename string) ([]byte, error) { - resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/contents/` + url.PathEscape(filename)) +func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) { + this.apiSem.Acquire(ctx, 1) + defer this.apiSem.Release(1) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } @@ -87,8 +104,16 @@ func (this *Application) repoFile(repo, filename string) ([]byte, error) { return cr.Content, nil } -func (this *Application) filesInDirectory(repo, dir string) ([]ReaddirEntry, error) { - resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/contents/` + dir) // n.b. $dir param not escaped +func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) { + this.apiSem.Acquire(ctx, 1) + defer this.apiSem.Release(1) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } @@ -121,13 +146,13 @@ func (this *Application) filesInDirectory(repo, dir string) ([]ReaddirEntry, err // imageFilesForRepo finds documentation images for the repository. // It searches the dist/ and doc/ subdirectories. -func (this *Application) imageFilesForRepo(repo string) ([]ReaddirEntry, error) { +func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]ReaddirEntry, error) { ret := []ReaddirEntry{} for _, dirName := range []string{`dist`, `doc`} { - files, err := this.filesInDirectory(repo, dirName) + files, err := this.filesInDirectory(ctx, repo, dirName) if err != nil { return nil, fmt.Errorf("readdir(%s): %w", dirName, err) } @@ -142,8 +167,16 @@ func (this *Application) imageFilesForRepo(repo string) ([]ReaddirEntry, error) return ret, nil } -func (this *Application) topicsForRepo(repo string) ([]string, error) { - resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/topics`) +func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) { + this.apiSem.Acquire(ctx, 1) + defer this.apiSem.Release(1) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } @@ -163,19 +196,27 @@ func (this *Application) topicsForRepo(repo string) ([]string, error) { } // renderMarkdown calls the remote Gitea server's own markdown renderer. -func (this *Application) renderMarkdown(repoName string, body string) ([]byte, error) { - req := MarkdownRequest{ +func (this *Application) renderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) { + this.apiSem.Acquire(ctx, 1) + defer this.apiSem.Release(1) + + jb, err := json.Marshal(MarkdownRequest{ Context: this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) + `/src/branch/master`, Mode: "gfm", // magic constant - Github Flavoured Markdown Text: body, - } - - jb, err := json.Marshal(req) + }) if err != nil { return nil, err } - resp, err := http.Post(this.cfg.Gitea.URL+`api/v1/markdown/`, `application/json`, bytes.NewReader(jb)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/`, bytes.NewReader(jb)) + if err != nil { + return nil, err + } + req.Header.Set(`Content-Type`, `application/json`) + req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(jb))) + + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } @@ -189,8 +230,18 @@ func (this *Application) renderMarkdown(repoName string, body string) ([]byte, e } // renderMarkdownRaw calls the remote Gitea server's own markdown renderer. -func (this *Application) renderMarkdownRaw(body []byte) ([]byte, error) { - resp, err := http.Post(this.cfg.Gitea.URL+`api/v1/markdown/raw`, `text/plain`, bytes.NewReader(body)) +func (this *Application) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) { + this.apiSem.Acquire(ctx, 1) + defer this.apiSem.Release(1) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/raw`, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set(`Content-Type`, `text/plain`) + req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(body))) + + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } diff --git a/config.toml.sample b/config.toml.sample index aa4f0dd..43544a6 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -5,6 +5,7 @@ BindTo="0.0.0.0:5656" [Gitea] URL="https://gitea.com/" Org="gitea" +MaxConnections=2 # Use zero for unlimited [Redirect] "old-project-name" = "new-project-name" diff --git a/go.mod b/go.mod index 21f55e5..396a55d 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module teafolio go 1.13 -require github.com/BurntSushi/toml v0.3.1 +require ( + github.com/BurntSushi/toml v0.3.1 + golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a +) diff --git a/go.sum b/go.sum index 9cb2df8..98d7877 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/main.go b/main.go index ec69fab..a9c7b57 100644 --- a/main.go +++ b/main.go @@ -14,12 +14,14 @@ import ( "strings" "github.com/BurntSushi/toml" + "golang.org/x/sync/semaphore" ) type Config struct { BindTo string Gitea struct { - URL, Org string + URL, Org string + MaxConnections int64 } Redirect map[string]string Template struct { @@ -33,6 +35,7 @@ type Application struct { cfg Config rxRepoPage, rxRepoImage *regexp.Regexp + apiSem *semaphore.Weighted } func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) { @@ -77,7 +80,9 @@ func (this *Application) internalError(w http.ResponseWriter, r *http.Request, e } func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) { - repos, err := this.repos() + ctx := r.Context() + + repos, err := this.repos(ctx) if err != nil { this.internalError(w, r, fmt.Errorf("listing repos: %w", err)) return @@ -85,7 +90,7 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) { topics := make(map[string][]string) for _, repo := range repos { - if t, err := this.topicsForRepo(repo.Name); err == nil { + if t, err := this.topicsForRepo(ctx, repo.Name); err == nil { topics[repo.Name] = t } } @@ -175,8 +180,9 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) { } func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) { + ctx := r.Context() - images, err := this.imageFilesForRepo(repoName) + images, err := this.imageFilesForRepo(ctx, repoName) if err != nil { this.internalError(w, r, fmt.Errorf("listing images: %w", err)) return @@ -193,11 +199,11 @@ func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repo } func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) { - + ctx := r.Context() repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) extraHead := "" - readme, err := this.repoFile(repoName, `README.md`) + readme, err := this.repoFile(ctx, repoName, `README.md`) if err != nil { this.internalError(w, r, fmt.Errorf("loading README.md: %w", err)) return @@ -217,20 +223,20 @@ func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoNa lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...) } - readmeHtml, err := this.renderMarkdown(repoName, strings.Join(lines, "\n")) + readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n")) if err != nil { this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err)) return } - images, err := this.imageFilesForRepo(repoName) + images, err := this.imageFilesForRepo(ctx, repoName) if err != nil { this.internalError(w, r, fmt.Errorf("listing images: %w", err)) return } // If the Git repository contains a top-level go.mod file, allow vanity imports - if goMod, err := this.repoFile(repoName, `go.mod`); err == nil { + if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil { // Check the first line should be `module MODULENAME\n` firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0] @@ -384,5 +390,12 @@ func main() { app.cfg.Gitea.URL += `/` } + // Create semaphore + if app.cfg.Gitea.MaxConnections == 0 { // unlimited + app.apiSem = semaphore.NewWeighted(99999) + } else { + app.apiSem = semaphore.NewWeighted(app.cfg.Gitea.MaxConnections) + } + log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app)) }