code.ivysaur.me/teafolio
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

support semaphore limiting + context-cancellation of Gitea API requests

Browse Source
This commit is contained in:
mappu 2020-05-24 18:39:24 +12:00
parent 66362fc856
commit 2eb39cbeac
5 changed files with 98 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
api.go
View File

@ -2,6 +2,7 @@ package main
import ( import (
"bytes" "bytes"
"context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
@ -45,8 +46,16 @@ type MarkdownRequest struct {
} }
// repos gets a list of Git repositories in this organisation. // repos gets a list of Git repositories in this organisation.
func (this *Application) repos() ([]Repo, error) { func (this *Application) repos(ctx context.Context) ([]Repo, error) {
resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/orgs/` + url.PathEscape(this.cfg.Gitea.Org) + `/repos`) this.apiSem.Acquire(ctx, 1)
defer this.apiSem.Release(1)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+`/repos`, nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -67,8 +76,16 @@ func (this *Application) repos() ([]Repo, error) {
// repoFile gets a single file from the default branch of the git repository // repoFile gets a single file from the default branch of the git repository
// Usually the default branch is `master`. // Usually the default branch is `master`.
func (this *Application) repoFile(repo, filename string) ([]byte, error) { func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) {
resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/contents/` + url.PathEscape(filename)) this.apiSem.Acquire(ctx, 1)
defer this.apiSem.Release(1)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -87,8 +104,16 @@ func (this *Application) repoFile(repo, filename string) ([]byte, error) {
return cr.Content, nil return cr.Content, nil
} }
func (this *Application) filesInDirectory(repo, dir string) ([]ReaddirEntry, error) { func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) {
resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/contents/` + dir) // n.b. $dir param not escaped this.apiSem.Acquire(ctx, 1)
defer this.apiSem.Release(1)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -121,13 +146,13 @@ func (this *Application) filesInDirectory(repo, dir string) ([]ReaddirEntry, err
// imageFilesForRepo finds documentation images for the repository. // imageFilesForRepo finds documentation images for the repository.
// It searches the dist/ and doc/ subdirectories. // It searches the dist/ and doc/ subdirectories.
func (this *Application) imageFilesForRepo(repo string) ([]ReaddirEntry, error) { func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]ReaddirEntry, error) {
ret := []ReaddirEntry{} ret := []ReaddirEntry{}
for _, dirName := range []string{`dist`, `doc`} { for _, dirName := range []string{`dist`, `doc`} {
files, err := this.filesInDirectory(repo, dirName) files, err := this.filesInDirectory(ctx, repo, dirName)
if err != nil { if err != nil {
return nil, fmt.Errorf("readdir(%s): %w", dirName, err) return nil, fmt.Errorf("readdir(%s): %w", dirName, err)
} }
@ -142,8 +167,16 @@ func (this *Application) imageFilesForRepo(repo string) ([]ReaddirEntry, error)
return ret, nil return ret, nil
} }
func (this *Application) topicsForRepo(repo string) ([]string, error) { func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
resp, err := http.Get(this.cfg.Gitea.URL + `api/v1/repos/` + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repo) + `/topics`) this.apiSem.Acquire(ctx, 1)
defer this.apiSem.Release(1)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -163,19 +196,27 @@ func (this *Application) topicsForRepo(repo string) ([]string, error) {
} }
// renderMarkdown calls the remote Gitea server's own markdown renderer. // renderMarkdown calls the remote Gitea server's own markdown renderer.
func (this *Application) renderMarkdown(repoName string, body string) ([]byte, error) { func (this *Application) renderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) {
req := MarkdownRequest{ this.apiSem.Acquire(ctx, 1)
defer this.apiSem.Release(1)
jb, err := json.Marshal(MarkdownRequest{
Context: this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) + `/src/branch/master`, Context: this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) + `/src/branch/master`,
Mode: "gfm", // magic constant - Github Flavoured Markdown Mode: "gfm", // magic constant - Github Flavoured Markdown
Text: body, Text: body,
} })
jb, err := json.Marshal(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
resp, err := http.Post(this.cfg.Gitea.URL+`api/v1/markdown/`, `application/json`, bytes.NewReader(jb)) req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/`, bytes.NewReader(jb))
if err != nil {
return nil, err
}
req.Header.Set(`Content-Type`, `application/json`)
req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(jb)))
resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -189,8 +230,18 @@ func (this *Application) renderMarkdown(repoName string, body string) ([]byte, e
} }
// renderMarkdownRaw calls the remote Gitea server's own markdown renderer. // renderMarkdownRaw calls the remote Gitea server's own markdown renderer.
func (this *Application) renderMarkdownRaw(body []byte) ([]byte, error) { func (this *Application) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) {
resp, err := http.Post(this.cfg.Gitea.URL+`api/v1/markdown/raw`, `text/plain`, bytes.NewReader(body)) this.apiSem.Acquire(ctx, 1)
defer this.apiSem.Release(1)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/raw`, bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set(`Content-Type`, `text/plain`)
req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(body)))
resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }

1
config.toml.sample
View File

@ -5,6 +5,7 @@ BindTo="0.0.0.0:5656"
[Gitea] [Gitea]
URL="https://gitea.com/" URL="https://gitea.com/"
Org="gitea" Org="gitea"
MaxConnections=2 # Use zero for unlimited
[Redirect] [Redirect]
"old-project-name" = "new-project-name" "old-project-name" = "new-project-name"

5
go.mod
View File

@ -2,4 +2,7 @@ module teafolio
go 1.13 go 1.13
require github.com/BurntSushi/toml v0.3.1 require (
github.com/BurntSushi/toml v0.3.1
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
)

2
go.sum
View File

@ -1,2 +1,4 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

31
main.go
View File

@ -14,12 +14,14 @@ import (
"strings" "strings"
"github.com/BurntSushi/toml" "github.com/BurntSushi/toml"
"golang.org/x/sync/semaphore"
) )
type Config struct { type Config struct {
BindTo string BindTo string
Gitea struct { Gitea struct {
URL, Org string URL, Org string
MaxConnections int64
} }
Redirect map[string]string Redirect map[string]string
Template struct { Template struct {
@ -33,6 +35,7 @@ type Application struct {
cfg Config cfg Config
rxRepoPage, rxRepoImage *regexp.Regexp rxRepoPage, rxRepoImage *regexp.Regexp
apiSem *semaphore.Weighted
} }
func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) { func (this *Application) Templatepage(w http.ResponseWriter, r *http.Request, pageDesc, extraHead string, cb func()) {
@ -77,7 +80,9 @@ func (this *Application) internalError(w http.ResponseWriter, r *http.Request, e
} }
func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) { func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
repos, err := this.repos() ctx := r.Context()
repos, err := this.repos(ctx)
if err != nil { if err != nil {
this.internalError(w, r, fmt.Errorf("listing repos: %w", err)) this.internalError(w, r, fmt.Errorf("listing repos: %w", err))
return return
@ -85,7 +90,7 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
topics := make(map[string][]string) topics := make(map[string][]string)
for _, repo := range repos { for _, repo := range repos {
if t, err := this.topicsForRepo(repo.Name); err == nil { if t, err := this.topicsForRepo(ctx, repo.Name); err == nil {
topics[repo.Name] = t topics[repo.Name] = t
} }
} }
@ -175,8 +180,9 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
} }
func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) { func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repoName string) {
ctx := r.Context()
images, err := this.imageFilesForRepo(repoName) images, err := this.imageFilesForRepo(ctx, repoName)
if err != nil { if err != nil {
this.internalError(w, r, fmt.Errorf("listing images: %w", err)) this.internalError(w, r, fmt.Errorf("listing images: %w", err))
return return
@ -193,11 +199,11 @@ func (this *Application) Bannerpage(w http.ResponseWriter, r *http.Request, repo
} }
func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) { func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoName string) {
ctx := r.Context()
repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) repoURL := this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName)
extraHead := "" extraHead := ""
readme, err := this.repoFile(repoName, `README.md`) readme, err := this.repoFile(ctx, repoName, `README.md`)
if err != nil { if err != nil {
this.internalError(w, r, fmt.Errorf("loading README.md: %w", err)) this.internalError(w, r, fmt.Errorf("loading README.md: %w", err))
return return
@ -217,20 +223,20 @@ func (this *Application) Repopage(w http.ResponseWriter, r *http.Request, repoNa
lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...) lines = append([]string{lines[0], lines[1], extraBadgesMd, ""}, lines[2:]...)
} }
readmeHtml, err := this.renderMarkdown(repoName, strings.Join(lines, "\n")) readmeHtml, err := this.renderMarkdown(ctx, repoName, strings.Join(lines, "\n"))
if err != nil { if err != nil {
this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err)) this.internalError(w, r, fmt.Errorf("rendering markdown: %w", err))
return return
} }
images, err := this.imageFilesForRepo(repoName) images, err := this.imageFilesForRepo(ctx, repoName)
if err != nil { if err != nil {
this.internalError(w, r, fmt.Errorf("listing images: %w", err)) this.internalError(w, r, fmt.Errorf("listing images: %w", err))
return return
} }
// If the Git repository contains a top-level go.mod file, allow vanity imports // If the Git repository contains a top-level go.mod file, allow vanity imports
if goMod, err := this.repoFile(repoName, `go.mod`); err == nil { if goMod, err := this.repoFile(ctx, repoName, `go.mod`); err == nil {
// Check the first line should be `module MODULENAME\n` // Check the first line should be `module MODULENAME\n`
firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0] firstLine := bytes.SplitN(goMod, []byte("\n"), 2)[0]
@ -384,5 +390,12 @@ func main() {
app.cfg.Gitea.URL += `/` app.cfg.Gitea.URL += `/`
} }
// Create semaphore
if app.cfg.Gitea.MaxConnections == 0 { // unlimited
app.apiSem = semaphore.NewWeighted(99999)
} else {
app.apiSem = semaphore.NewWeighted(app.cfg.Gitea.MaxConnections)
}
log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app)) log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app))
} }