package main import ( "bytes" "context" "encoding/json" "fmt" "io/ioutil" "log" "net/http" "net/url" "strings" "time" ) type Repo struct { Name string `json:"name"` Description string `json:"description"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } type ContentsResponse struct { Content []byte `json:"content"` // Assume base64 "encoding" parameter in Gitea response, and use Go's auto decode } type TopicsResponse struct { Topics []string `json:"topics"` } type ReaddirEntry struct { Name string `json:"name"` Path string `json:"path"` Size int `json:"size"` RawURL string `json:"download_url"` } func (rde ReaddirEntry) isImage() bool { return strings.HasSuffix(rde.Name, `.png`) || strings.HasSuffix(rde.Name, `.jpg`) || strings.HasSuffix(rde.Name, `.jpeg`) } type MarkdownRequest struct { Context string Mode string Text string Wiki bool } type BranchCommit struct { ID string `json:"id"` Message string `json:"message"` Timestamp time.Time `json:"timestamp"` } type Branch struct { Name string `json:"name"` Commit BranchCommit `json:"commit"` } type AuthorInfo struct { Name string `json:"name"` Email string `json:"email"` Date time.Time `json:"date"` } type CommitListEntryCommit struct { Message string `json:"message"` Author AuthorInfo `json:"author"` Committer AuthorInfo `json:"committer"` } type CommitListEntry struct { ID string `json:"sha"` Commit CommitListEntryCommit `json:"commit"` } func (this *Application) apiRequest(ctx context.Context, endpoint string, target interface{}) error { err := this.apiSem.Acquire(ctx, 1) if err != nil { return err // e.g. ctx closed } defer this.apiSem.Release(1) req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+endpoint, nil) if err != nil { return err } resp, err := http.DefaultClient.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != 200 { return fmt.Errorf("HTTP %d", resp.StatusCode) } err = json.NewDecoder(resp.Body).Decode(target) if err != nil { return err } return nil } func (this *Application) branches(ctx context.Context, repo string) ([]Branch, error) { var branches []Branch err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/branches`, &branches) if err != nil { return nil, err // e.g. ctx closed } return branches, nil } func (this *Application) commitsPage(ctx context.Context, repo, ref string, page, limit int) ([]CommitListEntry, error) { var ret []CommitListEntry err := this.apiRequest(ctx, fmt.Sprintf(`api/v1/repos/%s/%s/commits?page=%d&limit=%d`, url.PathEscape(this.cfg.Gitea.Org), url.PathEscape(repo), page, limit), &ret) if err != nil { return nil, err // e.g. ctx closed } return ret, nil } func (this *Application) commits(ctx context.Context, repo, ref string) ([]CommitListEntry, error) { var ret []CommitListEntry nextPage := 1 // Counting starts at 1 for { page, err := this.commitsPage(ctx, repo, ref, nextPage, 300) if err != nil { return nil, err } if len(page) == 0 && len(ret) > 0 { break // Found enough already } ret = append(ret, page...) nextPage += 1 } if len(ret) == 0 { return nil, fmt.Errorf("no commits found") } return ret, nil } func (this *Application) oldestCommit(ctx context.Context, repo, ref string) (CommitListEntry, error) { cc, err := this.commits(ctx, repo, ref) if err != nil { return CommitListEntry{}, err } // Take the last listed entry // TODO maybe need to iterate/sort? return cc[len(cc)-1], nil } // reposPage gets a single page of the list of Git repositories in this organisation. func (this *Application) reposPage(ctx context.Context, page, limit int) ([]Repo, error) { var repos []Repo err := this.apiRequest(ctx, `api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), &repos) if err != nil { return nil, err } return repos, nil } // repos gets a list of Git repositories in this organisation. It may have to // make multiple network requests. func (this *Application) repos(ctx context.Context) ([]Repo, error) { // Seems like gitea-1.13.0-rc1 returns 30 results by default, and supports up to a limit of 100 // Make a much larger request ret := make([]Repo, 0) nextPage := 1 // Counting starts at 1 for { page, err := this.reposPage(ctx, nextPage, 300) if err != nil { return nil, err } if len(page) == 0 && len(ret) > 0 { break // Found enough already } ret = append(ret, page...) nextPage += 1 } // The Created/Modified times aren't very good // Replace them with the earliest/latest commit dates we can find for i, rr := range ret { // The most recent commit will be the head of one of the branches (easy to find) brs, err := this.branches(ctx, rr.Name) if err != nil { log.Printf("loading branches for '%s': %s", rr.Name, err) continue } newestCommit := time.Unix(0, 0) // sentinel for _, br := range brs { if br.Commit.Timestamp.After(newestCommit) { newestCommit = br.Commit.Timestamp } } if !newestCommit.Equal(time.Unix(0, 0)) { ret[i].UpdatedAt = newestCommit // replace it } } // Separate loop for oldest-commits, in case we needed to continue/break out // of the earliest-commit loop for i, rr := range ret { // The oldest commit needs us to page through the commit history to find it oldestCommit, err := this.oldestCommit(ctx, rr.Name, "") if err != nil { log.Printf("finding oldest commit for '%s': %s", rr.Name, err) continue } ret[i].CreatedAt = oldestCommit.Commit.Author.Date } return ret, nil } // repoFile gets a single file from the default branch of the git repository // Usually the default branch is `master`. func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) { var cr ContentsResponse err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), &cr) if err != nil { return nil, err } return cr.Content, nil } func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) { err := this.apiSem.Acquire(ctx, 1) if err != nil { return nil, err // e.g. ctx closed } defer this.apiSem.Release(1) req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped if err != nil { return nil, err } resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != 200 { // "No files found" happens with a HTTP 500/404 error depending on Gitea version. Catch this special case if resp.StatusCode == 500 || resp.StatusCode == 404 { b, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } if strings.Contains(string(b), `does not exist`) { return []ReaddirEntry{}, nil // no files found } } return nil, fmt.Errorf("HTTP %d", resp.StatusCode) } var ret []ReaddirEntry err = json.NewDecoder(resp.Body).Decode(&ret) if err != nil { return nil, err } return ret, nil } // imageFilesForRepo finds documentation images for the repository. // It searches the dist/ and doc/ subdirectories. func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]ReaddirEntry, error) { ret := []ReaddirEntry{} for _, dirName := range []string{`dist`, `doc`} { files, err := this.filesInDirectory(ctx, repo, dirName) if err != nil { return nil, fmt.Errorf("readdir(%s): %w", dirName, err) } for _, f := range files { if f.isImage() { ret = append(ret, f) } } } return ret, nil } func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) { var tr TopicsResponse err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, &tr) if err != nil { return nil, err } return tr.Topics, nil } // renderMarkdown calls the remote Gitea server's own markdown renderer. func (this *Application) renderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) { err := this.apiSem.Acquire(ctx, 1) if err != nil { return nil, err // e.g. ctx closed } defer this.apiSem.Release(1) jb, err := json.Marshal(MarkdownRequest{ Context: this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) + `/src/branch/master`, Mode: "gfm", // magic constant - Github Flavoured Markdown Text: body, }) if err != nil { return nil, err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/`, bytes.NewReader(jb)) if err != nil { return nil, err } req.Header.Set(`Content-Type`, `application/json`) req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(jb))) resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("HTTP %d", resp.StatusCode) } return ioutil.ReadAll(resp.Body) } // renderMarkdownRaw calls the remote Gitea server's own markdown renderer. func (this *Application) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) { err := this.apiSem.Acquire(ctx, 1) if err != nil { return nil, err // e.g. ctx closed } defer this.apiSem.Release(1) req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/raw`, bytes.NewReader(body)) if err != nil { return nil, err } req.Header.Set(`Content-Type`, `text/plain`) req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(body))) resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("HTTP %d", resp.StatusCode) } return ioutil.ReadAll(resp.Body) }