mappu
90cd2b6440
For codesite-migrated repositories, looking at the oldest commit is preferable to determine the "create date". But for forked projects, looking at the oldest commit is incorrect for when we started the project If inferring the real create date has to be manual, then let's rely on the Gitea metadata - it's faster and can be modified by hand if needed
373 lines
9.3 KiB
Go
373 lines
9.3 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type Repo struct {
|
|
Name string `json:"name"`
|
|
Description string `json:"description"`
|
|
GiteaCreated time.Time `json:"created_at"`
|
|
GiteaUpdated time.Time `json:"updated_at"`
|
|
|
|
newestCommit time.Time
|
|
topics []string
|
|
}
|
|
|
|
func (this *Application) populateCommitInfo(ctx context.Context, rr *Repo) {
|
|
|
|
// The most recent commit will be the head of one of the branches (easy to find)
|
|
|
|
brs, err := this.branches(ctx, rr.Name)
|
|
if err != nil {
|
|
log.Printf("loading branches for '%s': %s", rr.Name, err)
|
|
rr.newestCommit = rr.GiteaUpdated // best guess
|
|
|
|
} else {
|
|
|
|
newestCommit := time.Unix(0, 0) // sentinel
|
|
for _, br := range brs {
|
|
if br.Commit.Timestamp.After(newestCommit) {
|
|
newestCommit = br.Commit.Timestamp
|
|
}
|
|
}
|
|
if !newestCommit.Equal(time.Unix(0, 0)) {
|
|
rr.newestCommit = newestCommit // replace it
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
type ContentsResponse struct {
|
|
Content []byte `json:"content"` // Assume base64 "encoding" parameter in Gitea response, and use Go's auto decode
|
|
}
|
|
|
|
type TopicsResponse struct {
|
|
Topics []string `json:"topics"`
|
|
}
|
|
|
|
type ReaddirEntry struct {
|
|
Name string `json:"name"`
|
|
Path string `json:"path"`
|
|
Size int `json:"size"`
|
|
RawURL string `json:"download_url"`
|
|
}
|
|
|
|
func (rde ReaddirEntry) isImage() bool {
|
|
return strings.HasSuffix(rde.Name, `.png`) || strings.HasSuffix(rde.Name, `.jpg`) || strings.HasSuffix(rde.Name, `.jpeg`)
|
|
}
|
|
|
|
type MarkdownRequest struct {
|
|
Context string
|
|
Mode string
|
|
Text string
|
|
Wiki bool
|
|
}
|
|
|
|
type BranchCommit struct {
|
|
ID string `json:"id"`
|
|
Message string `json:"message"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
}
|
|
|
|
type Branch struct {
|
|
Name string `json:"name"`
|
|
Commit BranchCommit `json:"commit"`
|
|
}
|
|
|
|
type AuthorInfo struct {
|
|
Name string `json:"name"`
|
|
Email string `json:"email"`
|
|
Date time.Time `json:"date"`
|
|
}
|
|
|
|
type CommitListEntryCommit struct {
|
|
Message string `json:"message"`
|
|
Author AuthorInfo `json:"author"`
|
|
Committer AuthorInfo `json:"committer"`
|
|
}
|
|
|
|
type CommitListEntry struct {
|
|
ID string `json:"sha"`
|
|
Commit CommitListEntryCommit `json:"commit"`
|
|
}
|
|
|
|
func (this *Application) apiRequest(ctx context.Context, endpoint string, target interface{}) error {
|
|
err := this.apiSem.Acquire(ctx, 1)
|
|
if err != nil {
|
|
return err // e.g. ctx closed
|
|
}
|
|
defer this.apiSem.Release(1)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+endpoint, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != 200 {
|
|
return fmt.Errorf("HTTP %d", resp.StatusCode)
|
|
}
|
|
|
|
err = json.NewDecoder(resp.Body).Decode(target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (this *Application) branches(ctx context.Context, repo string) ([]Branch, error) {
|
|
var branches []Branch
|
|
|
|
err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/branches`, &branches)
|
|
if err != nil {
|
|
return nil, err // e.g. ctx closed
|
|
}
|
|
|
|
return branches, nil
|
|
}
|
|
|
|
func (this *Application) commitsPage(ctx context.Context, repo, ref string, page, limit int) ([]CommitListEntry, error) {
|
|
var ret []CommitListEntry
|
|
|
|
err := this.apiRequest(ctx, fmt.Sprintf(`api/v1/repos/%s/%s/commits?page=%d&limit=%d`, url.PathEscape(this.cfg.Gitea.Org), url.PathEscape(repo), page, limit), &ret)
|
|
if err != nil {
|
|
return nil, err // e.g. ctx closed
|
|
}
|
|
|
|
return ret, nil
|
|
}
|
|
|
|
func (this *Application) commits(ctx context.Context, repo, ref string) ([]CommitListEntry, error) {
|
|
var ret []CommitListEntry
|
|
|
|
nextPage := 1 // Counting starts at 1
|
|
for {
|
|
page, err := this.commitsPage(ctx, repo, ref, nextPage, 300)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(page) == 0 && len(ret) > 0 {
|
|
break // Found enough already
|
|
}
|
|
|
|
ret = append(ret, page...)
|
|
nextPage += 1
|
|
}
|
|
|
|
if len(ret) == 0 {
|
|
return nil, fmt.Errorf("no commits found")
|
|
}
|
|
|
|
return ret, nil
|
|
}
|
|
|
|
// reposPage gets a single page of the list of Git repositories in this organisation.
|
|
func (this *Application) reposPage(ctx context.Context, page, limit int) ([]Repo, error) {
|
|
var repos []Repo
|
|
|
|
err := this.apiRequest(ctx, `api/v1/orgs/`+url.PathEscape(this.cfg.Gitea.Org)+fmt.Sprintf(`/repos?page=%d&limit=%d`, page, limit), &repos)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return repos, nil
|
|
}
|
|
|
|
// repos gets a list of Git repositories in this organisation. It may have to
|
|
// make multiple network requests.
|
|
func (this *Application) repos(ctx context.Context) ([]Repo, error) {
|
|
// Seems like gitea-1.13.0-rc1 returns 30 results by default, and supports up to a limit of 100
|
|
// Make a much larger request
|
|
|
|
ret := make([]Repo, 0)
|
|
|
|
nextPage := 1 // Counting starts at 1
|
|
for {
|
|
page, err := this.reposPage(ctx, nextPage, 300)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(page) == 0 && len(ret) > 0 {
|
|
break // Found enough already
|
|
}
|
|
|
|
ret = append(ret, page...)
|
|
nextPage += 1
|
|
}
|
|
|
|
return ret, nil
|
|
}
|
|
|
|
// repoFile gets a single file from the default branch of the git repository
|
|
// Usually the default branch is `master`.
|
|
func (this *Application) repoFile(ctx context.Context, repo, filename string) ([]byte, error) {
|
|
var cr ContentsResponse
|
|
|
|
err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+url.PathEscape(filename), &cr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return cr.Content, nil
|
|
}
|
|
|
|
func (this *Application) filesInDirectory(ctx context.Context, repo, dir string) ([]ReaddirEntry, error) {
|
|
err := this.apiSem.Acquire(ctx, 1)
|
|
if err != nil {
|
|
return nil, err // e.g. ctx closed
|
|
}
|
|
defer this.apiSem.Release(1)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, this.cfg.Gitea.URL+`api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/contents/`+dir, nil) // n.b. $dir param not escaped
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != 200 {
|
|
|
|
// "No files found" happens with a HTTP 500/404 error depending on Gitea version. Catch this special case
|
|
if resp.StatusCode == 500 || resp.StatusCode == 404 {
|
|
b, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if strings.Contains(string(b), `does not exist`) {
|
|
return []ReaddirEntry{}, nil // no files found
|
|
}
|
|
}
|
|
|
|
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
|
}
|
|
|
|
var ret []ReaddirEntry
|
|
err = json.NewDecoder(resp.Body).Decode(&ret)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return ret, nil
|
|
}
|
|
|
|
// imageFilesForRepo finds documentation images for the repository.
|
|
// It searches the dist/ and doc/ subdirectories.
|
|
func (this *Application) imageFilesForRepo(ctx context.Context, repo string) ([]ReaddirEntry, error) {
|
|
|
|
ret := []ReaddirEntry{}
|
|
|
|
for _, dirName := range []string{`dist`, `doc`} {
|
|
|
|
files, err := this.filesInDirectory(ctx, repo, dirName)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("readdir(%s): %w", dirName, err)
|
|
}
|
|
|
|
for _, f := range files {
|
|
if f.isImage() {
|
|
ret = append(ret, f)
|
|
}
|
|
}
|
|
}
|
|
|
|
return ret, nil
|
|
}
|
|
|
|
func (this *Application) topicsForRepo(ctx context.Context, repo string) ([]string, error) {
|
|
var tr TopicsResponse
|
|
|
|
err := this.apiRequest(ctx, `api/v1/repos/`+url.PathEscape(this.cfg.Gitea.Org)+`/`+url.PathEscape(repo)+`/topics`, &tr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return tr.Topics, nil
|
|
}
|
|
|
|
// renderMarkdown calls the remote Gitea server's own markdown renderer.
|
|
func (this *Application) renderMarkdown(ctx context.Context, repoName string, body string) ([]byte, error) {
|
|
err := this.apiSem.Acquire(ctx, 1)
|
|
if err != nil {
|
|
return nil, err // e.g. ctx closed
|
|
}
|
|
defer this.apiSem.Release(1)
|
|
|
|
jb, err := json.Marshal(MarkdownRequest{
|
|
Context: this.cfg.Gitea.URL + url.PathEscape(this.cfg.Gitea.Org) + `/` + url.PathEscape(repoName) + `/src/branch/master`,
|
|
Mode: "gfm", // magic constant - Github Flavoured Markdown
|
|
Text: body,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/`, bytes.NewReader(jb))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set(`Content-Type`, `application/json`)
|
|
req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(jb)))
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != 200 {
|
|
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
|
}
|
|
|
|
return ioutil.ReadAll(resp.Body)
|
|
}
|
|
|
|
// renderMarkdownRaw calls the remote Gitea server's own markdown renderer.
|
|
func (this *Application) renderMarkdownRaw(ctx context.Context, body []byte) ([]byte, error) {
|
|
err := this.apiSem.Acquire(ctx, 1)
|
|
if err != nil {
|
|
return nil, err // e.g. ctx closed
|
|
}
|
|
defer this.apiSem.Release(1)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, this.cfg.Gitea.URL+`api/v1/markdown/raw`, bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set(`Content-Type`, `text/plain`)
|
|
req.Header.Set(`Content-Length`, fmt.Sprintf("%d", len(body)))
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != 200 {
|
|
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
|
}
|
|
|
|
return ioutil.ReadAll(resp.Body)
|
|
}
|