diff --git a/api.go b/api.go index ca307e8..6d66bda 100644 --- a/api.go +++ b/api.go @@ -14,10 +14,48 @@ import ( ) type Repo struct { - Name string `json:"name"` - Description string `json:"description"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + Name string `json:"name"` + Description string `json:"description"` + GiteaCreated time.Time `json:"created_at"` + GiteaUpdated time.Time `json:"updated_at"` + + oldestCommit time.Time + newestCommit time.Time + topics []string +} + +func (this *Application) populateCommitInfo(ctx context.Context, rr *Repo) { + + // The most recent commit will be the head of one of the branches (easy to find) + + brs, err := this.branches(ctx, rr.Name) + if err != nil { + log.Printf("loading branches for '%s': %s", rr.Name, err) + rr.newestCommit = rr.GiteaUpdated // best guess + + } else { + + newestCommit := time.Unix(0, 0) // sentinel + for _, br := range brs { + if br.Commit.Timestamp.After(newestCommit) { + newestCommit = br.Commit.Timestamp + } + } + if !newestCommit.Equal(time.Unix(0, 0)) { + rr.newestCommit = newestCommit // replace it + } + } + + // The oldest commit needs us to page through the commit history to find it + + oldestCommit, err := this.oldestCommit(ctx, rr.Name, "") + if err != nil { + log.Printf("finding oldest commit for '%s': %s", rr.Name, err) + rr.oldestCommit = rr.GiteaCreated // best guess + + } else { + rr.oldestCommit = oldestCommit.Commit.Author.Date + } } type ContentsResponse struct { @@ -197,46 +235,6 @@ func (this *Application) repos(ctx context.Context) ([]Repo, error) { nextPage += 1 } - // The Created/Modified times aren't very good - // Replace them with the earliest/latest commit dates we can find - - for i, rr := range ret { - - // The most recent commit will be the head of one of the branches (easy to find) - - brs, err := this.branches(ctx, rr.Name) - if err != nil { - log.Printf("loading branches for '%s': %s", rr.Name, err) - continue - } - - newestCommit := time.Unix(0, 0) // sentinel - for _, br := range brs { - if br.Commit.Timestamp.After(newestCommit) { - newestCommit = br.Commit.Timestamp - } - } - if !newestCommit.Equal(time.Unix(0, 0)) { - ret[i].UpdatedAt = newestCommit // replace it - } - } - - // Separate loop for oldest-commits, in case we needed to continue/break out - // of the earliest-commit loop - - for i, rr := range ret { - - // The oldest commit needs us to page through the commit history to find it - - oldestCommit, err := this.oldestCommit(ctx, rr.Name, "") - if err != nil { - log.Printf("finding oldest commit for '%s': %s", rr.Name, err) - continue - } - - ret[i].CreatedAt = oldestCommit.Commit.Author.Date - } - return ret, nil } diff --git a/main.go b/main.go index 255efe5..6f93d8c 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,13 @@ package main import ( + "context" "flag" "log" "net/http" "regexp" "strings" + "sync" "github.com/BurntSushi/toml" "golang.org/x/sync/semaphore" @@ -30,6 +32,10 @@ type Application struct { rxRepoPage, rxRepoImage *regexp.Regexp apiSem *semaphore.Weighted + + reposMut sync.RWMutex + reposCache []Repo // Sorted by recently-created-first + reposAlphabeticalOrder map[string]int } func main() { @@ -58,6 +64,9 @@ func main() { app.apiSem = semaphore.NewWeighted(app.cfg.Gitea.MaxConnections) } + // Sync worker + go app.syncWorker(context.Background()) + log.Printf("Starting web server on [%s]...", app.cfg.BindTo) log.Fatal(http.ListenAndServe(app.cfg.BindTo, &app)) } diff --git a/pages.go b/pages.go index 6df2931..bf34cb7 100644 --- a/pages.go +++ b/pages.go @@ -7,7 +7,6 @@ import ( "log" "net/http" "net/url" - "sort" "strings" ) @@ -52,36 +51,27 @@ func (this *Application) internalError(w http.ResponseWriter, r *http.Request, e http.Error(w, "An internal error occurred.", 500) } -func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() +func (this *Application) Delay(w http.ResponseWriter, r *http.Request) { + this.Templatepage(w, r, "Loading...", "", func() { + fmt.Fprintf(w, ` +

Loading, please wait...

+ + + `) + }) +} - repos, err := this.repos(ctx) - if err != nil { - this.internalError(w, r, fmt.Errorf("listing repos: %w", err)) +func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) { + + this.reposMut.RLock() + defer this.reposMut.RUnlock() + + if len(this.reposCache) == 0 { + // We haven't loaded the repositories from Gitea yet + this.Delay(w, r) return } - topics := make(map[string][]string) - for _, repo := range repos { - if t, err := this.topicsForRepo(ctx, repo.Name); err == nil { - topics[repo.Name] = t - } - } - - // Sort repos once alphabetically, to get alphabetical indexes... - sort.Slice(repos, func(i, j int) bool { - return repos[i].Name < repos[j].Name - }) - alphabeticalOrderIndexes := make(map[string]int, len(repos)) - for idx, repo := range repos { - alphabeticalOrderIndexes[repo.Name] = idx - } - - // But then make sure the final sort is by most-recently-created - sort.Slice(repos, func(i, j int) bool { - return repos[i].CreatedAt.After(repos[j].CreatedAt) - }) - // Ready for template this.Templatepage(w, r, "", "", func() { @@ -94,10 +84,10 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) { -

Projects (`+fmt.Sprintf("%d", len(repos))+`)

+

Projects (`+fmt.Sprintf("%d", len(this.reposCache))+`)

`) - for _, repo := range repos { + for _, repo := range this.reposCache { pageHref := html.EscapeString(`/` + url.PathEscape(repo.Name)) normalisedDesc := repo.Description @@ -116,17 +106,17 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) { } rowClass := "" - for _, topic := range topics[repo.Name] { + for _, topic := range repo.topics { rowClass += `taggedWith-` + topic + ` ` } fmt.Fprint(w, `
@@ -137,7 +127,7 @@ func (this *Application) Homepage(w http.ResponseWriter, r *http.Request) {
`) - for _, topic := range topics[repo.Name] { + for _, topic := range repo.topics { fmt.Fprint(w, ``+html.EscapeString(topic)+` `) } fmt.Fprint(w, ` diff --git a/sync.go b/sync.go new file mode 100644 index 0000000..c9cfc2a --- /dev/null +++ b/sync.go @@ -0,0 +1,110 @@ +package main + +import ( + "context" + "log" + "sort" + "time" +) + +func (this *Application) sync(ctx context.Context) (bool, error) { + + // List repositories on Gitea + repos, err := this.repos(ctx) + if err != nil { + return false, err + } + + // Compare this list of repositories to our existing one + // If the repository is new, or if it's update-time has changed since we last + // saw it, then re-refresh its real git commit timestamps + // Otherwise copy them from the previous version + + this.reposMut.RLock() // readonly + + anyChanges := false + if len(repos) != len(this.reposCache) { + anyChanges = true + } + + for i, rr := range repos { + if idx, ok := this.reposAlphabeticalOrder[rr.Name]; ok && this.reposCache[idx].GiteaUpdated == rr.GiteaUpdated { + // Already exists in cache with same Gitea update time + // Copy timestamps + repos[i] = this.reposCache[idx] + + } else { + + // New repo, or Gitea has updated timestamp + anyChanges = true + + // Refresh timestamps + this.populateCommitInfo(ctx, &rr) + + // Refresh topics + if t, err := this.topicsForRepo(ctx, rr.Name); err == nil { + rr.topics = t + } + + // Save + repos[i] = rr + } + } + + this.reposMut.RUnlock() + + // + if !anyChanges { + return false, nil // nothing to do + } + + // We have a final updated repos array + + // Sort repos once alphabetically, to get alphabetical indexes... + sort.Slice(repos, func(i, j int) bool { + return repos[i].Name < repos[j].Name + }) + alphabeticalOrderIndexes := make(map[string]int, len(repos)) + for idx, repo := range repos { + alphabeticalOrderIndexes[repo.Name] = idx + } + + // But then make sure the final sort is by most-recently-created + sort.Slice(repos, func(i, j int) bool { + return repos[i].oldestCommit.After(repos[j].oldestCommit) + }) + + // Commit our changes for the other threads to look at + this.reposMut.Lock() + this.reposCache = repos + this.reposAlphabeticalOrder = alphabeticalOrderIndexes + this.reposMut.Unlock() + + // Done + return true, nil +} + +func (this *Application) syncWorker(ctx context.Context) { + + t := time.NewTicker(30 * time.Minute) + defer t.Stop() + + for { + anyChanges, err := this.sync(ctx) + if err != nil { + // log and continue + log.Printf("Refreshing repositories: %s", err.Error()) + } + if anyChanges { + log.Printf("Repositories updated") + } + + select { + case <-t.C: + continue + + case <-ctx.Done(): + return + } + } +}