doc/README: add v157 changelog

.gitignore

@@ -0,0 +1,4 @@
+_dist/
+sites/*/data/
+sites/*/wwwroot/
+shields_cache/

.hgignore

@@ -1,6 +0,0 @@
-mode:regex
-^_dist/
-^sites/[^/]+/data/
-^sites/[^/]+/wwwroot/
-
-^shields_cache/

.hgtags

@@ -1,4 +0,0 @@
-42a17645b5b21d7fe395767de7fa3e26ee999014 release-r54
-0f89ae041c2ee60cc1ea308d047fce816b19c490 release-r64
-d4733a95c3428db8722ce0d0350d17bbbabc8720 release-r72
-7c92f9e2e4818d74eded59ad516d7d58b4072f8d release-r97

README.md

@@ -0,0 +1,102 @@
+# codesite
+
+A static site generator for a portfolio website.
+
+This script was used to generate the https://code.ivysaur.me/ website between 2013 and 2020, when it was replaced by a [teafolio](https://code.ivysaur.me/teafolio) instance.
+
+Written in PHP, Bash
+
+## Features
+
+- Generates static website, minimising server load and decreasing attack surface compared to dynamic server-side PHP
+- Automatic thumbnailing and sprite sheet generation
+- Download file attachments per-project
+- One-click rebuild, one-click deploy
+- Parallel generation
+
+## Changelog
+
+2020-05-24: v157
+- Sunset release
+- Rewrite `codesite` Git repository history for full public release
+- Feature: Add support for autogenerating codesite projects from a Git repository (`update_git_remote.sh`)
+- Feature: Support converting a codesite project into a readonly, archive Git repository (`codesite2git`)
+- Feature: Support bulk pushing `codesite2git`-converted repositories to a Gitea instance
+
+2018-12-30: v138
+- (Private release only)
+- Feature: Support `[git-repository]` tags to link to a custom VCS repository
+- Feature: Support markdown link syntax
+- Feature: Support markdown multiline comments that include code syntax highlight tag
+- Enhancement: Support Go subpackages under `[go-get]` tags
+- Enhancement: Strip markdown image urls
+- Enhancement: Trim leading blank lines
+- Fix a cosmetic issue with favicons and document rendering mode in some versions of IE
+
+2017-10-28: v132
+- Fix an issue with not applying project rename redirections in the new URL format
+- Fix a cosmetic issue with classifying downloads named with numeric strings
+- Fix a cosmetic issue with truncated sheilds caused by ID collisions
+- [⬇️ code_v132.tar.xz](.dist-archive/code_v132.tar.xz) *(106.49 KiB)*
+
+2017-04-23: v126
+- Breaking: BASEURL is now a mandatory field
+- Feature: Support `[go-get]` tags
+- Feature: Canonical paths for SEO
+- Feature: Use extension-less paths by default (old .html paths still supported via redirect)
+- Feature: Support `[url]` tags
+- Enhancement: Cache-busting parameters on homepage thumbnails
+- [⬇️ code_v126.tar.xz](.dist-archive/code_v126.tar.xz) *(106.39 KiB)*
+
+2016-04-19: v118
+- Feature: Classify download artefacts by matching CHANGELOG entry
+- Feature: Allow sorting by lifespan, artefacts, release entries
+- Feature: Separate site generation code from site data repositories
+- Enhancement: Update supplied CSS normalize script
+- Enhancement: Read parallelism from number of CPUs
+- [⬇️ code_v118.tar.xz](.dist-archive/code_v118.tar.xz) *(103.18 KiB)*
+
+2015-11-08: v97
+- Feature: Support BBCode b/i/spoiler/entry tags
+- Feature: Add more CSS styles to allow per-site customistaion
+- Feature: Use http://shields.io images where appropriate
+- Feature: Optional blurbs=off, article_header={string}, shields_prefix=true configuration directives
+- Fix an issue with project update-time detection
+- Fix a cosmetic issue with whitespace
+- [⬇️ code_v97.tar.xz](.dist-archive/code_v97.tar.xz) *(107.54 KiB)*
+
+2015-04-05: v72
+- Feature: Support redirecting old project names
+- Feature: Add file hash in download URLs to prevent filename collisions
+- Fix an issue generating spritesheets even if no project images are present
+- Don't include `ctime` when estimating project update time
+- [⬇️ code_v72.tar.xz](.dist-archive/code_v72.tar.xz) *(106.79 KiB)*
+
+2015-04-05: v64
+- Feature: Support sorting projects
+- [⬇️ code_v64.tar.xz](.dist-archive/code_v64.tar.xz) *(106.45 KiB)*
+
+2015-04-04: v54
+- Feature: Support multiple code sites
+- Fix an issue with parallel builds on some versions of windows
+- Fix an issue corrupting URL links with multiple parameters
+- Fix a cosmetic issue with page overflow
+- Fix a cosmetic issue with whitespace on code elements
+- [⬇️ code_v54.tar.xz](.dist-archive/code_v54.tar.xz) *(105.79 KiB)*
+
+2014-07-02: v39
+- Feature: Tags
+- Feature: Generate pages in parallel
+- Enhancement: Support raw HTML sections in page content
+- Fix an issue with URLs containing spaces
+- Fix a cosmetic issue with image thumbnail backgrounds
+- Fix a cosmetic issue with download sort order
+- Fix a cosmetic issue with page layout
+- [⬇️ code_v39.zip](.dist-archive/code_v39.zip) *(13.11 KiB)*
+
+2013-09-28: v13
+- Initial public source code release
+- [⬇️ code_v13.zip](.dist-archive/code_v13.zip) *(9.02 KiB)*
+
+2013-09-21: v3
+- Initial deployment

_dist/README.txt

@@ -1,61 +0,0 @@
-A static site generator for a portfolio website.
-
-This script is currently in use to generate the code.ivysaur.me website.
-
-Written in PHP, Bash
-
-=FEATURES=
-
-- Generates static website, minimising server load and decreasing attack surface compared to dynamic server-side PHP
-- Automatic thumbnailing and sprite sheet generation
-- Download file attachments per-project
-- One-click rebuild, one-click deploy
-- Parallel generation
-
-=CHANGELOG=
-
-2016-04-19: v118
-- Feature: Classify download artefacts by matching CHANGELOG entry
-- Feature: Allow sorting by lifespan, artefacts, release entries
-- Feature: Separate site generation code from site data repositories
-- Enhancement: Update supplied CSS normalize script
-- Enhancement: Read parallelism from number of CPUs
-
-2015-11-08: v97
-- Feature: Support BBCode b/i/spoiler/entry tags
-- Feature: Add more CSS styles to allow per-site customistaion
-- Feature: Use http://shields.io images where appropriate
-- Feature: Optional blurbs=off, article_header={string}, shields_prefix=true configuration directives
-- Fix an issue with project update-time detection
-- Fix a cosmetic issue with whitespace
-
-2015-04-05: v72
-- Feature: Support redirecting old project names
-- Feature: Add file hash in download URLs to prevent filename collisions
-- Fix an issue generating spritesheets even if no project images are present
-- Don't include `ctime` when estimating project update time
-
-2015-04-05: v64
-- Feature: Support sorting projects
-
-2015-04-04: v54
-- Feature: Support multiple code sites
-- Fix an issue with parallel builds on some versions of windows
-- Fix an issue corrupting URL links with multiple parameters
-- Fix a cosmetic issue with page overflow
-- Fix a cosmetic issue with whitespace on code elements
-
-2014-07-02: v39
-- Feature: Tags
-- Feature: Generate pages in parallel
-- Enhancement: Support raw HTML sections in page content
-- Fix an issue with URLs containing spaces
-- Fix a cosmetic issue with image thumbnail backgrounds
-- Fix a cosmetic issue with download sort order
-- Fix a cosmetic issue with page layout
-
-2013-09-28: v13
-- Initial public source code release
-
-2013-09-21: v3
-- Initial deployment

build-worker

@@ -0,0 +1,27 @@
+#!/usr/bin/php
+<?php
+
+require __DIR__ . '/lib/bootstrap.php';
+
+function main($args) {
+	$config  = setup_vars();
+	
+	$total   = $args[0];
+	$pos     = $args[1];
+	
+	// Perform build tasks
+	
+	if ($pos == 0) {
+		buildcommon();
+		if (array_key_exists('redirect', $config)) {
+			buildredirects( $config['redirect'] );
+		}
+		if (array_key_exists('golang-subpackages', $config)) {
+			buildgosubpackages( $config['golang-subpackages'] );
+		}
+	} else {
+		buildprojects($pos, array_decimate(listprojects(), $total, $pos));
+	}
+}
+
+main(array_slice($_SERVER['argv'], 1));

codesite2git

@@ -0,0 +1,334 @@
+#!/usr/bin/php
+<?php
+
+require __DIR__.'/lib/bootstrap.php';
+
+class CProjectGitExporter extends CProject {
+
+    function __construct(string $projdirname, string $projname) {
+        CProject::__construct($projdirname, $projname);
+
+        $this->allowText2Html = false; // Don't produce HTML when processing the download block
+    }
+
+    function ExportAsGit(string $dest) {
+
+        // Create destination directory
+        if (is_dir($dest)) {
+            throw new Exception("Destination path '{$dest}' already exists");
+        }
+        mkdir($dest);
+        shell_exec('cd '. escapeshellarg($dest).' && git init');
+            
+        // Parse download information blocks
+        $this->filterLongDescArea();
+
+        // Prepare to start modifying the generated README.md file
+        $this->longdesc = preg_replace('~\\[url=([^\\]]+?)\\](.+?)\\[/url\\]~m', '[\\2](\\1)', $this->longdesc);
+        $this->longdesc = str_replace("\xEF\xBB\xBF", "", $this->longdesc); // Remove interior UTF-8 BOM markers
+        $this->longdesc = str_replace("\r", "", $this->longdesc); // dos2unix
+
+        // Add main header and "written in" badge
+        $lines = explode("\n", $this->longdesc);
+        $header = ["# ".$this->projname, ""];
+        if (strlen($this->subtag) > 0) {
+            $header[] = "![](https://img.shields.io/badge/written%20in-" . rawurlencode($this->subtag) . "-blue)";
+            $header[] = "";
+        }
+        $lines = array_merge($header, $lines);
+        
+        // Modify some lines to standard markdown format:
+        foreach($lines as $i => $line) {
+
+            // Convert ==HEADERS== to ## Headers
+            if (strlen($line) > 0 && $line[0] == '=') {
+                $tmp = rtrim($line, '=');
+                $indent = substr_count($tmp, '=');
+
+                // Normalise the capitalisation for header text
+                $header_text = substr($tmp, $indent);
+                if ($header_text != "TODO" && $header_text != "FIXME" && $header_text != "WARNING" && $header_text != "API") {
+                    $header_text = ucwords(strtolower($header_text));
+                }
+
+                // raise markdown-equivalent header level by 1 to compensate for new header
+                $lines[$i] = $line = str_repeat('#', $indent + 1).' '.$header_text;
+            }
+
+            // Convert multiline single-backtick to multi-backtick
+            // Heuristic: if there is only a single backtick in the line
+            if (substr_count($line, '`') == 1) {
+                $lines[$i] = $line = str_replace('`', "\n```\n", $line);
+            }
+
+            // If the previous line started with a hyphen, and this line didn't, we need an 
+            // extra line separator
+            if ($i > 0 && strlen($line) > 0 &&
+                $line[0] != '-' && strlen($lines[$i-1]) > 0 && $lines[$i-1][0] == '-'
+            ) {
+                $lines[$i] = $line = "\n".$line;
+            }
+        }
+        
+        // Copy all images to a doc/ subdirectory
+        if (count($this->images) > 0) {
+            mkdir($dest.'/doc');
+            foreach($this->images as $img) {
+                // Normalise some legacy file extension variants while we're here
+                copy($this->dir.$img, $dest.'/doc/'.str_replace(['.jpeg', '.JPG', '.JPEG'], ['.jpg', '.jpg', '.jpg'], $img));
+            }
+        }
+
+        // Copy all downloads to a dist-archive/ subdirectory
+        if (count($this->downloads) > 0) {
+
+            mkdir($dest.'/dist-archive');
+            foreach($this->downloads as $file) {
+                copy($this->dir.$file, $dest.'/dist-archive/'.$file);
+            }
+
+            if (! $this->downloads_section_was_replaced) {
+                // Add our own downloads section
+                $lines[] = $this->renderDownloadsBlock($this->downloads, true);
+            }
+        }
+
+        // Save final README
+        file_put_contents($dest.'/README.md', implode("\n", $lines));
+
+        // Extra properties file
+        $ctime = $this->lastupdate - ($this->lifespan * 3600); // Lifespans are measured in hours
+
+        if (strlen($this->shortdesc) > 0) {
+            $this->shortdesc = rtrim($this->shortdesc, '.').'.';
+            $this->shortdesc[0] = strtoupper($this->shortdesc[0]);
+        }
+
+        file_put_contents($dest.'/.legacy-codesite.toml', "# Converted with codesite2git
+
+project_name=".json_encode($this->projname)."
+short_description=".json_encode($this->shortdesc)."
+written_in_lang=".json_encode($this->subtag)."
+topics=".json_encode($this->tags)."
+ctime=".$ctime."
+mtime=".$this->lastupdate."
+
+");
+
+        // Git commit everything
+        // Once for the meta with ctime; once for all files with the mtime
+        $command = (
+            'cd '. escapeshellarg($dest).' && '.
+            'git add .legacy-codesite.toml ; '.
+            'GIT_COMMITTER_DATE="'.date(DATE_ISO8601, $ctime).'" git commit -m "initial meta commit" --date '.escapeshellarg(date(DATE_ISO8601, $ctime)).' ; '.
+            'git add -A ; '.
+            'GIT_COMMITTER_DATE="'.date(DATE_ISO8601, $this->lastupdate).'" git commit -m "commit all archived files" --date '.escapeshellarg(date(DATE_ISO8601, $this->lastupdate))
+        );
+        //echo $command."\n";
+        shell_exec($command);
+
+    }
+
+    public function renderDownloadsBlock($render_downloads, $include_header=false) { // override
+		if (! count($render_downloads)) {
+			return;
+		}
+		
+		$ret = "\n";
+        if ($include_header) {
+            $ret .= "## Download\n\n";
+        }
+        foreach($render_downloads as $filename) {
+            // It's possible to make a general URL that links directly to the raw download endpoint
+            // But because our markdown is rendered in context of the repo, we can't properly make a relative link
+            // We could make an absolute link, but that's probably a bad idea w.r.t. any future Git migrations
+            // Just link to the file in-repo although it won't be a RAW download link
+
+            $ret .= "- [⬇️ {$filename}](dist-archive/".rawurlencode($filename).") "; // raw/branch/master/dist-archive/{$filename}
+            $ret .= "*(".fbytes(filesize($this->dir.$filename)).")*\n";
+        }
+        
+		return $ret;
+	}
+
+}
+
+class Gitea {
+    protected $url;
+    protected $org;
+    protected $token;
+
+    public function __construct(string $url, string $org, string $token) {
+        $this->url = $url;
+        $this->org = $org;
+        $this->token = $token;
+    }
+
+    public function repoExists(string $repo): bool {
+
+        $resp = @file_get_contents($this->url.'api/v1/repos/'.$this->org.'/'.$repo);
+        if ($resp === false) {
+            return false; // not conclusive but it will suffice
+        }
+        if (strpos($resp, 'does not exist')) {
+            return false;
+        }
+
+        $inf = json_decode($resp, true);
+        if (is_array($inf) && $inf['name'] == $repo) {
+            return true; // definitely yes
+        }
+
+        throw new \Exception("Couldn't decide whether repo {$repo} exists\n\n".var_export($resp, true));
+    }
+
+    public function createRepo(string $repo) {
+        $command = ('curl -sS -X POST ' .
+            escapeshellarg($this->url.'api/v1/admin/users/'.$this->org.'/repos?access_token='.urlencode($this->token)) . ' '.
+            '-H "accept: application/json" -H "Content-Type: application/json" -d ' . escapeshellarg(json_encode(['name' => $repo]))
+        );
+
+        $code = 0;
+        $output = [];
+        exec($command, $output, $code);
+        if ($code == 0) {
+            return; // success
+        }
+
+        throw new \Exception("failed to create repository {$repo} (exit code {$code})\n\n".var_export($output, true));
+    }
+
+    public function updateRepoProperties(string $repo, string $description) {
+        $json_update = [
+            'archived' => true,
+            'description' => $description,
+
+            // SPECIAL https://code.ivysaur.me/ - don't leave in commit {{{
+            'has_wiki' => false,
+            'website' => 'https://code.ivysaur.me/'.$repo,
+            // }}}
+        ];
+
+        $command = ('curl -sS -X PATCH ' .
+            escapeshellarg($this->url.'api/v1/repos/'.$this->org.'/'.$repo.'?access_token='.urlencode($this->token)) . ' '.
+            '-H "accept: application/json" -H "Content-Type: application/json" -d ' . escapeshellarg(json_encode($json_update))
+        );
+
+        $code = 0;
+        $output = [];
+        exec($command, $output, $code);
+        if ($code == 0) {
+            return; // success
+        }
+
+        throw new \Exception("failed to update repository {$repo} properties: (exit code {$code})\n\n".var_export($output, true));
+    }
+
+    public function setRepoTopics(string $repo, array $topics) {
+        $command = ('curl -sS -X PUT ' .
+            escapeshellarg($this->url.'api/v1/repos/'.$this->org.'/'.$repo.'/topics?access_token='.urlencode($this->token)) . ' '.
+            '-H "accept: application/json" -H "Content-Type: application/json" -d ' . escapeshellarg(json_encode(['topics' => $topics]))
+        );
+
+        $code = 0;
+        $output = [];
+        exec($command, $output, $code);
+        if ($code == 0) {
+            return; // success
+        }
+
+        throw new \Exception("failed to update repository {$repo} topics: (exit code {$code})\n\n".var_export($output, true));
+    }
+
+    public function gitRemoteUrlFor(string $repo): string {
+        return $this->url.$this->org.'/'.$repo.'.git';
+    }
+}
+
+function codesite2git(string $projdirname, string $projname, string $dest): CProjectGitExporter {
+
+    // Parse existing project
+    $c = new CProjectGitExporter($projdirname, $projname);
+    // var_dump($c);
+    $c->ExportAsGit($dest);
+
+    return $c;
+}
+
+function usage() {
+    die("Usage:
+    codesite2git [FLAGS...]
+
+Options:
+    --single CODESITE_ROOT_PATH PROJECT_DIR_NAME PROJECT_REAL_NAME DEST_DIR
+    --all    CODESITE_ROOT_PATH TEMP_DIR GITEA_URL GITEA_ORG GITEA_AUTH_TOKEN
+");
+    die(1);
+}
+
+function main(array $argv) {
+    if (count($argv) < 2) {
+        usage();
+    }
+
+    if ($argv[1] == '--single') {
+        if (count($argv) != 6) {
+            usage();
+        }
+
+        $config = setup_vars($argv[2]);
+        codesite2git($argv[3], $argv[4], $argv[5]);
+
+    } else if ($argv[1] == "--all") {
+        if (count($argv) != 7) {
+            usage();
+        }
+
+        $config = setup_vars($argv[2]);
+        $repos = glob(BASEDIR.'data/*');
+        
+        $temp_dir = $argv[3];
+        $gitea = new Gitea($argv[4], $argv[5], $argv[6]);
+
+        foreach($repos as $i => $path) {
+
+            $projdir = basename($path);
+            $reponame = explode('-', $projdir, 2)[1];
+
+            echo sprintf("[%3d/%3d] Processing '%s'... ", $i+1, count($repos), $reponame);
+
+            // Convert to git
+            $git_repo_dir = $temp_dir.'/'.$projdir.'-archive.git';
+            $c = codesite2git($projdir, $reponame, $git_repo_dir);
+
+            // Check if Gitea repo already exists
+            if ($gitea->repoExists($reponame)) {
+                throw new \Exception("Abandoning process since repo {$reponame} already exists in Gitea - move it out of the source directory first");
+            }
+
+            // Create new Gitea repo
+            $gitea->createRepo($reponame);
+            if (! $gitea->repoExists($reponame)) {
+                throw new \Exception("Created repo {$reponame} successfully but it doesn't seem to exist?");
+            }
+
+            // Add git remote and push
+            shell_exec(
+                'cd '.escapeshellarg($git_repo_dir) .' && '.
+                'git remote add origin '.escapeshellarg($gitea->gitRemoteUrlFor($reponame)).' && '.
+                'git push origin master'
+            );
+
+            // Set Gitea topics + description
+            $gitea->updateRepoProperties($reponame, $c->shortdesc);
+            $gitea->setRepoTopics($reponame, $c->tags);
+
+            echo " Done\n";
+        }
+
+    } else {
+        usage();
+    }
+}
+
+main($_SERVER['argv']);

lib/CProject.php

@@ -2,19 +2,26 @@
 
 class CProject {
 	
-	private $dir;
+	protected $dir;
 	
+	/**
+	 * @var string $projname
+	 */
 	public $projname;
+
+	/**
+	 * @var string $shortdesc
+	 */
 	public $shortdesc = '(no description)';
 	public $subtag = '';
 	public $lastupdate = 0;
 	public $numreleases = 0;
 	
-	private $longdesc = '';
-	private $prefix_html = '';
-	private $images = array();
-	private $downloads = array();
-	private $downloads_hashes = array();
+	protected $longdesc = '';
+	protected $prefix_html = '';
+	protected $images = array();
+	protected $downloads = array();
+	protected $downloads_hashes = array();
 	
 	public $downloads_section_was_replaced = false;
 	
@@ -24,6 +31,11 @@ class CProject {
 	
 	public $homeimage = null;
 	
+	protected $go_get_target = '';
+	protected $git_repo = '';
+
+	protected $allowText2Html = true;
+	
 	public function __construct($dirname, $projname) {
 		$this->dir = BASEDIR.'data/'.$dirname.'/';
 		$this->projname = $projname;
@@ -71,33 +83,59 @@ class CProject {
 					$this->subtag = rtrim($matches[1], ' .');
 				}
 				
+				// Find `go-get` tags
+				$this->longdesc = preg_replace_callback('~\[go-get\](.+)\[/go-get\]~', function($matches) {
+					$this->go_get_target = $matches[1];
+					return '';
+				}, $this->longdesc);
+				
 				// Find tags
 				if (preg_match('~Tags: ([^\\n]+)~', $this->longdesc, $matches)) {
 					$this->tags = array_map('trim', explode(',', $matches[1]));
 				}
+
+				// Find `entry` tags
+				$this->longdesc = preg_replace(
+					'~\\[entry=([^\\]]+?)\\](.+?)\\[/entry\\]~m',
+					'[url='.BASEURL.'\\1/]\\2[/url]', // hesc still hasn't happened, transform bbcode->bbcode
+					$this->longdesc
+				);
 				
-				// Extract short description
-				$parts = explode("\n", $this->longdesc);
-				$this->shortdesc = array_shift($parts);
-				$this->shortdesc[0] = strtolower($this->shortdesc[0]); // cosmetic lowercase
+				// Strip out any markdown image links
+				//   [![](doc/image1.thumb.png)](doc/image1.png)
+				$this->longdesc = preg_replace('~\\[!.+?\\)\\]\\(.+?\\)~m', '', $this->longdesc);
 				
-				// Filters for longdesc	
-				
-				$prefix_html = '';
-				$this->longdesc = preg_replace_callback('~\nWritten in ([^\\n]+)~ms', function($matches) use (&$prefix_html) {
-					$prefix_html .= (
+				// Find "Written in" tags				
+				$this->prefix_html = '';
+				$this->longdesc = preg_replace_callback('~\nWritten in ([^\\n]+)~ms', function($matches) {
+					$this->prefix_html .= (
 						(SHIELDS_PREFIX ? mkshield('build', 'success', 'brightgreen').' ' : '').
 						mkshield('written in', rtrim($matches[1], '.'), 'blue')
 					);
 					return '';
 				}, $this->longdesc);
-				
+
+				// Find 'git-repository' tags
+				$this->longdesc = preg_replace_callback('~\[git\](.+)\[/git\]~', function($matches) {
+					$this->git_repo = $matches[1];
+					if (strlen($this->prefix_html) > 0) {
+						$this->prefix_html .= '&nbsp';
+					}
+					$this->prefix_html .= '<a href="'.hesc($this->git_repo).'">'.mkshield('vcs', 'git', 'yellowgreen', ['logo' => 'git']).'</a>';
+					return '';
+				}, $this->longdesc);
+
+				// Collapse multiple blank lines
+				$this->longdesc = ltrim($this->longdesc, "\n");
 				while(strpos($this->longdesc, "\n\n\n") !== false) {
 					$this->longdesc = str_replace("\n\n\n", "\n\n", $this->longdesc);
 				}
 				$this->longdesc = rtrim($this->longdesc, "\n")."\n";
-
-				$this->prefix_html = $prefix_html;
+				
+				// Extract short description (last)
+				$parts = explode("\n", $this->longdesc);
+				$this->shortdesc = array_shift($parts);
+				$this->shortdesc[0] = strtolower($this->shortdesc[0]); // cosmetic lowercase
 				
 				continue;
 			}
@@ -181,7 +219,7 @@ class CProject {
 			}
 			foreach($this->downloads as $idx => $filename) {
 				foreach(array_keys($known_tags) as $tagname) {
-					if (stripos($filename, $tagname) !== false) {
+					if (stripos($filename, (string)$tagname) !== false) {
 						$found_idx[$idx] = $tagname;
 						
 						$render_per_tag[$tagname][$idx] = $filename;
@@ -206,7 +244,9 @@ class CProject {
 			
 			$this->longdesc = substr($this->longdesc, 0, strlen($this->longdesc)-1); // Strip the extra NL we added
 			
-			$this->longdesc = text2html($this->longdesc);
+			if ($this->allowText2Html) {
+				$this->longdesc = text2html($this->longdesc);
+			}
 			foreach($known_tags as $tag_name => $tag_idx) {
 				$this->longdesc = str_replace(
 					'${{TAG_'.$tag_idx.'}}',
@@ -215,7 +255,9 @@ class CProject {
 				);
 			}
 			
-			$this->longdesc = str_replace("</ul>\n<br />", "</ul>", $this->longdesc);
+			if ($this->allowText2Html) {
+				$this->longdesc = str_replace("</ul>\n<br />", "</ul>", $this->longdesc);
+			}
 			
 			// Skip displaying the global downloads area
 			// This flag also indicates that the content has been pre-HTMLified
@@ -269,8 +311,20 @@ class CProject {
 		
 		ob_start();
 		$this->index();
-		$idxfile = template($this->projname.' | '.SITE_TITLE, ob_get_clean());
-		file_put_contents(BASEDIR.'wwwroot/'.$this->projname.'.html', $idxfile);
+		
+		$extra_head_items = [];
+		
+		$extra_head_items[] = '<link rel="canonical" href="'.hesc(BASEURL.$this->projname).'">'; // TODO include golang `go get` meta if necessary
+		
+		if (strlen($this->go_get_target) > 0) {
+			$extra_head_items[] = '<meta name="go-import" content="'.hesc($this->go_get_target).'">'; // TODO include golang `go get` meta if necessary
+		}
+		
+		$idxfile = template($this->projname.' | '.SITE_TITLE, ob_get_clean(), implode("\n", $extra_head_items));
+		mkdir(BASEDIR.'wwwroot/'.$this->projname);
+		
+		file_put_contents(BASEDIR.'wwwroot/'.$this->projname.'/index.html', $idxfile); // new URL format
+		file_put_contents(BASEDIR.'wwwroot/'.$this->projname.'.html', redirecthtml(BASEURL.$this->projname)); // old URL format
 	}
 	
 	public function getClassAttr() {
@@ -294,7 +348,7 @@ class CProject {
 	<ul class="<?=$include_header ? 'downloads-large' : 'downloads-small' ?>">
 <?php foreach($render_downloads as $idx => $filename) { ?> 
 		<li>
-			<a href="srv/<?=hesc($this->downloads_hashes[$idx])?>/<?=hesc(rawurlencode($filename))?>"><?=hesc($filename)?></a>
+			<a href="<?=BASEURL?>srv/<?=hesc($this->downloads_hashes[$idx])?>/<?=hesc(rawurlencode($filename))?>"><?=hesc($filename)?></a>
 			<small>
 				<?=hesc(fbytes(filesize(BASEDIR.'wwwroot/srv/'.$this->downloads_hashes[$idx].'/'.$filename)))?>
 			</small>
@@ -338,7 +392,7 @@ class CProject {
 <?php if (count($this->images)) { ?> 
 	<div class="projimg">
 	<?php foreach($this->images as $idx => $origname) { ?> 
-		<a href="img/<?=hesc(urlencode($this->projname))?>_<?=$idx?>.<?=str_ext($origname)?>"><img src="img/<?=hesc(urlencode($this->projname))?>_<?=$idx?>_thumb.jpg" class="thumbimage"></a>
+		<a href="<?=BASEURL?>img/<?=hesc(urlencode($this->projname))?>_<?=$idx?>.<?=str_ext($origname)?>"><img src="<?=BASEURL?>img/<?=hesc(urlencode($this->projname))?>_<?=$idx?>_thumb.jpg" class="thumbimage"></a>
 	<?php } ?> 
 	</div>
 	

lib/bootstrap.php

@@ -1,4 +1,3 @@
-#!/usr/bin/php
 <?php
 
 ini_set('display_errors', 'On');
@@ -11,13 +10,18 @@ require __DIR__.'/CProject.php';
 
 define('SHIELDS_CACHE_DIR', __DIR__.'/../shields_cache/');
 
-function main($args) {
-	$basedir = './';
-	$total   = $args[0];
-	$pos     = $args[1];
+/**
+ * Set up global defines for a given codesite project
+ * Should be called once we have the root path for a codesite project.
+ * 
+ */
+function setup_vars(string $basedir="./"): array {
 	
 	// Parse configuration
-	
+	if (! is_file($basedir.'config.ini')) {
+		die("[FATAL] Non-file '${basedir}config.ini'!\n");
+	}
+
 	$config = @parse_ini_file(
 		$basedir . 'config.ini',
 		true,
@@ -25,10 +29,11 @@ function main($args) {
 	);
 	
 	if ($config === false) {
-		die("[FATAL] Couldn't load '${basedir}/config.ini'!\n");
+		die("[FATAL] Couldn't load '${basedir}config.ini'!\n");
 	}
 	
 	define('BASEDIR',       $basedir);
+	define('BASEURL',       trim($config['codesite']['baseurl']));
 	define('SITE_TITLE',    trim($config['codesite']['title']));
 	define('PAGE_THUMB_W',  intval($config['codesite']['page_thumb_w']));
 	define('PAGE_THUMB_H',  intval($config['codesite']['page_thumb_h']));
@@ -38,16 +43,5 @@ function main($args) {
 	define('ARTICLE_HEADER', (isset($config['codesite']['article_header']) ? $config['codesite']['article_header'] : 'ABOUT') );	
 	define('SHIELDS_PREFIX', isset($config['codesite']['shields_prefix']));
 	
-	// Perform build tasks
-	
-	if ($pos == 0) {
-		buildcommon();
-		if (array_key_exists('redirect', $config)) {
-			buildredirects( $config['redirect'] );
-		}
-	} else {
-		buildprojects($pos, array_decimate(listprojects(), $total, $pos));
-	}
+	return $config;
 }
-
-main(array_slice($_SERVER['argv'], 1));

lib/template.php

@@ -1,18 +1,21 @@
 <?php
 
-function template($title, $content) {
+function template($title, $content, $extra_head='') {
 	ob_start();
 ?>
 <!DOCTYPE html>
 <html>
 	<head>
+		<title><?=hesc($title)?></title>
+		<meta http-equiv="X-UA-Compatible" content="IE=edge"> 
 		<meta charset="UTF-8">
 		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" >
 		<meta name="viewport" content="width=960" >
-		<link type="text/css" rel="stylesheet" href="static/normalize.css">
-		<link type="text/css" rel="stylesheet" href="static/style.css">
-		<script type="text/javascript" src="static/site.js"></script>
-		<title><?=hesc($title)?></title>
+		<?=$extra_head?> 
+		<link rel="icon" href="<?=BASEURL?>static/favicon.ico" type="image/x-icon">
+		<link type="text/css" rel="stylesheet" href="<?=BASEURL?>static/normalize.css">
+		<link type="text/css" rel="stylesheet" href="<?=BASEURL?>static/style.css">
+		<script type="text/javascript" src="<?=BASEURL?>static/site.js"></script>		
 	</head>
 	<body>
 		<div id="container">
@@ -108,6 +111,10 @@ function buildcommon() {
 		array_map('imagedestroy', $handles); // free
 	}
 	
+	// Cache-busting stylesheet
+
+	$style = '.homeimage-sprite { background-image: url("logos.jpg?'.md5_file(BASEDIR.'wwwroot/logos.jpg').'"); }';
+
 	// Build index page
 
 	ob_start();
@@ -119,6 +126,10 @@ function buildcommon() {
 		<!-- }} --> 
 	<?php } ?> 
 		
+		<style type="text/css">
+	<?php echo $style; ?> 
+		</style>
+
 		<table id="projtable-main" class="projtable">
 	<?php foreach ($plist as $i => $pr) { ?> 
 			<tr class="<?=$pr->getClassAttr()?>"
@@ -130,13 +141,13 @@ function buildcommon() {
 				data-sort-ls="-<?=$pr->lifespan?>"
 			>
 				<td>
-					<a href="<?=hesc(urlencode($pr->projname))?>.html"><?=(is_null($handle_lookup[$pr->projname]) ? '<div class="no-image"></div>' : '<div class="homeimage homeimage-sprite" style="background-position:0 -'.($handle_lookup[$pr->projname]*INDEX_THUMB_H).'px"></div>')?></a>
+					<a href="<?=hesc(BASEURL.urlencode($pr->projname))?>/"><?=(is_null($handle_lookup[$pr->projname]) ? '<div class="no-image"></div>' : '<div class="homeimage homeimage-sprite" style="background-position:0 -'.($handle_lookup[$pr->projname]*INDEX_THUMB_H).'px"></div>')?></a>
 				</td>
 				<td>
 					<strong><?=hesc(str_replace('_', ' ', $pr->projname))?></strong><?php if (SHOW_BLURBS) { ?>,
 					<?=hesc($pr->shortdesc)?> 
 					<?php } ?> 
-					<a href="<?=hesc(urlencode($pr->projname))?>.html" class="article-read-more">more...</a>
+					<a href="<?=hesc(BASEURL.urlencode($pr->projname))?>/" class="article-read-more">more...</a>
 		<?php if (strlen($pr->subtag) || count($pr->tags)) { ?> 
 					<br>
 					<small>
@@ -155,20 +166,49 @@ function buildcommon() {
 		</table>
 <?php
 
-	$index = template(SITE_TITLE, ob_get_clean());
+	$extra_head = '<link rel="canonical" href="'.hesc(BASEURL).'">';
+
+	$index = template(SITE_TITLE, ob_get_clean(), $extra_head);
 	file_put_contents(BASEDIR.'wwwroot/index.html', $index);
 	
 	// Done
 }
 
+function redirecthtml($target) {
+	ob_start();
+?>
+<meta http-equiv="refresh" content="0; url=<?=hesc($target)?>">
+<a href="<?=hesc($target)?>">Moved &raquo;</a>
+<?php
+	return ob_get_clean();
+}
+
 function buildredirects($redirects) {
 	foreach($redirects as $oldname => $newname) {
-		ob_start();
-?>
-<meta http-equiv="refresh" content="0; url=<?=hesc($newname)?>.html">
-<a href="<?=hesc($newname)?>.html">Moved &raquo;</a>
-<?php
-		$page = ob_get_clean();
+	
+		$page = redirecthtml(BASEURL.$newname.'/');
+		
+		// old format
 		file_put_contents(BASEDIR.'wwwroot/'.$oldname.'.html', $page);
+		
+		// new format
+		mkdir(BASEDIR.'wwwroot/'.$oldname);
+		file_put_contents(BASEDIR.'wwwroot/'.$oldname.'/index.html', $page);
+	}
+}
+
+function buildgosubpackages($packages) {
+	foreach($packages as $path => $goGetStr) {
+	
+		$page = (
+			'<meta name="go-import" content="'.hesc($goGetStr).'">'.
+			"\n".
+			redirecthtml(BASEURL)
+		);
+		
+		// new directory format only
+		mkdir_all(BASEDIR.'wwwroot/'.$path);
+		file_put_contents(BASEDIR.'wwwroot/'.$path.'/index.html', $page);
+	
 	}
 }

lib/util.php

@@ -1,20 +1,29 @@
 <?php
 
-function mkshield($left_str, $right_str, $color_str) {
+function mkshield($left_str, $right_str, $color_str, $params=[]) {
 	$filename = sprintf(
 		"%s-%s-%s.svg", 
 		rawurlencode(str_replace('-', '--', $left_str)),
 		rawurlencode(str_replace('-', '--', $right_str)),
 		rawurlencode($color_str)
 	);
+	if (count($params) > 0) {
+		$filename .= '?' . http_build_query($params);
+	}
 	
-	$cache_path = SHIELDS_CACHE_DIR.$filename;
+	$cache_path = SHIELDS_CACHE_DIR.urlencode($filename);
 	
 	if (file_exists($cache_path)) {
 		return file_get_contents($cache_path);
 		
 	} else {
 		$retn = file_get_contents('https://img.shields.io/badge/'.$filename);
+		
+		// We need unique IDs
+		$prefix = substr(sha1($filename), 8).'-';
+		$retn = str_replace('id="',  'id="'.$prefix,  $retn);
+		$retn = str_replace('url(#', 'url(#'.$prefix, $retn);
+		
 		file_put_contents($cache_path, $retn);
 		return $retn;
 		
@@ -87,7 +96,7 @@ function fbytes($size, $suffixes='B|KiB|MiB|GiB|TiB') {
 		array_shift($sxlist);
 		$size /= 1024;
 	}
-	return number_format($size, 2).array_shift($sxlist);
+	return number_format($size, 2).' '.array_shift($sxlist);
 }
 
 function str_ext($sz) {
@@ -103,6 +112,11 @@ function hesc($sz) {
 	return @htmlentities($sz, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8');
 }
 
+function mkdir_all($path) {
+	$epath = escapeshellarg($path);
+	`mkdir -p ${epath}`;
+}
+
 function text2html($sz) {
 
 	$identity = function($sz) {
@@ -133,13 +147,20 @@ function text2html($sz) {
 		$base = hesc($sz);	
 		
 		$base = preg_replace('~^=+(.+)=+~m', '<strong>\\1</strong>', $base);
-		$base = preg_replace('~(https?://[^ \\r\\n\\t]+)~i', '<a href="\\1">\\1</a>', $base);
+		$base = preg_replace('~\\[url=([^\\]]+?)\\](.+?)\\[/url\\]~m', '<a href="\\1">\\2</a>', $base);
+
+		$base = preg_replace('~\\[([^\\]]+?)\\]\\((https?://.+?)\\)~m', '<a href="\\2">\\1</a>', $base); // Support markdown-style URLs
+
+		$base = preg_replace('~([^="])(https?://[^ \\r\\n\\t]+)~i', '\\1<a href="\\2">\\2</a>', $base); // Support standalone URLs
 		$base = preg_replace('~\\[b\\](.+?)\\[/b\\]~m', '<strong>\\1</strong>', $base);
 		$base = preg_replace('~\\[i\\](.+?)\\[/i\\]~m', '<i>\\1</i>', $base);
 		$base = preg_replace('~\\[spoiler\\](.+?)\\[/spoiler\\]~m', '<span class="spoiler">\\1</span>', $base);
-		$base = preg_replace('~\\[entry=([^\\]]+?)\\](.+?)\\[/entry\\]~m', '<a href="\\1.html">\\2</a>', $base);
 		$base = preg_replace('~\n- ~ms', "\n&bull; ", $base);
 		
+		$base = preg_replace('~^```.+$~m', '`', $base); // Convert ```html to single `{}` element
+
+		// TODO support markdown tables
+
 		$btparts = explode('`', $base);
 		if (count($btparts) > 1 && (count($btparts) % 2)) {
 			for ($i = 1, $e = count($btparts); $i < $e; $i += 2) {
@@ -147,7 +168,7 @@ function text2html($sz) {
 				if (strpos($btparts[$i], "\n") !== false) {
 					$class .= ' code-multiline';
 				}
-				$btparts[$i] = '<span class="'.$class.'">'.$btparts[$i].'</span>';
+				$btparts[$i] = '<span class="'.$class.'">'.ltrim($btparts[$i], "\n").'</span>';
 			}
 		}
 		

mkdist.sh

@@ -3,7 +3,12 @@
 set -eu
 
 tar cJvf "codesite-$(date +%s).tar.xz" \
-	rebuild.php rebuild.sh sites/codesite.example.com static_global \
+	--exclude='sites/codesite.example.com/wwwroot' \
+	rebuild.sh \
+	sites/codesite.example.com \
+	static \
+	lib \
 	--owner=0 --group=0
 
 read -p "Press any key to continue..."
+

rebuild.sh

@@ -9,44 +9,45 @@ numcpus() {
 }
 
 buildsite() {
+	local site="$1"
 
-	echo "Site: ${1}"
+	echo "Site: ${site}"
+	(
+		cd "$site"
 
-	pushd "$1" >/dev/null
+		echo "Cleaning wwwroot directory..."
 
-	echo "Cleaning wwwroot directory..."
-
-	if [[ -d wwwroot ]] ; then
-		rm -r wwwroot
-	fi	
-	mkdir -p wwwroot/{img,srv,static}
-	
-	echo "Copying static resources..."
-	
-	if [[ ! -d static ]] ; then
-		mkdir static
-	fi
-	cp "${APP_DIR}/static/"* wwwroot/static || true
-	cp static/* wwwroot/static || true
-	
-	for htm in footer header homepage_blurb ; do
-		if [[ ! -f "${htm}.htm" ]] ; then
-			touch "${htm}.htm"
-		fi
-	done
-
-	echo "Building pages..."
-
-	local threadcount=$(numcpus)
-	for i in $(seq 0 "$threadcount") ; do
-		"${APP_DIR}/lib/bootstrap.php" "$threadcount" "$i" &
-	done
-	wait
-
-	echo "Site: ${1} finished."
-	echo ""
+		if [[ -d wwwroot ]] ; then
+			rm -r wwwroot
+		fi	
+		mkdir -p wwwroot/{img,srv,static}
 		
-	popd >/dev/null
+		echo "Copying static resources..."
+		
+		if [[ ! -d static ]] ; then
+			mkdir static
+		fi
+		cp "${APP_DIR}/static/"* wwwroot/static || true
+		cp static/* wwwroot/static || true
+		
+		for htm in footer header homepage_blurb ; do
+			if [[ ! -f "${htm}.htm" ]] ; then
+				touch "${htm}.htm"
+			fi
+		done
+
+		echo "Building pages..."
+
+		local threadcount=$(numcpus)
+		for i in $(seq 0 "$threadcount") ; do
+			"${APP_DIR}/build-worker" "$threadcount" "$i" &
+		done
+		wait
+
+		echo "Site: ${site} finished."
+		echo ""
+	)
+
 }
 
 usage() {

update_git_remote.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# This script updates/rebuilds a codesite directory from a Git repository.
+#
+
+set -eu
+
+main() {
+
+    if [[ $# -ne 3 ]] ; then
+        echo "Usage: update_git_remote DATA_DIR GIT_REMOTE_URI EXTRA_CONTENT" >&2
+        exit 1
+    fi
+    local datadir="$1"
+    local gitremote="$2"
+    local extra="${3:-}"
+
+    #
+
+    echo "Updating ${datadir}..."
+
+    if [[ ! -d $datadir ]] ; then
+        mkdir $datadir
+    fi
+
+    local clone=$(mktemp -d)
+    (
+        cd "${clone}"
+        git clone "${gitremote}" .
+    )
+    cp "${clone}/README.md" "${datadir}/README.txt"
+
+    rm -fr "${clone}"
+
+    # Ensure LF endings
+    sed -i 's/\x0D$//' "${datadir}/README.txt"
+
+    # Remove all shields (codesite inserts its own)
+    sed -i /shields.io/d "${datadir}/README.txt"
+
+    # Remove top-level header (codesite inserts its own)
+    sed -i -re '/^#([^#])/d' "${datadir}/README.txt"
+
+    # Remove leading blank lines (since we removed the top-level header)
+    sed -i '/./,$!d' "${datadir}/README.txt"
+
+    # Convert headings from github-flavored markdown to codesite style (\U for Uppercase)
+    sed -i -re 's/^## (.+)/=\U\1=/' "${datadir}/README.txt"
+
+    # Add extra metadata so that codesite generator can link the repository
+    echo "" >> "${datadir}/README.txt"
+    echo "[git]${gitremote}[/git]" >> "${datadir}/README.txt"
+
+    # Add any extra per-repository metadata, on a line after the description
+    if [[ -n "${extra}" ]] ; then
+        sed -i 3i"${extra}" "${datadir}/README.txt"
+    fi
+}
+
+main "$@"