support markdown links; strip markdown image urls; trim leading blank lines; support markdown multiline comments that include highlight tag

.hgtags

@@ -2,3 +2,6 @@
 0f89ae041c2ee60cc1ea308d047fce816b19c490 release-r64
 d4733a95c3428db8722ce0d0350d17bbbabc8720 release-r72
 7c92f9e2e4818d74eded59ad516d7d58b4072f8d release-r97
+b1426986ff5f265f79d6412c0e81ccc7cae652ff release-r118
+2b11aaaad98718025d020c9fbaa451f65d413477 release-r126
+36cb5fdcbd0450a32ba9e88f7582a78b0b50db94 release-r132

_dist/README.txt

@@ -14,6 +14,19 @@ Written in PHP, Bash
 
 =CHANGELOG=
 
+2017-10-28: v132
+- Fix an issue with not applying project rename redirections in the new URL format
+- Fix a cosmetic issue with classifying downloads named with numeric strings
+- Fix a cosmetic issue with truncated sheilds caused by ID collisions
+
+2017-04-23: v126
+- Breaking: BASEURL is now a mandatory field
+- Feature: Support `[go-get]` tags
+- Feature: Canonical paths for SEO
+- Feature: Use extension-less paths by default (old .html paths still supported via redirect)
+- Feature: Support `[url]` tags
+- Enhancement: Cache-busting parameters on homepage thumbnails
+
 2016-04-19: v118
 - Feature: Classify download artefacts by matching CHANGELOG entry
 - Feature: Allow sorting by lifespan, artefacts, release entries

lib/CProject.php

@@ -24,6 +24,9 @@ class CProject {
 	
 	public $homeimage = null;
 	
+	protected $go_get_target = '';
+	protected $git_repo = '';
+	
 	public function __construct($dirname, $projname) {
 		$this->dir = BASEDIR.'data/'.$dirname.'/';
 		$this->projname = $projname;
@@ -71,33 +74,59 @@ class CProject {
 					$this->subtag = rtrim($matches[1], ' .');
 				}
 				
+				// Find `go-get` tags
+				$this->longdesc = preg_replace_callback('~\[go-get\](.+)\[/go-get\]~', function($matches) {
+					$this->go_get_target = $matches[1];
+					return '';
+				}, $this->longdesc);
+				
 				// Find tags
 				if (preg_match('~Tags: ([^\\n]+)~', $this->longdesc, $matches)) {
 					$this->tags = array_map('trim', explode(',', $matches[1]));
 				}
+
+				// Find `entry` tags
+				$this->longdesc = preg_replace(
+					'~\\[entry=([^\\]]+?)\\](.+?)\\[/entry\\]~m',
+					'[url='.BASEURL.'\\1/]\\2[/url]', // hesc still hasn't happened, transform bbcode->bbcode
+					$this->longdesc
+				);
 				
-				// Extract short description
-				$parts = explode("\n", $this->longdesc);
-				$this->shortdesc = array_shift($parts);
-				$this->shortdesc[0] = strtolower($this->shortdesc[0]); // cosmetic lowercase
+				// Strip out any markdown image links
+				//   [![](doc/image1.thumb.png)](doc/image1.png)
+				$this->longdesc = preg_replace('~\\[!.+?\\)\\]\\(.+?\\)~m', '', $this->longdesc);
 				
-				// Filters for longdesc	
-				
-				$prefix_html = '';
-				$this->longdesc = preg_replace_callback('~\nWritten in ([^\\n]+)~ms', function($matches) use (&$prefix_html) {
-					$prefix_html .= (
+				// Find "Written in" tags				
+				$this->prefix_html = '';
+				$this->longdesc = preg_replace_callback('~\nWritten in ([^\\n]+)~ms', function($matches) {
+					$this->prefix_html .= (
 						(SHIELDS_PREFIX ? mkshield('build', 'success', 'brightgreen').' ' : '').
 						mkshield('written in', rtrim($matches[1], '.'), 'blue')
 					);
 					return '';
 				}, $this->longdesc);
-				
+
+				// Find 'git-repository' tags
+				$this->longdesc = preg_replace_callback('~\[git\](.+)\[/git\]~', function($matches) {
+					$this->git_repo = $matches[1];
+					if (strlen($this->prefix_html) > 0) {
+						$this->prefix_html .= '&nbsp';
+					}
+					$this->prefix_html .= '<a href="'.hesc($this->git_repo).'">'.mkshield('vcs', 'git', 'yellowgreen', ['logo' => 'git']).'</a>';
+					return '';
+				}, $this->longdesc);
+
+				// Collapse multiple blank lines
+				$this->longdesc = ltrim($this->longdesc, "\n");
 				while(strpos($this->longdesc, "\n\n\n") !== false) {
 					$this->longdesc = str_replace("\n\n\n", "\n\n", $this->longdesc);
 				}
 				$this->longdesc = rtrim($this->longdesc, "\n")."\n";
-
-				$this->prefix_html = $prefix_html;
+				
+				// Extract short description (last)
+				$parts = explode("\n", $this->longdesc);
+				$this->shortdesc = array_shift($parts);
+				$this->shortdesc[0] = strtolower($this->shortdesc[0]); // cosmetic lowercase
 				
 				continue;
 			}
@@ -181,7 +210,7 @@ class CProject {
 			}
 			foreach($this->downloads as $idx => $filename) {
 				foreach(array_keys($known_tags) as $tagname) {
-					if (stripos($filename, $tagname) !== false) {
+					if (stripos($filename, (string)$tagname) !== false) {
 						$found_idx[$idx] = $tagname;
 						
 						$render_per_tag[$tagname][$idx] = $filename;
@@ -269,8 +298,20 @@ class CProject {
 		
 		ob_start();
 		$this->index();
-		$idxfile = template($this->projname.' | '.SITE_TITLE, ob_get_clean());
-		file_put_contents(BASEDIR.'wwwroot/'.$this->projname.'.html', $idxfile);
+		
+		$extra_head_items = [];
+		
+		$extra_head_items[] = '<link rel="canonical" href="'.hesc(BASEURL.$this->projname).'">'; // TODO include golang `go get` meta if necessary
+		
+		if (strlen($this->go_get_target) > 0) {
+			$extra_head_items[] = '<meta name="go-import" content="'.hesc($this->go_get_target).'">'; // TODO include golang `go get` meta if necessary
+		}
+		
+		$idxfile = template($this->projname.' | '.SITE_TITLE, ob_get_clean(), implode("\n", $extra_head_items));
+		mkdir(BASEDIR.'wwwroot/'.$this->projname);
+		
+		file_put_contents(BASEDIR.'wwwroot/'.$this->projname.'/index.html', $idxfile); // new URL format
+		file_put_contents(BASEDIR.'wwwroot/'.$this->projname.'.html', redirecthtml(BASEURL.$this->projname)); // old URL format
 	}
 	
 	public function getClassAttr() {
@@ -294,7 +335,7 @@ class CProject {
 	<ul class="<?=$include_header ? 'downloads-large' : 'downloads-small' ?>">
 <?php foreach($render_downloads as $idx => $filename) { ?> 
 		<li>
-			<a href="srv/<?=hesc($this->downloads_hashes[$idx])?>/<?=hesc(rawurlencode($filename))?>"><?=hesc($filename)?></a>
+			<a href="<?=BASEURL?>srv/<?=hesc($this->downloads_hashes[$idx])?>/<?=hesc(rawurlencode($filename))?>"><?=hesc($filename)?></a>
 			<small>
 				<?=hesc(fbytes(filesize(BASEDIR.'wwwroot/srv/'.$this->downloads_hashes[$idx].'/'.$filename)))?>
 			</small>
@@ -338,7 +379,7 @@ class CProject {
 <?php if (count($this->images)) { ?> 
 	<div class="projimg">
 	<?php foreach($this->images as $idx => $origname) { ?> 
-		<a href="img/<?=hesc(urlencode($this->projname))?>_<?=$idx?>.<?=str_ext($origname)?>"><img src="img/<?=hesc(urlencode($this->projname))?>_<?=$idx?>_thumb.jpg" class="thumbimage"></a>
+		<a href="<?=BASEURL?>img/<?=hesc(urlencode($this->projname))?>_<?=$idx?>.<?=str_ext($origname)?>"><img src="<?=BASEURL?>img/<?=hesc(urlencode($this->projname))?>_<?=$idx?>_thumb.jpg" class="thumbimage"></a>
 	<?php } ?> 
 	</div>
 	

lib/bootstrap.php

@@ -29,6 +29,7 @@ function main($args) {
 	}
 	
 	define('BASEDIR',       $basedir);
+	define('BASEURL',       trim($config['codesite']['baseurl']));
 	define('SITE_TITLE',    trim($config['codesite']['title']));
 	define('PAGE_THUMB_W',  intval($config['codesite']['page_thumb_w']));
 	define('PAGE_THUMB_H',  intval($config['codesite']['page_thumb_h']));
@@ -45,6 +46,9 @@ function main($args) {
 		if (array_key_exists('redirect', $config)) {
 			buildredirects( $config['redirect'] );
 		}
+		if (array_key_exists('golang-subpackages', $config)) {
+			buildgosubpackages( $config['golang-subpackages'] );
+		}
 	} else {
 		buildprojects($pos, array_decimate(listprojects(), $total, $pos));
 	}

lib/template.php

@@ -1,18 +1,21 @@
 <?php
 
-function template($title, $content) {
+function template($title, $content, $extra_head='') {
 	ob_start();
 ?>
 <!DOCTYPE html>
 <html>
 	<head>
+		<title><?=hesc($title)?></title>
+		<meta http-equiv="X-UA-Compatible" content="IE=edge"> 
 		<meta charset="UTF-8">
 		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" >
 		<meta name="viewport" content="width=960" >
-		<link type="text/css" rel="stylesheet" href="static/normalize.css">
-		<link type="text/css" rel="stylesheet" href="static/style.css">
-		<script type="text/javascript" src="static/site.js"></script>
-		<title><?=hesc($title)?></title>
+		<?=$extra_head?> 
+		<link rel="icon" href="<?=BASEURL?>static/favicon.ico" type="image/x-icon">
+		<link type="text/css" rel="stylesheet" href="<?=BASEURL?>static/normalize.css">
+		<link type="text/css" rel="stylesheet" href="<?=BASEURL?>static/style.css">
+		<script type="text/javascript" src="<?=BASEURL?>static/site.js"></script>		
 	</head>
 	<body>
 		<div id="container">
@@ -108,6 +111,10 @@ function buildcommon() {
 		array_map('imagedestroy', $handles); // free
 	}
 	
+	// Cache-busting stylesheet
+
+	$style = '.homeimage-sprite { background-image: url("logos.jpg?'.md5_file(BASEDIR.'wwwroot/logos.jpg').'"); }';
+
 	// Build index page
 
 	ob_start();
@@ -119,6 +126,10 @@ function buildcommon() {
 		<!-- }} --> 
 	<?php } ?> 
 		
+		<style type="text/css">
+	<?php echo $style; ?> 
+		</style>
+
 		<table id="projtable-main" class="projtable">
 	<?php foreach ($plist as $i => $pr) { ?> 
 			<tr class="<?=$pr->getClassAttr()?>"
@@ -130,13 +141,13 @@ function buildcommon() {
 				data-sort-ls="-<?=$pr->lifespan?>"
 			>
 				<td>
-					<a href="<?=hesc(urlencode($pr->projname))?>.html"><?=(is_null($handle_lookup[$pr->projname]) ? '<div class="no-image"></div>' : '<div class="homeimage homeimage-sprite" style="background-position:0 -'.($handle_lookup[$pr->projname]*INDEX_THUMB_H).'px"></div>')?></a>
+					<a href="<?=hesc(BASEURL.urlencode($pr->projname))?>/"><?=(is_null($handle_lookup[$pr->projname]) ? '<div class="no-image"></div>' : '<div class="homeimage homeimage-sprite" style="background-position:0 -'.($handle_lookup[$pr->projname]*INDEX_THUMB_H).'px"></div>')?></a>
 				</td>
 				<td>
 					<strong><?=hesc(str_replace('_', ' ', $pr->projname))?></strong><?php if (SHOW_BLURBS) { ?>,
 					<?=hesc($pr->shortdesc)?> 
 					<?php } ?> 
-					<a href="<?=hesc(urlencode($pr->projname))?>.html" class="article-read-more">more...</a>
+					<a href="<?=hesc(BASEURL.urlencode($pr->projname))?>/" class="article-read-more">more...</a>
 		<?php if (strlen($pr->subtag) || count($pr->tags)) { ?> 
 					<br>
 					<small>
@@ -155,20 +166,49 @@ function buildcommon() {
 		</table>
 <?php
 
-	$index = template(SITE_TITLE, ob_get_clean());
+	$extra_head = '<link rel="canonical" href="'.hesc(BASEURL).'">';
+
+	$index = template(SITE_TITLE, ob_get_clean(), $extra_head);
 	file_put_contents(BASEDIR.'wwwroot/index.html', $index);
 	
 	// Done
 }
 
+function redirecthtml($target) {
+	ob_start();
+?>
+<meta http-equiv="refresh" content="0; url=<?=hesc($target)?>">
+<a href="<?=hesc($target)?>">Moved &raquo;</a>
+<?php
+	return ob_get_clean();
+}
+
 function buildredirects($redirects) {
 	foreach($redirects as $oldname => $newname) {
-		ob_start();
-?>
-<meta http-equiv="refresh" content="0; url=<?=hesc($newname)?>.html">
-<a href="<?=hesc($newname)?>.html">Moved &raquo;</a>
-<?php
-		$page = ob_get_clean();
+	
+		$page = redirecthtml(BASEURL.$newname.'/');
+		
+		// old format
 		file_put_contents(BASEDIR.'wwwroot/'.$oldname.'.html', $page);
+		
+		// new format
+		mkdir(BASEDIR.'wwwroot/'.$oldname);
+		file_put_contents(BASEDIR.'wwwroot/'.$oldname.'/index.html', $page);
+	}
+}
+
+function buildgosubpackages($packages) {
+	foreach($packages as $path => $goGetStr) {
+	
+		$page = (
+			'<meta name="go-import" content="'.hesc($goGetStr).'">'.
+			"\n".
+			redirecthtml(BASEURL)
+		);
+		
+		// new directory format only
+		mkdir_all(BASEDIR.'wwwroot/'.$path);
+		file_put_contents(BASEDIR.'wwwroot/'.$path.'/index.html', $page);
+	
 	}
 }

lib/util.php

@@ -1,20 +1,29 @@
 <?php
 
-function mkshield($left_str, $right_str, $color_str) {
+function mkshield($left_str, $right_str, $color_str, $params=[]) {
 	$filename = sprintf(
 		"%s-%s-%s.svg", 
 		rawurlencode(str_replace('-', '--', $left_str)),
 		rawurlencode(str_replace('-', '--', $right_str)),
 		rawurlencode($color_str)
 	);
+	if (count($params) > 0) {
+		$filename .= '?' . http_build_query($params);
+	}
 	
-	$cache_path = SHIELDS_CACHE_DIR.$filename;
+	$cache_path = SHIELDS_CACHE_DIR.urlencode($filename);
 	
 	if (file_exists($cache_path)) {
 		return file_get_contents($cache_path);
 		
 	} else {
 		$retn = file_get_contents('https://img.shields.io/badge/'.$filename);
+		
+		// We need unique IDs
+		$prefix = substr(sha1($filename), 8).'-';
+		$retn = str_replace('id="',  'id="'.$prefix,  $retn);
+		$retn = str_replace('url(#', 'url(#'.$prefix, $retn);
+		
 		file_put_contents($cache_path, $retn);
 		return $retn;
 		
@@ -103,6 +112,11 @@ function hesc($sz) {
 	return @htmlentities($sz, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8');
 }
 
+function mkdir_all($path) {
+	$epath = escapeshellarg($path);
+	`mkdir -p ${epath}`;
+}
+
 function text2html($sz) {
 
 	$identity = function($sz) {
@@ -133,13 +147,20 @@ function text2html($sz) {
 		$base = hesc($sz);	
 		
 		$base = preg_replace('~^=+(.+)=+~m', '<strong>\\1</strong>', $base);
-		$base = preg_replace('~(https?://[^ \\r\\n\\t]+)~i', '<a href="\\1">\\1</a>', $base);
+		$base = preg_replace('~\\[url=([^\\]]+?)\\](.+?)\\[/url\\]~m', '<a href="\\1">\\2</a>', $base);
+
+		$base = preg_replace('~\\[([^\\]]+?)\\]\\((https?://.+?)\\)~m', '<a href="\\2">\\1</a>', $base); // Support markdown-style URLs
+
+		$base = preg_replace('~([^="])(https?://[^ \\r\\n\\t]+)~i', '\\1<a href="\\2">\\2</a>', $base); // Support standalone URLs
 		$base = preg_replace('~\\[b\\](.+?)\\[/b\\]~m', '<strong>\\1</strong>', $base);
 		$base = preg_replace('~\\[i\\](.+?)\\[/i\\]~m', '<i>\\1</i>', $base);
 		$base = preg_replace('~\\[spoiler\\](.+?)\\[/spoiler\\]~m', '<span class="spoiler">\\1</span>', $base);
-		$base = preg_replace('~\\[entry=([^\\]]+?)\\](.+?)\\[/entry\\]~m', '<a href="\\1.html">\\2</a>', $base);
 		$base = preg_replace('~\n- ~ms', "\n&bull; ", $base);
 		
+		$base = preg_replace('~^```.+$~m', '`', $base); // Convert ```html to single `{}` element
+
+		// TODO support markdown tables
+
 		$btparts = explode('`', $base);
 		if (count($btparts) > 1 && (count($btparts) % 2)) {
 			for ($i = 1, $e = count($btparts); $i < $e; $i += 2) {
@@ -147,7 +168,7 @@ function text2html($sz) {
 				if (strpos($btparts[$i], "\n") !== false) {
 					$class .= ' code-multiline';
 				}
-				$btparts[$i] = '<span class="'.$class.'">'.$btparts[$i].'</span>';
+				$btparts[$i] = '<span class="'.$class.'">'.ltrim($btparts[$i], "\n").'</span>';
 			}
 		}
 		

mkdist.sh

@@ -3,7 +3,12 @@
 set -eu
 
 tar cJvf "codesite-$(date +%s).tar.xz" \
-	rebuild.php rebuild.sh sites/codesite.example.com static_global \
+	--exclude='sites/codesite.example.com/wwwroot' \
+	rebuild.sh \
+	sites/codesite.example.com \
+	static \
+	lib \
 	--owner=0 --group=0
 
 read -p "Press any key to continue..."
+

rebuild.sh

@@ -9,44 +9,45 @@ numcpus() {
 }
 
 buildsite() {
+	local site="$1"
 
-	echo "Site: ${1}"
+	echo "Site: ${site}"
+	(
+		cd "$site"
 
-	pushd "$1" >/dev/null
+		echo "Cleaning wwwroot directory..."
 
-	echo "Cleaning wwwroot directory..."
-
-	if [[ -d wwwroot ]] ; then
-		rm -r wwwroot
-	fi	
-	mkdir -p wwwroot/{img,srv,static}
-	
-	echo "Copying static resources..."
-	
-	if [[ ! -d static ]] ; then
-		mkdir static
-	fi
-	cp "${APP_DIR}/static/"* wwwroot/static || true
-	cp static/* wwwroot/static || true
-	
-	for htm in footer header homepage_blurb ; do
-		if [[ ! -f "${htm}.htm" ]] ; then
-			touch "${htm}.htm"
-		fi
-	done
-
-	echo "Building pages..."
-
-	local threadcount=$(numcpus)
-	for i in $(seq 0 "$threadcount") ; do
-		"${APP_DIR}/lib/bootstrap.php" "$threadcount" "$i" &
-	done
-	wait
-
-	echo "Site: ${1} finished."
-	echo ""
+		if [[ -d wwwroot ]] ; then
+			rm -r wwwroot
+		fi	
+		mkdir -p wwwroot/{img,srv,static}
 		
-	popd >/dev/null
+		echo "Copying static resources..."
+		
+		if [[ ! -d static ]] ; then
+			mkdir static
+		fi
+		cp "${APP_DIR}/static/"* wwwroot/static || true
+		cp static/* wwwroot/static || true
+		
+		for htm in footer header homepage_blurb ; do
+			if [[ ! -f "${htm}.htm" ]] ; then
+				touch "${htm}.htm"
+			fi
+		done
+
+		echo "Building pages..."
+
+		local threadcount=$(numcpus)
+		for i in $(seq 0 "$threadcount") ; do
+			"${APP_DIR}/lib/bootstrap.php" "$threadcount" "$i" &
+		done
+		wait
+
+		echo "Site: ${site} finished."
+		echo ""
+	)
+
 }
 
 usage() {