<?php

// MALscrape

// There are probably about 14,000 anime on the site as of 2012-03.

define('DB_FILENAME', 'malscrape.db');
define('BUNCHSIZE',   50);

function dbh() {
	static $db = null;
	if (! is_null($db)) return $db;

	$db = new \PDO('sqlite:' . DB_FILENAME);

	$db->query('CREATE TABLE IF NOT EXISTS malscrape (
		last_run INTEGER,
	);');
	$db->query('CREATE TABLE IF NOT EXISTS anime (
		id INTEGER,
		type VARCHAR(16),
		air_status VARCHAR(255),
		air_date VARCHAR(255),
		episodes INTEGER,
		duration INTEGER,
		score DOUBLE,
		vote_count INTEGER,
		description TEXT
	);');
	$db->query('CREATE TABLE IF NOT EXISTS names (
		name VARCHAR(255),
		anime_id INTEGER
	);');
	$db->query('CREATE TABLE IF NOT EXISTS genre_links (
		genre_id INTEGER,
		anime_id INTEGER
	);');
	$db->query('CREATE TABLE IF NOT EXISTS producer_links (
		producer_id INTEGER,
		anime_id INTEGER
	);');
	$db->query('CREATE TABLE IF NOT EXISTS genres (
		id INTEGER,
		name VARCHAR(128)
	);');
	$db->query('CREATE TABLE IF NOT EXISTS producers (
		id INTEGER,
		name VARCHAR(128)
	);');
	$db->query('CREATE TABLE IF NOT EXISTS related (
		from_id INTEGER,
		to_id INTEGER
	);');
	
	return $db;
}

function between(&$sz, $left, $right) {
	$lpos = strpos($sz, $left);
	if ($lpos === false) return '';
	$lpos += strlen($left);
	$rpos = strpos($sz, $right, $lpos);
	if ($rpos === false) return '';
	return substr($sz, $lpos, $rpos - $lpos);
}

function following($s) {
	$apos = strrpos($s, '>');
	return ($apos === false) ? $s : substr($s, $apos+1);
}

function nobr($s) {
	return str_replace('<br />', '', $s);
}

function atags($s) {
	$ret = array();
	$rpos = 0;
	while (($apos = strpos($s, '<a', $rpos)) !== false) {
		$epos = strpos($s, '>', $apos) + 1;
		$rpos = strpos($s, '<', $epos);
		$ret[] = substr($s, $epos, $rpos - $epos);
	}
	return $ret;
}

function aids($s) {
	$ret = array();
	$rpos = 0;
	while (($apos = strpos($s, '.net/anime/', $rpos)) !== false) {
		$apos += 11; // strlen(.net/anime/)
		$rpos = strpos($s, '/', $apos);
		$ret[] = intval(substr($s, $apos, $rpos - $apos));
	}
	return $ret;
}

function dtol($d) {
	$total = 0;
	$words = explode(' ', $d);
	for ($i = 0, $e = count($words); $i < $e; $i++) {
		if ($words[$i] == 'hr.') $total += intval($words[$i - 1]) * 60;
		if ($words[$i] == 'min.') $total += intval($words[$i - 1]);
	}
	return $total;
}

function dbh_fetchall($sql) {
	$result_set = dbh()->query($sql);
	return $result_set->fetchAll( \PDO::FETCH_ASSOC );
}

function dbh_map($sql, $from_col, $to_col) {
	$rows = dbh_fetchall($sql);
	$map = array();
	
	foreach($rows as $row) {
		$map[$row[$from_col]] = $row[$to_col];
	}
	return $map;
}

function scrape($from, $to) {
	
	$db = dbh();
	$db->query('BEGIN TRANSACTION;');
	
	// Retrieve genre and producers into maps
	
	$genres    = dbh_map('SELECT * FROM genres',    'name', 'id');
	$producers = dbh_map('SELECT * FROM producers', 'name', 'id');
	
	$last_count_genres    = count($genres);
	$last_count_producers = count($producers);
	
	$next_genre    = count($genres)    ? max(array_values($genres))    + 1 : 1;
	$next_producer = count($producers) ? max(array_values($producers)) + 1 : 1;
	
	// Prepare queries
	
	$stmt_anime = $db->prepare('
		INSERT INTO anime (
			id, type, air_status, air_date, episodes,
			duration, score, vote_count, description
		) VALUES (
			?, ?, ?, ?, ?, ?, ?, ?, ?
		);
	');
	$stmt_names = $db->prepare('
		INSERT INTO names ( name, anime_id ) VALUES ( ?, ? );
	');
	$stmt_genre_links = $db->prepare('
		INSERT INTO genre_links ( genre_id, anime_id ) VALUES ( ?, ? );
	');
	$stmt_producer_links = $db->prepare('
		INSERT INTO producer_links ( producer_id, anime_id ) VALUES ( ?, ? );
	');
	$stmt_relations = $db->prepare('
		INSERT INTO related ( from_id, to_id ) VALUES ( ?, ? );
	');
	
	// Begin session
	
	for ($id = $from; $id <= $to; $id++) {
	
		// Retrieve webpage
	
		echo "Retrieving anime #".$id."...\n";
		$page = @file_get_contents('http://myanimelist.net/anime/'.$id);
		
		if ($page === false) {
			echo "Couldn't reach MAL, has the internet gone down? ";
			sleep(10);
			echo "Trying again...\n";
			$id -= 1;
			continue;
		}
		
		if (strpos($page, 'No series found, check the series id') !== false) {
			echo "(dead)\n";
			continue;
		}
		
		// Scrape data
		
		$names = array(
			following(between($page, '<h1>', '</h1>'))
		);
		$names += explode(',', between($page, '_text">English:</span> ', '<'));
		$names += explode(',', between($page, '_text">Japanese:</span> ', '<'));
		$names += explode(',', between($page, '_text">Synonyms:</span> ', '<'));
		$names = array_map('trim', $names);
		
		$an_producers = atags(between($page, '_text">Producers:</span> ', '</div>'));
		if (count($an_producers) == 1 && $an_producers[0] == 'add some') {
			// junk
			$an_producers = array();
		}
		
		$an_genres = atags(between($page, '_text">Genres:</span> ', '</div>'));
		
		$relations = aids(between($page, '<h2>Related Anime</h2>', '<div'));
		
		// Scrape and insert into DB
		
		$stmt_anime->execute(array(
			/* ID        */ $id,
			/* Type      */ between($page, '_text">Type:</span> ', '<'),
			/* Air stat. */ between($page, '_text">Status:</span> ', '<'),
			/* Air date  */ between($page, '_text">Aired:</span> ', '<'),
			/* Episodes  */ intval(between($page, '_text">Episodes:</span> ', '<')),
			/* Duration  */ dtol(between($page, '_text">Duration:</span> ', '<')),
			/* Score     */ floatval(between($page, '_text">Score:</span> ', '<')),
			/* Votecount */ intval(between($page, '(scored by ', ' user')),
			/* Descript. */ nobr(between($page, 'Synopsis</h2>', '</td>'))
		));
		
		foreach($names as $name) {
			$stmt_names->execute(array(
				// name, anime_id
				$name, $id
			));
		}
		
		foreach($an_genres as $genre) {
			if (isset($genres[$genre])) {
				$genre_id = $genres[$genre];
			} else {
				$genre_id = $next_genre++;
				$genres[$genre] = $genre_id;
			}
			
			$stmt_genre_links->execute(array(
				// genre_id, anime_id
				$genre_id, $id
			));
		}
		
		if (count($an_producers)) foreach($an_producers as $producer) {
			if (isset($producers[$producer])) {
				$producer_id = $producers[$producer];
			} else {
				$producer_id = $next_producer++;
				$producers[$producer] = $producer_id;
			}		
		
			$stmt_producer_links->execute(array(
				// producer_id, anime_id
				$producer_id, $id
			));
		}
		
		if (count($relations)) foreach($relations as $relation) {
			$stmt_relations->execute(array(
				// from, to
				$id, $relation
			));
		}
	}
	
	// Update the producer and genre tables
	
	if (count($genres) != $last_count_genres) {
		$stmt_genre = $db->prepare('
			INSERT INTO genres ( id, name ) VALUES ( ?, ? );
		');
		$genre_list = array_keys($genres);
		for ($i = $last_count_genres, $e = count($genres); $i < $e; $i++) {
			$stmt_genre->execute(array(
				$genre_list[$i], $genres[$genre_list[$i]]
			));
		}
	}
	
	if (count($producers) != $last_count_producers) {
		$stmt_producer = $db->prepare('
			INSERT INTO producers ( id, name ) VALUES ( ?, ? );
		');
		$producer_list = array_keys($producers);
		for ($i = $last_count_producers, $e = count($producers); $i < $e; $i++){
			$stmt_producer->execute(array(
				$producer_list[$i], $producers[$producer_list[$i]]
			));
		}
	}
	
	$db->query('UPDATE malscrape SET last_run = '.intval(time()).';');
	
	$db->query('COMMIT TRANSACTION;');
	
	echo "Batch completed.\n";
}

function scrape_safe($from, $to) {
	$base = $from;
	while ($base < ($to + 1)) {
		$until = $base + BUNCHSIZE ;
		if ($until > $to) $until = $to + 1;
		
		echo "Starting bunch ".$base." - ".($until-1)." ...\n";
		@copy(DB_FILENAME, DB_FILENAME.'-bak-up_to_'.($base-1).'.db');
		scrape($base, $until - 1);
		
		$base = $until;
	};
}

//
// Command-line usage
//

if (count($_SERVER['argv']) !== 3) {
	die('Usage: php '.$_SERVER['argv'][0]." start_id end_id\n");
}

scrape_safe( intval($_SERVER['argv'][1]), intval($_SERVER['argv'][2]) );
