package main

type Token string

func tokenize(line string) []Token {
	ret := make([]Token, 0)

	wip := Token("")
	i := -1

first:
	for {
		i += 1
		if i >= len(line) {
			break // exhausted
		}

		char := line[i]

		// Advance to first non-whitespace token
		if char == ' ' || char == '\t' {
			continue
		}

		i -= 1
	second:
		for {
			i += 1
			if i >= len(line) {
				break // exhausted
			}

			char := line[i]

			// If we reach whitespace, cut off
			if char == ' ' || char == '\t' {
				ret = append(ret, wip)
				wip = Token("")
				continue first
			}

			// If we reach line-separator, cut off
			if char == '\n' || char == ';' {
				if len(wip) > 0 {
					ret = append(ret, wip)
				}
				ret = append(ret, Token("\n"))
				wip = Token("")
				continue first
			}

			// If we reach ) or (, cut off
			if char == ')' || char == '(' {
				if len(wip) > 0 {
					ret = append(ret, wip)
				}
				ret = append(ret, Token(char))
				wip = Token("")
				continue first
			}

			// If we reach | or &, cut off
			if char == '|' || char == '&' {

				if len(wip) > 0 {
					ret = append(ret, wip)
				}
				wip = Token("")

				// Greedy consume ONE more |/& if present
				tok := Token(char)
				if len(line) > i && line[i+1] == char {
					tok = Token(string(char) + string(char))
					i += 1
				}
				ret = append(ret, tok)

				//.
				continue first
			}

			// If we reach ', inner-loop until we reach another ', then append to our WIP token

			// If we reach ", inner-loop until we reach another ", then replace out escape sequences, then append to our WIP token
			if char == '"' {
				for {
					i += 1
					if i >= len(line) {
						break // exhausted
					}
					char = line[i]

					if char != '"' {
						wip = Token(string(wip) + string(char))
					} else {
						ret = append(ret, wip)
						wip = Token("")
						continue second
					}
				}
			}

			// If we reach #, inner-loop until we reach \n, then ignore

			// If we reach OTHER-PRINTABLE, inner-loop until we reach whitespace
			// Otherwise, add it to our WIP token
			wip = Token(string(wip) + string(char))

		}

	}

	if len(wip) > 0 {
		ret = append(ret, wip)
	}

	return ret
}

func (this *Token) isLineSeparator() bool {
	return *this == ";" || *this == "\n"
}
