diff --git a/lexer/lexer.go b/lexer/lexer.go index c7c978e..893c9b0 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -65,6 +65,15 @@ func Fields(input string) ([]string, error) { } else if c == '\\' { return nil, fmt.Errorf(`Unexpected \ at char %d`, pos) + } else if c == '(' || c == ')' || c == '?' || c == ',' || c == '+' || c == '*' || c == '-' || c == '/' || c == '%' || c == ';' || c == '=' { + // Tokenize separately, even if they appear touching another top-level token + // Should still be safe to re-join + if len(wip) != 0 { + ret = append(ret, wip) + wip = "" + } + ret = append(ret, string(c)) + } else { wip += string(c) } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index ef79db6..f8e5f54 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -59,6 +59,24 @@ func TestLexer(t *testing.T) { expectErr: false, }, + // Special characters lexed as separate tokens, but only at top level + + testCase{ + input: `3+5*(2.3/6);`, + expect: []string{"3", `+`, "5", "*", "(", "2.3", "/", "6", ")", ";"}, + expectErr: false, + }, + testCase{ + input: `SELECT "3+5*(2.3/6)" AS expression;`, + expect: []string{"SELECT", `"3+5*(2.3/6)"`, "AS", "expression", ";"}, + expectErr: false, + }, + testCase{ + input: `INSERT INTO foo (bar, baz) VALUES (?, ?);`, + expect: []string{"INSERT", "INTO", "foo", "(", "bar", ",", "baz", ")", "VALUES", "(", "?", ",", "?", ")", ";"}, + expectErr: false, + }, + // Errors testCase{ @@ -99,7 +117,7 @@ func TestLexer(t *testing.T) { } if !reflect.DeepEqual(out, tc.expect) { - t.Errorf("Test %q got %v, expected %v", tc.input, out, tc.expect) + t.Errorf("Test %q\n- got: %#v\n- expected %#v", tc.input, out, tc.expect) } }