package scanner_test import ( "testing" assert "git.sharkk.net/Go/Assert" "git.sharkk.net/Sharkk/Mako/scanner" "git.sharkk.net/Sharkk/Mako/types" ) // Helper function to check token equality func checkToken(t *testing.T, token types.Token, expectedType types.TokenType, expectedLexeme string, expectedLine int, expectedColumn int) { assert.Equal(t, expectedType, token.Type) assert.Equal(t, expectedLexeme, token.Lexeme) assert.Equal(t, expectedLine, token.Line) assert.Equal(t, expectedColumn, token.Column) } func TestSingleTokens(t *testing.T) { tests := []struct { source string tokType types.TokenType lexeme string line int column int }{ {"(", types.LEFT_PAREN, "(", 1, 1}, {")", types.RIGHT_PAREN, ")", 1, 1}, {",", types.COMMA, ",", 1, 1}, {"+", types.PLUS, "+", 1, 1}, {"-", types.MINUS, "-", 1, 1}, {"*", types.STAR, "*", 1, 1}, {"/", types.SLASH, "/", 1, 1}, {"=", types.EQUAL, "=", 1, 1}, {"==", types.EQUAL_EQUAL, "==", 1, 1}, {"!=", types.BANG_EQUAL, "!=", 1, 1}, {"<", types.LESS, "<", 1, 1}, {"<=", types.LESS_EQUAL, "<=", 1, 1}, {">", types.GREATER, ">", 1, 1}, {">=", types.GREATER_EQUAL, ">=", 1, 1}, {"if", types.IF, "if", 1, 1}, {"then", types.THEN, "then", 1, 1}, {"elseif", types.ELSEIF, "elseif", 1, 1}, {"else", types.ELSE, "else", 1, 1}, {"end", types.END, "end", 1, 1}, {"fn", types.FN, "fn", 1, 1}, {"return", types.RETURN, "return", 1, 1}, {"echo", types.ECHO, "echo", 1, 1}, {"true", types.TRUE, "true", 1, 1}, {"false", types.FALSE, "false", 1, 1}, {"nil", types.NIL, "nil", 1, 1}, {"and", types.AND, "and", 1, 1}, {"or", types.OR, "or", 1, 1}, {"identifier", types.IDENTIFIER, "identifier", 1, 1}, {"...", types.ELLIPSIS, "...", 1, 1}, } for _, test := range tests { s := scanner.New(test.source) token := s.NextToken() checkToken(t, token, test.tokType, test.lexeme, test.line, test.column) // Next token should be EOF token = s.NextToken() checkToken(t, token, types.EOF, "", test.line, test.column+len(test.lexeme)) } } func TestNumbers(t *testing.T) { tests := []struct { source string lexeme string value float64 }{ {"123", "123", 123.0}, {"123.456", "123.456", 123.456}, {"0.123", "0.123", 0.123}, {"0", "0", 0.0}, } for _, test := range tests { s := scanner.New(test.source) token := s.NextToken() assert.Equal(t, types.NUMBER, token.Type) assert.Equal(t, test.lexeme, token.Lexeme) assert.Equal(t, test.value, token.Literal.(float64)) } } func TestStrings(t *testing.T) { tests := []struct { source string lexeme string value string }{ {"\"hello\"", "\"hello\"", "hello"}, {"\"\"", "\"\"", ""}, {"\"hello world\"", "\"hello world\"", "hello world"}, } for _, test := range tests { s := scanner.New(test.source) token := s.NextToken() assert.Equal(t, types.STRING, token.Type) assert.Equal(t, test.lexeme, token.Lexeme) assert.Equal(t, test.value, token.Literal.(string)) } } func TestComments(t *testing.T) { s := scanner.New("// This is a comment\nx = 5") token := s.NextToken() checkToken(t, token, types.IDENTIFIER, "x", 2, 1) token = s.NextToken() checkToken(t, token, types.EQUAL, "=", 2, 3) token = s.NextToken() checkToken(t, token, types.NUMBER, "5", 2, 5) token = s.NextToken() checkToken(t, token, types.EOF, "", 2, 6) } func TestMultipleTokens(t *testing.T) { source := "fn add(a, b) return a + b end" s := scanner.New(source) expected := []struct { tokType types.TokenType lexeme string }{ {types.FN, "fn"}, {types.IDENTIFIER, "add"}, {types.LEFT_PAREN, "("}, {types.IDENTIFIER, "a"}, {types.COMMA, ","}, {types.IDENTIFIER, "b"}, {types.RIGHT_PAREN, ")"}, {types.RETURN, "return"}, {types.IDENTIFIER, "a"}, {types.PLUS, "+"}, {types.IDENTIFIER, "b"}, {types.END, "end"}, {types.EOF, ""}, } for _, exp := range expected { token := s.NextToken() assert.Equal(t, exp.tokType, token.Type) assert.Equal(t, exp.lexeme, token.Lexeme) } } func TestScanTokens(t *testing.T) { source := "fn add(a, b) return a + b end" s := scanner.New(source) tokens := s.ScanTokens() assert.Equal(t, 13, len(tokens)) assert.Equal(t, types.FN, tokens[0].Type) assert.Equal(t, types.EOF, tokens[12].Type) } func TestLineAndColumn(t *testing.T) { source := "x = 1\ny = 2" s := scanner.New(source) token := s.NextToken() // x checkToken(t, token, types.IDENTIFIER, "x", 1, 1) token = s.NextToken() // = checkToken(t, token, types.EQUAL, "=", 1, 3) token = s.NextToken() // 1 checkToken(t, token, types.NUMBER, "1", 1, 5) token = s.NextToken() // y checkToken(t, token, types.IDENTIFIER, "y", 2, 1) token = s.NextToken() // = checkToken(t, token, types.EQUAL, "=", 2, 3) token = s.NextToken() // 2 checkToken(t, token, types.NUMBER, "2", 2, 5) } func TestErrors(t *testing.T) { // Unterminated string s := scanner.New("\"unterminated") token := s.NextToken() assert.Equal(t, types.ERROR, token.Type) // Invalid character s = scanner.New("@") token = s.NextToken() assert.Equal(t, types.ERROR, token.Type) // Standalone ! without = s = scanner.New("!") token = s.NextToken() assert.Equal(t, types.ERROR, token.Type) } func TestLiterals(t *testing.T) { // Test true literal s := scanner.New("true") token := s.NextToken() assert.Equal(t, types.TRUE, token.Type) assert.Equal(t, true, token.Literal.(bool)) // Test false literal s = scanner.New("false") token = s.NextToken() assert.Equal(t, types.FALSE, token.Type) assert.Equal(t, false, token.Literal.(bool)) // Test nil literal s = scanner.New("nil") token = s.NextToken() assert.Equal(t, types.NIL, token.Type) assert.Nil(t, token.Literal) } func TestWhitespace(t *testing.T) { s := scanner.New(" \t \r\n x") token := s.NextToken() checkToken(t, token, types.IDENTIFIER, "x", 2, 3) }