Mako/scanner/scanner_test.go

package scanner_test

import (
	"testing"

	assert "git.sharkk.net/Go/Assert"
	"git.sharkk.net/Sharkk/Mako/scanner"
	"git.sharkk.net/Sharkk/Mako/types"
)

// Helper function to check token equality
func checkToken(t *testing.T, token types.Token, expectedType types.TokenType, expectedLexeme string, expectedLine int, expectedColumn int) {
	assert.Equal(t, expectedType, token.Type)
	assert.Equal(t, expectedLexeme, token.Lexeme)
	assert.Equal(t, expectedLine, token.Line)
	assert.Equal(t, expectedColumn, token.Column)
}

func TestSingleTokens(t *testing.T) {
	tests := []struct {
		source  string
		tokType types.TokenType
		lexeme  string
		line    int
		column  int
	}{
		{"(", types.LEFT_PAREN, "(", 1, 1},
		{")", types.RIGHT_PAREN, ")", 1, 1},
		{",", types.COMMA, ",", 1, 1},
		{"+", types.PLUS, "+", 1, 1},
		{"-", types.MINUS, "-", 1, 1},
		{"*", types.STAR, "*", 1, 1},
		{"/", types.SLASH, "/", 1, 1},
		{"=", types.EQUAL, "=", 1, 1},
		{"==", types.EQUAL_EQUAL, "==", 1, 1},
		{"!=", types.BANG_EQUAL, "!=", 1, 1},
		{"<", types.LESS, "<", 1, 1},
		{"<=", types.LESS_EQUAL, "<=", 1, 1},
		{">", types.GREATER, ">", 1, 1},
		{">=", types.GREATER_EQUAL, ">=", 1, 1},
		{"if", types.IF, "if", 1, 1},
		{"then", types.THEN, "then", 1, 1},
		{"elseif", types.ELSEIF, "elseif", 1, 1},
		{"else", types.ELSE, "else", 1, 1},
		{"end", types.END, "end", 1, 1},
		{"fn", types.FN, "fn", 1, 1},
		{"return", types.RETURN, "return", 1, 1},
		{"echo", types.ECHO, "echo", 1, 1},
		{"true", types.TRUE, "true", 1, 1},
		{"false", types.FALSE, "false", 1, 1},
		{"nil", types.NIL, "nil", 1, 1},
		{"and", types.AND, "and", 1, 1},
		{"or", types.OR, "or", 1, 1},
		{"identifier", types.IDENTIFIER, "identifier", 1, 1},
		{"...", types.ELLIPSIS, "...", 1, 1},
	}

	for _, test := range tests {
		s := scanner.New(test.source)
		token := s.NextToken()

		checkToken(t, token, test.tokType, test.lexeme, test.line, test.column)

		// Next token should be EOF
		token = s.NextToken()
		checkToken(t, token, types.EOF, "", test.line, test.column+len(test.lexeme))
	}
}

func TestNumbers(t *testing.T) {
	tests := []struct {
		source string
		lexeme string
		value  float64
	}{
		{"123", "123", 123.0},
		{"123.456", "123.456", 123.456},
		{"0.123", "0.123", 0.123},
		{"0", "0", 0.0},
	}

	for _, test := range tests {
		s := scanner.New(test.source)
		token := s.NextToken()

		assert.Equal(t, types.NUMBER, token.Type)
		assert.Equal(t, test.lexeme, token.Lexeme)
		assert.Equal(t, test.value, token.Literal.(float64))
	}
}

func TestStrings(t *testing.T) {
	tests := []struct {
		source string
		lexeme string
		value  string
	}{
		{"\"hello\"", "\"hello\"", "hello"},
		{"\"\"", "\"\"", ""},
		{"\"hello world\"", "\"hello world\"", "hello world"},
	}

	for _, test := range tests {
		s := scanner.New(test.source)
		token := s.NextToken()

		assert.Equal(t, types.STRING, token.Type)
		assert.Equal(t, test.lexeme, token.Lexeme)
		assert.Equal(t, test.value, token.Literal.(string))
	}
}

func TestComments(t *testing.T) {
	s := scanner.New("// This is a comment\nx = 5")

	token := s.NextToken()
	checkToken(t, token, types.IDENTIFIER, "x", 2, 1)

	token = s.NextToken()
	checkToken(t, token, types.EQUAL, "=", 2, 3)

	token = s.NextToken()
	checkToken(t, token, types.NUMBER, "5", 2, 5)

	token = s.NextToken()
	checkToken(t, token, types.EOF, "", 2, 6)
}

func TestMultipleTokens(t *testing.T) {
	source := "fn add(a, b) return a + b end"
	s := scanner.New(source)

	expected := []struct {
		tokType types.TokenType
		lexeme  string
	}{
		{types.FN, "fn"},
		{types.IDENTIFIER, "add"},
		{types.LEFT_PAREN, "("},
		{types.IDENTIFIER, "a"},
		{types.COMMA, ","},
		{types.IDENTIFIER, "b"},
		{types.RIGHT_PAREN, ")"},
		{types.RETURN, "return"},
		{types.IDENTIFIER, "a"},
		{types.PLUS, "+"},
		{types.IDENTIFIER, "b"},
		{types.END, "end"},
		{types.EOF, ""},
	}

	for _, exp := range expected {
		token := s.NextToken()
		assert.Equal(t, exp.tokType, token.Type)
		assert.Equal(t, exp.lexeme, token.Lexeme)
	}
}

func TestScanTokens(t *testing.T) {
	source := "fn add(a, b) return a + b end"
	s := scanner.New(source)

	tokens := s.ScanTokens()

	assert.Equal(t, 13, len(tokens))
	assert.Equal(t, types.FN, tokens[0].Type)
	assert.Equal(t, types.EOF, tokens[12].Type)
}

func TestLineAndColumn(t *testing.T) {
	source := "x = 1\ny = 2"
	s := scanner.New(source)

	token := s.NextToken() // x
	checkToken(t, token, types.IDENTIFIER, "x", 1, 1)

	token = s.NextToken() // =
	checkToken(t, token, types.EQUAL, "=", 1, 3)

	token = s.NextToken() // 1
	checkToken(t, token, types.NUMBER, "1", 1, 5)

	token = s.NextToken() // y
	checkToken(t, token, types.IDENTIFIER, "y", 2, 1)

	token = s.NextToken() // =
	checkToken(t, token, types.EQUAL, "=", 2, 3)

	token = s.NextToken() // 2
	checkToken(t, token, types.NUMBER, "2", 2, 5)
}

func TestErrors(t *testing.T) {
	// Unterminated string
	s := scanner.New("\"unterminated")
	token := s.NextToken()
	assert.Equal(t, types.ERROR, token.Type)

	// Invalid character
	s = scanner.New("@")
	token = s.NextToken()
	assert.Equal(t, types.ERROR, token.Type)

	// Standalone ! without =
	s = scanner.New("!")
	token = s.NextToken()
	assert.Equal(t, types.ERROR, token.Type)
}

func TestLiterals(t *testing.T) {
	// Test true literal
	s := scanner.New("true")
	token := s.NextToken()
	assert.Equal(t, types.TRUE, token.Type)
	assert.Equal(t, true, token.Literal.(bool))

	// Test false literal
	s = scanner.New("false")
	token = s.NextToken()
	assert.Equal(t, types.FALSE, token.Type)
	assert.Equal(t, false, token.Literal.(bool))

	// Test nil literal
	s = scanner.New("nil")
	token = s.NextToken()
	assert.Equal(t, types.NIL, token.Type)
	assert.Nil(t, token.Literal)
}

func TestWhitespace(t *testing.T) {
	s := scanner.New("   \t  \r\n  x")
	token := s.NextToken()
	checkToken(t, token, types.IDENTIFIER, "x", 2, 3)
}