package parser import ( "strings" "testing" ) func TestExtendedNumberLiterals(t *testing.T) { tests := []struct { input string expected float64 desc string }{ // Hexadecimal {"0x10", 16.0, "lowercase hex"}, {"0X10", 16.0, "uppercase hex"}, {"0xff", 255.0, "hex with letters"}, {"0XFF", 255.0, "hex with uppercase letters"}, {"0x0", 0.0, "hex zero"}, {"0xDEADBEEF", 3735928559.0, "large hex"}, // Binary {"0b1010", 10.0, "lowercase binary"}, {"0B1010", 10.0, "uppercase binary"}, {"0b0", 0.0, "binary zero"}, {"0b1", 1.0, "binary one"}, {"0b11111111", 255.0, "8-bit binary"}, // Scientific notation {"1e2", 100.0, "simple scientific"}, {"1E2", 100.0, "uppercase E"}, {"1.5e2", 150.0, "decimal with exponent"}, {"2e-1", 0.2, "negative exponent"}, {"1.23e+4", 12300.0, "positive exponent with +"}, {"3.14159e0", 3.14159, "zero exponent"}, {"1e10", 1e10, "large exponent"}, // Regular decimals (should still work) {"42", 42.0, "integer"}, {"3.14", 3.14, "decimal"}, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) expr := p.parseExpression(LOWEST) checkParserErrors(t, p) testNumberLiteral(t, expr, tt.expected) }) } } func TestExtendedNumberAssignments(t *testing.T) { tests := []struct { input string identifier string expected float64 desc string }{ {"hex = 0xFF", "hex", 255.0, "hex assignment"}, {"bin = 0b1111", "bin", 15.0, "binary assignment"}, {"sci = 1.5e3", "sci", 1500.0, "scientific assignment"}, {"large = 0xDEADBEEF", "large", 3735928559.0, "large hex"}, {"small = 2e-5", "small", 0.00002, "small scientific"}, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) program := p.ParseProgram() checkParserErrors(t, p) if len(program.Statements) != 1 { t.Fatalf("expected 1 statement, got %d", len(program.Statements)) } stmt, ok := program.Statements[0].(*AssignStatement) if !ok { t.Fatalf("expected AssignStatement, got %T", program.Statements[0]) } if stmt.Name.Value != tt.identifier { t.Errorf("expected identifier %s, got %s", tt.identifier, stmt.Name.Value) } testNumberLiteral(t, stmt.Value, tt.expected) }) } } func TestExtendedNumberExpressions(t *testing.T) { tests := []struct { input string expected string desc string }{ {"0x10 + 0b1010", "(16.00 + 10.00)", "hex + binary"}, {"1e2 * 0xFF", "(100.00 * 255.00)", "scientific * hex"}, {"0b11 - 1e1", "(3.00 - 10.00)", "binary - scientific"}, {"(0x10 + 0b10) * 1e1", "((16.00 + 2.00) * 10.00)", "mixed with precedence"}, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) expr := p.parseExpression(LOWEST) checkParserErrors(t, p) if expr.String() != tt.expected { t.Errorf("expected %s, got %s", tt.expected, expr.String()) } }) } } func TestExtendedNumberErrors(t *testing.T) { tests := []struct { input string expectedError string desc string }{ {"0x", "could not parse '0x' as hexadecimal number", "incomplete hex"}, {"0b", "could not parse '0b' as binary number", "incomplete binary"}, {"0xGHI", "could not parse '0xGHI' as hexadecimal number", "invalid hex digits"}, {"0b123", "could not parse '0b123' as binary number", "invalid binary digits"}, {"1e", "could not parse '1e' as number", "incomplete scientific"}, {"1e+", "could not parse '1e+' as number", "scientific without digits"}, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) p.parseExpression(LOWEST) errors := p.Errors() if len(errors) == 0 { t.Fatalf("expected parsing errors, got none") } found := false for _, err := range errors { if strings.Contains(err.Message, tt.expectedError) { found = true break } } if !found { errorMsgs := make([]string, len(errors)) for i, err := range errors { errorMsgs[i] = err.Message } t.Errorf("expected error containing %q, got %v", tt.expectedError, errorMsgs) } }) } } func TestExtendedNumberStringRepresentation(t *testing.T) { tests := []struct { input string expected string desc string }{ {"0xFF", "255.00", "hex string representation"}, {"0b1111", "15.00", "binary string representation"}, {"1e3", "1000.00", "scientific string representation"}, {"1.5e2", "150.00", "decimal scientific string representation"}, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) expr := p.parseExpression(LOWEST) checkParserErrors(t, p) if expr.String() != tt.expected { t.Errorf("expected %s, got %s", tt.expected, expr.String()) } }) } } func TestTableWithExtendedNumbers(t *testing.T) { tests := []struct { input string expected string desc string }{ {"{0xFF, 0b1010}", "{255.00, 10.00}", "array with hex and binary"}, {"{hex = 0xFF, bin = 0b1010}", "{hex = 255.00, bin = 10.00}", "hash with extended numbers"}, {"{1e2, 0x10, 0b10}", "{100.00, 16.00, 2.00}", "mixed number formats"}, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) expr := p.parseExpression(LOWEST) checkParserErrors(t, p) if expr.String() != tt.expected { t.Errorf("expected %s, got %s", tt.expected, expr.String()) } }) } } func TestLexerExtendedNumbers(t *testing.T) { tests := []struct { input string expected []Token desc string }{ { "0xFF + 0b1010", []Token{ {Type: NUMBER, Literal: "0xFF"}, {Type: PLUS, Literal: "+"}, {Type: NUMBER, Literal: "0b1010"}, {Type: EOF, Literal: ""}, }, "hex and binary tokens", }, { "1.5e-3 * 2E+4", []Token{ {Type: NUMBER, Literal: "1.5e-3"}, {Type: STAR, Literal: "*"}, {Type: NUMBER, Literal: "2E+4"}, {Type: EOF, Literal: ""}, }, "scientific notation tokens", }, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { l := NewLexer(tt.input) for i, expectedToken := range tt.expected { tok := l.NextToken() if tok.Type != expectedToken.Type { t.Errorf("token %d: expected type %v, got %v", i, expectedToken.Type, tok.Type) } if tok.Literal != expectedToken.Literal { t.Errorf("token %d: expected literal %s, got %s", i, expectedToken.Literal, tok.Literal) } } }) } } // Additional existing tests would remain unchanged... func TestLiterals(t *testing.T) { tests := []struct { input string expected any }{ {"42", 42.0}, {"3.14", 3.14}, {`"hello"`, "hello"}, {"true", true}, {"false", false}, {"nil", nil}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) expr := p.parseExpression(LOWEST) checkParserErrors(t, p) switch expected := tt.expected.(type) { case float64: testNumberLiteral(t, expr, expected) case string: testStringLiteral(t, expr, expected) case bool: testBooleanLiteral(t, expr, expected) case nil: testNilLiteral(t, expr) } }) } } func TestTableLiterals(t *testing.T) { tests := []struct { input string expectedPairs int isArray bool description string }{ {"{}", 0, true, "empty table"}, {"{1, 2, 3}", 3, true, "array-like table"}, {"{a = 1, b = 2}", 2, false, "hash-like table"}, {`{"hello", "world"}`, 2, true, "string array"}, {"{x = true, y = false}", 2, false, "boolean hash"}, {"{1}", 1, true, "single element array"}, {"{key = nil}", 1, false, "nil value hash"}, } for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) expr := p.parseExpression(LOWEST) checkParserErrors(t, p) table, ok := expr.(*TableLiteral) if !ok { t.Fatalf("expected TableLiteral, got %T", expr) } if len(table.Pairs) != tt.expectedPairs { t.Errorf("expected %d pairs, got %d", tt.expectedPairs, len(table.Pairs)) } if table.IsArray() != tt.isArray { t.Errorf("expected IsArray() = %t, got %t", tt.isArray, table.IsArray()) } }) } } func TestTableAssignments(t *testing.T) { tests := []struct { input string identifier string pairCount int isArray bool description string }{ {"arr = {1, 2, 3}", "arr", 3, true, "array assignment"}, {"hash = {x = 1, y = 2}", "hash", 2, false, "hash assignment"}, {"empty = {}", "empty", 0, true, "empty table assignment"}, } for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) program := p.ParseProgram() checkParserErrors(t, p) if len(program.Statements) != 1 { t.Fatalf("expected 1 statement, got %d", len(program.Statements)) } stmt, ok := program.Statements[0].(*AssignStatement) if !ok { t.Fatalf("expected AssignStatement, got %T", program.Statements[0]) } if stmt.Name.Value != tt.identifier { t.Errorf("expected identifier %s, got %s", tt.identifier, stmt.Name.Value) } table, ok := stmt.Value.(*TableLiteral) if !ok { t.Fatalf("expected TableLiteral, got %T", stmt.Value) } if len(table.Pairs) != tt.pairCount { t.Errorf("expected %d pairs, got %d", tt.pairCount, len(table.Pairs)) } if table.IsArray() != tt.isArray { t.Errorf("expected IsArray() = %t, got %t", tt.isArray, table.IsArray()) } }) } } func TestTableStringRepresentation(t *testing.T) { tests := []struct { input string expected string }{ {"{}", "{}"}, {"{1, 2}", "{1.00, 2.00}"}, {"{x = 1}", "{x = 1.00}"}, {"{a = 1, b = 2}", "{a = 1.00, b = 2.00}"}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) expr := p.parseExpression(LOWEST) checkParserErrors(t, p) if expr.String() != tt.expected { t.Errorf("expected %s, got %s", tt.expected, expr.String()) } }) } } func TestAssignStatements(t *testing.T) { tests := []struct { input string expectedIdentifier string expectedValue any isExpression bool }{ {"x = 42", "x", 42.0, false}, {"name = \"test\"", "name", "test", false}, {"flag = true", "flag", true, false}, {"ptr = nil", "ptr", nil, false}, {"result = 3 + 4", "result", "(3.00 + 4.00)", true}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) program := p.ParseProgram() checkParserErrors(t, p) if len(program.Statements) != 1 { t.Fatalf("expected 1 statement, got %d", len(program.Statements)) } stmt, ok := program.Statements[0].(*AssignStatement) if !ok { t.Fatalf("expected AssignStatement, got %T", program.Statements[0]) } if stmt.Name.Value != tt.expectedIdentifier { t.Errorf("expected identifier %s, got %s", tt.expectedIdentifier, stmt.Name.Value) } if tt.isExpression { if stmt.Value.String() != tt.expectedValue.(string) { t.Errorf("expected expression %s, got %s", tt.expectedValue.(string), stmt.Value.String()) } } else { switch expected := tt.expectedValue.(type) { case float64: testNumberLiteral(t, stmt.Value, expected) case string: testStringLiteral(t, stmt.Value, expected) case bool: testBooleanLiteral(t, stmt.Value, expected) case nil: testNilLiteral(t, stmt.Value) } } }) } } func TestInfixExpressions(t *testing.T) { tests := []struct { input string leftValue any operator string rightValue any }{ {"5 + 5", 5.0, "+", 5.0}, {"5 - 5", 5.0, "-", 5.0}, {"5 * 5", 5.0, "*", 5.0}, {"5 / 5", 5.0, "/", 5.0}, {"true + false", true, "+", false}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) expr := p.parseExpression(LOWEST) checkParserErrors(t, p) testInfixExpression(t, expr, tt.leftValue, tt.operator, tt.rightValue) }) } } func TestOperatorPrecedence(t *testing.T) { tests := []struct { input string expected string }{ {"1 + 2 * 3", "(1.00 + (2.00 * 3.00))"}, {"2 * 3 + 4", "((2.00 * 3.00) + 4.00)"}, {"(1 + 2) * 3", "((1.00 + 2.00) * 3.00)"}, {"1 + 2 - 3", "((1.00 + 2.00) - 3.00)"}, {"2 * 3 / 4", "((2.00 * 3.00) / 4.00)"}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) expr := p.parseExpression(LOWEST) checkParserErrors(t, p) if expr.String() != tt.expected { t.Errorf("expected %s, got %s", tt.expected, expr.String()) } }) } } func TestParsingErrors(t *testing.T) { tests := []struct { input string expectedError string line int column int }{ {"= 5", "assignment operator '=' without left-hand side identifier", 1, 1}, {"x =", "expected expression after assignment operator", 1, 3}, {"(1 + 2", "expected next token to be )", 1, 7}, {"+ 5", "unexpected operator '+'", 1, 1}, {"1 +", "expected expression after operator '+'", 1, 3}, {"@", "unexpected token '@'", 1, 1}, {"invalid@", "unexpected identifier", 1, 1}, {"{1, 2", "expected next token to be }", 1, 6}, {"{a =", "expected expression after assignment operator", 1, 4}, {"{a = 1,", "expected next token to be }", 1, 8}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) switch tt.input { case "(1 + 2", "+ 5", "1 +", "{1, 2", "{a =", "{a = 1,": p.parseExpression(LOWEST) default: p.ParseProgram() } errors := p.Errors() if len(errors) == 0 { t.Fatalf("expected parsing errors, got none") } found := false for _, err := range errors { if strings.Contains(err.Message, tt.expectedError) { found = true if err.Line != tt.line { t.Errorf("expected error at line %d, got line %d", tt.line, err.Line) } break } } if !found { errorMsgs := make([]string, len(errors)) for i, err := range errors { errorMsgs[i] = err.Message } t.Errorf("expected error containing %q, got %v", tt.expectedError, errorMsgs) } }) } } func TestErrorRecovery(t *testing.T) { input := `x = 42 = 5 y = "hello"` l := NewLexer(input) p := NewParser(l) program := p.ParseProgram() if !p.HasErrors() { t.Fatal("expected parsing errors") } errors := p.Errors() found := false for _, err := range errors { if strings.Contains(err.Message, "assignment operator '=' without left-hand side identifier") { found = true if err.Line != 2 { t.Errorf("expected error at line 2, got line %d", err.Line) } break } } if !found { t.Error("expected specific assignment error") } validStatements := 0 for _, stmt := range program.Statements { if stmt != nil { validStatements++ } } if validStatements < 2 { t.Errorf("expected at least 2 valid statements, got %d", validStatements) } } func TestProgram(t *testing.T) { input := `x = 42 y = "hello" z = true + false` l := NewLexer(input) p := NewParser(l) program := p.ParseProgram() checkParserErrors(t, p) if len(program.Statements) != 3 { t.Fatalf("expected 3 statements, got %d", len(program.Statements)) } expectedIdentifiers := []string{"x", "y", "z"} for i, expectedIdent := range expectedIdentifiers { stmt, ok := program.Statements[i].(*AssignStatement) if !ok { t.Fatalf("statement %d: expected AssignStatement, got %T", i, program.Statements[i]) } if stmt.Name.Value != expectedIdent { t.Errorf("statement %d: expected identifier %s, got %s", i, expectedIdent, stmt.Name.Value) } } } func TestErrorMessages(t *testing.T) { tests := []struct { input string expectedMessage string }{ {"= 5", "Parse error at line 1, column 1: assignment operator '=' without left-hand side identifier (near '=')"}, {"x =", "Parse error at line 1, column 3: expected expression after assignment operator (near '')"}, {"(", "Parse error at line 1, column 1: unexpected end of input (near '')"}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { l := NewLexer(tt.input) p := NewParser(l) p.ParseProgram() if !p.HasErrors() { t.Fatal("expected parsing errors") } errors := p.Errors() if len(errors) == 0 { t.Fatal("expected at least one error") } errorStr := errors[0].Error() if !strings.Contains(errorStr, "Parse error at line") { t.Errorf("expected formatted error message, got: %s", errorStr) } }) } } // Helper functions for testing specific node types func testNumberLiteral(t *testing.T, expr Expression, expected float64) { t.Helper() num, ok := expr.(*NumberLiteral) if !ok { t.Fatalf("expected NumberLiteral, got %T", expr) } if num.Value != expected { t.Errorf("expected %f, got %f", expected, num.Value) } } func testStringLiteral(t *testing.T, expr Expression, expected string) { t.Helper() str, ok := expr.(*StringLiteral) if !ok { t.Fatalf("expected StringLiteral, got %T", expr) } if str.Value != expected { t.Errorf("expected %s, got %s", expected, str.Value) } } func testBooleanLiteral(t *testing.T, expr Expression, expected bool) { t.Helper() boolean, ok := expr.(*BooleanLiteral) if !ok { t.Fatalf("expected BooleanLiteral, got %T", expr) } if boolean.Value != expected { t.Errorf("expected %t, got %t", expected, boolean.Value) } } func testNilLiteral(t *testing.T, expr Expression) { t.Helper() _, ok := expr.(*NilLiteral) if !ok { t.Fatalf("expected NilLiteral, got %T", expr) } } func testIdentifier(t *testing.T, expr Expression, expected string) { t.Helper() ident, ok := expr.(*Identifier) if !ok { t.Fatalf("expected Identifier, got %T", expr) } if ident.Value != expected { t.Errorf("expected %s, got %s", expected, ident.Value) } } func testInfixExpression(t *testing.T, expr Expression, left any, operator string, right any) { t.Helper() infix, ok := expr.(*InfixExpression) if !ok { t.Fatalf("expected InfixExpression, got %T", expr) } if infix.Operator != operator { t.Errorf("expected operator %s, got %s", operator, infix.Operator) } switch leftVal := left.(type) { case float64: testNumberLiteral(t, infix.Left, leftVal) case string: testIdentifier(t, infix.Left, leftVal) case bool: testBooleanLiteral(t, infix.Left, leftVal) } switch rightVal := right.(type) { case float64: testNumberLiteral(t, infix.Right, rightVal) case string: testIdentifier(t, infix.Right, rightVal) case bool: testBooleanLiteral(t, infix.Right, rightVal) } } func checkParserErrors(t *testing.T, p *Parser) { t.Helper() errors := p.Errors() if len(errors) == 0 { return } t.Errorf("parser has %d errors", len(errors)) for _, err := range errors { t.Errorf("parser error: %s", err.Error()) } t.FailNow() }