From 4b5faae944cd1647649eb7d8ed74c23aabb7938b Mon Sep 17 00:00:00 2001 From: Sky Johnson Date: Tue, 10 Jun 2025 10:25:56 -0500 Subject: [PATCH] add table access --- parser/ast.go | 24 ++++++++- parser/lexer.go | 16 +++--- parser/parser.go | 70 +++++++++++++++++++++--- parser/tests/assignments_test.go | 88 +++++++++++++++++++++++++++++-- parser/tests/conditionals_test.go | 77 +++++++++++++++++++++++++-- parser/tests/parser_test.go | 74 ++++++++++++++++++++++++-- parser/token.go | 24 +++++---- 7 files changed, 337 insertions(+), 36 deletions(-) diff --git a/parser/ast.go b/parser/ast.go index bb09fc6..a684676 100644 --- a/parser/ast.go +++ b/parser/ast.go @@ -34,7 +34,7 @@ func (p *Program) String() string { // AssignStatement represents variable assignment type AssignStatement struct { - Name *Identifier + Name Expression // Changed from *Identifier to Expression for member access Value Expression } @@ -157,6 +157,28 @@ func (ie *InfixExpression) String() string { return fmt.Sprintf("(%s %s %s)", ie.Left.String(), ie.Operator, ie.Right.String()) } +// IndexExpression represents table[key] access +type IndexExpression struct { + Left Expression + Index Expression +} + +func (ie *IndexExpression) expressionNode() {} +func (ie *IndexExpression) String() string { + return fmt.Sprintf("%s[%s]", ie.Left.String(), ie.Index.String()) +} + +// DotExpression represents table.key access +type DotExpression struct { + Left Expression + Key string +} + +func (de *DotExpression) expressionNode() {} +func (de *DotExpression) String() string { + return fmt.Sprintf("%s.%s", de.Left.String(), de.Key) +} + // TablePair represents a key-value pair in a table type TablePair struct { Key Expression // nil for array-style elements diff --git a/parser/lexer.go b/parser/lexer.go index 45efac4..f6b21b9 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -218,6 +218,8 @@ func (l *Lexer) NextToken() Token { tok = Token{Type: STAR, Literal: string(l.ch), Line: l.line, Column: l.column} case '/': tok = Token{Type: SLASH, Literal: string(l.ch), Line: l.line, Column: l.column} + case '.': + tok = Token{Type: DOT, Literal: string(l.ch), Line: l.line, Column: l.column} case '(': tok = Token{Type: LPAREN, Literal: string(l.ch), Line: l.line, Column: l.column} case ')': @@ -226,18 +228,20 @@ func (l *Lexer) NextToken() Token { tok = Token{Type: LBRACE, Literal: string(l.ch), Line: l.line, Column: l.column} case '}': tok = Token{Type: RBRACE, Literal: string(l.ch), Line: l.line, Column: l.column} - case ',': - tok = Token{Type: COMMA, Literal: string(l.ch), Line: l.line, Column: l.column} - case '"': - tok.Type = STRING - tok.Literal = l.readString() case '[': if l.peekChar() == '[' { tok.Type = STRING tok.Literal = l.readMultilineString() } else { - tok = Token{Type: ILLEGAL, Literal: string(l.ch), Line: l.line, Column: l.column} + tok = Token{Type: LBRACKET, Literal: string(l.ch), Line: l.line, Column: l.column} } + case ']': + tok = Token{Type: RBRACKET, Literal: string(l.ch), Line: l.line, Column: l.column} + case ',': + tok = Token{Type: COMMA, Literal: string(l.ch), Line: l.line, Column: l.column} + case '"': + tok.Type = STRING + tok.Literal = l.readString() case 0: tok.Literal = "" tok.Type = EOF diff --git a/parser/parser.go b/parser/parser.go index f921b55..54eba42 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -54,6 +54,8 @@ func NewParser(lexer *Lexer) *Parser { p.registerInfix(MINUS, p.parseInfixExpression) p.registerInfix(SLASH, p.parseInfixExpression) p.registerInfix(STAR, p.parseInfixExpression) + p.registerInfix(DOT, p.parseDotExpression) + p.registerInfix(LBRACKET, p.parseIndexExpression) // Read two tokens, so curToken and peekToken are both set p.nextToken() @@ -98,11 +100,8 @@ func (p *Parser) ParseProgram() *Program { func (p *Parser) parseStatement() Statement { switch p.curToken.Type { case IDENT: - if p.peekTokenIs(ASSIGN) { - return p.parseAssignStatement() - } - p.addError("unexpected identifier, expected assignment or declaration") - return nil + // Try to parse as assignment (handles both simple and member access) + return p.parseAssignStatement() case IF: return p.parseIfStatement() case ECHO: @@ -125,12 +124,27 @@ func (p *Parser) parseStatement() Statement { func (p *Parser) parseAssignStatement() *AssignStatement { stmt := &AssignStatement{} - if !p.curTokenIs(IDENT) { - p.addError("expected identifier for assignment") + // Parse left-hand side expression (can be identifier or member access) + stmt.Name = p.ParseExpression(LOWEST) + if stmt.Name == nil { + p.addError("expected expression for assignment left-hand side") return nil } - stmt.Name = &Identifier{Value: p.curToken.Literal} + // Check if next token is assignment operator + if !p.peekTokenIs(ASSIGN) { + p.addError("unexpected identifier, expected assignment or declaration") + return nil + } + + // Validate assignment target + switch stmt.Name.(type) { + case *Identifier, *DotExpression, *IndexExpression: + // Valid assignment targets + default: + p.addError("invalid assignment target") + return nil + } if !p.expectPeek(ASSIGN) { return nil @@ -467,6 +481,38 @@ func (p *Parser) parseInfixExpression(left Expression) Expression { return expression } +func (p *Parser) parseDotExpression(left Expression) Expression { + if !p.expectPeek(IDENT) { + p.addError("expected identifier after '.'") + return nil + } + + return &DotExpression{ + Left: left, + Key: p.curToken.Literal, + } +} + +func (p *Parser) parseIndexExpression(left Expression) Expression { + p.nextToken() // move past '[' + + index := p.ParseExpression(LOWEST) + if index == nil { + p.addError("expected expression inside brackets") + return nil + } + + if !p.expectPeek(RBRACKET) { + p.addError("expected ']' after index expression") + return nil + } + + return &IndexExpression{ + Left: left, + Index: index, + } +} + // Helper methods func (p *Parser) curTokenIs(t TokenType) bool { return p.curToken.Type == t @@ -517,6 +563,8 @@ func (p *Parser) noPrefixParseFnError(t TokenType) { message = "unexpected closing parenthesis" case RBRACE: message = "unexpected closing brace" + case RBRACKET: + message = "unexpected closing bracket" case EOF: message = "unexpected end of input" default: @@ -582,6 +630,8 @@ func tokenTypeString(t TokenType) string { return "*" case SLASH: return "/" + case DOT: + return "." case LPAREN: return "(" case RPAREN: @@ -590,6 +640,10 @@ func tokenTypeString(t TokenType) string { return "{" case RBRACE: return "}" + case LBRACKET: + return "[" + case RBRACKET: + return "]" case COMMA: return "," case VAR: diff --git a/parser/tests/assignments_test.go b/parser/tests/assignments_test.go index 88b7bbc..3055b17 100644 --- a/parser/tests/assignments_test.go +++ b/parser/tests/assignments_test.go @@ -36,8 +36,14 @@ func TestAssignStatements(t *testing.T) { t.Fatalf("expected AssignStatement, got %T", program.Statements[0]) } - if stmt.Name.Value != tt.expectedIdentifier { - t.Errorf("expected identifier %s, got %s", tt.expectedIdentifier, stmt.Name.Value) + // Check that Name is an Identifier + ident, ok := stmt.Name.(*parser.Identifier) + if !ok { + t.Fatalf("expected Identifier for Name, got %T", stmt.Name) + } + + if ident.Value != tt.expectedIdentifier { + t.Errorf("expected identifier %s, got %s", tt.expectedIdentifier, ident.Value) } if tt.isExpression { @@ -60,6 +66,74 @@ func TestAssignStatements(t *testing.T) { } } +func TestMemberAccessAssignment(t *testing.T) { + tests := []struct { + input string + expected string + description string + }{ + {"table.key = 42", "table.key = 42.00", "dot notation assignment"}, + {"arr[1] = \"hello\"", "arr[1.00] = \"hello\"", "bracket notation assignment"}, + {"obj.nested.deep = true", "obj.nested.deep = true", "chained dot assignment"}, + {"matrix[1][2] = 3.14", "matrix[1.00][2.00] = 3.14", "chained bracket assignment"}, + {"data[\"key\"].value = nil", "data[\"key\"].value = nil", "mixed access assignment"}, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + l := parser.NewLexer(tt.input) + p := parser.NewParser(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + if len(program.Statements) != 1 { + t.Fatalf("expected 1 statement, got %d", len(program.Statements)) + } + + stmt, ok := program.Statements[0].(*parser.AssignStatement) + if !ok { + t.Fatalf("expected AssignStatement, got %T", program.Statements[0]) + } + + if stmt.String() != tt.expected { + t.Errorf("expected %s, got %s", tt.expected, stmt.String()) + } + }) + } +} + +func TestMemberAccessExpressions(t *testing.T) { + tests := []struct { + input string + expected string + description string + }{ + {"echo table.key", "echo table.key", "dot access"}, + {"echo arr[1]", "echo arr[1.00]", "bracket access"}, + {"echo obj.nested.deep", "echo obj.nested.deep", "chained dot access"}, + {"echo matrix[1][2]", "echo matrix[1.00][2.00]", "chained bracket access"}, + {"echo data[\"key\"].value", "echo data[\"key\"].value", "mixed access"}, + {"echo table.key + arr[0]", "echo (table.key + arr[0.00])", "member access in expression"}, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + l := parser.NewLexer(tt.input) + p := parser.NewParser(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + if len(program.Statements) != 1 { + t.Fatalf("expected 1 statement, got %d", len(program.Statements)) + } + + if program.Statements[0].String() != tt.expected { + t.Errorf("expected %s, got %s", tt.expected, program.Statements[0].String()) + } + }) + } +} + func TestTableAssignments(t *testing.T) { tests := []struct { input string @@ -89,8 +163,14 @@ func TestTableAssignments(t *testing.T) { t.Fatalf("expected AssignStatement, got %T", program.Statements[0]) } - if stmt.Name.Value != tt.identifier { - t.Errorf("expected identifier %s, got %s", tt.identifier, stmt.Name.Value) + // Check that Name is an Identifier + ident, ok := stmt.Name.(*parser.Identifier) + if !ok { + t.Fatalf("expected Identifier for Name, got %T", stmt.Name) + } + + if ident.Value != tt.identifier { + t.Errorf("expected identifier %s, got %s", tt.identifier, ident.Value) } table, ok := stmt.Value.(*parser.TableLiteral) diff --git a/parser/tests/conditionals_test.go b/parser/tests/conditionals_test.go index 76196ca..6fdcfb8 100644 --- a/parser/tests/conditionals_test.go +++ b/parser/tests/conditionals_test.go @@ -37,8 +37,14 @@ end` t.Fatalf("expected AssignStatement in body, got %T", stmt.Body[0]) } - if bodyStmt.Name.Value != "x" { - t.Errorf("expected identifier 'x', got %s", bodyStmt.Name.Value) + // Check that Name is an Identifier + ident, ok := bodyStmt.Name.(*parser.Identifier) + if !ok { + t.Fatalf("expected Identifier for Name, got %T", bodyStmt.Name) + } + + if ident.Value != "x" { + t.Errorf("expected identifier 'x', got %s", ident.Value) } } @@ -78,8 +84,14 @@ end` t.Fatalf("expected AssignStatement in else, got %T", stmt.Else[0]) } - if elseStmt.Name.Value != "x" { - t.Errorf("expected identifier 'x', got %s", elseStmt.Name.Value) + // Check that Name is an Identifier + ident, ok := elseStmt.Name.(*parser.Identifier) + if !ok { + t.Fatalf("expected Identifier for Name, got %T", elseStmt.Name) + } + + if ident.Value != "x" { + t.Errorf("expected identifier 'x', got %s", ident.Value) } } @@ -122,6 +134,63 @@ end` } } +func TestConditionalWithMemberAccess(t *testing.T) { + input := `if table.flag then + arr[1] = "updated" + obj.nested.count = obj.nested.count + 1 +end` + + l := parser.NewLexer(input) + p := parser.NewParser(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + if len(program.Statements) != 1 { + t.Fatalf("expected 1 statement, got %d", len(program.Statements)) + } + + stmt, ok := program.Statements[0].(*parser.IfStatement) + if !ok { + t.Fatalf("expected IfStatement, got %T", program.Statements[0]) + } + + // Check condition is dot expression + dotExpr, ok := stmt.Condition.(*parser.DotExpression) + if !ok { + t.Fatalf("expected DotExpression condition, got %T", stmt.Condition) + } + + if dotExpr.Key != "flag" { + t.Errorf("expected key 'flag', got %s", dotExpr.Key) + } + + if len(stmt.Body) != 2 { + t.Fatalf("expected 2 body statements, got %d", len(stmt.Body)) + } + + // First assignment: arr[1] = "updated" + assign1, ok := stmt.Body[0].(*parser.AssignStatement) + if !ok { + t.Fatalf("expected AssignStatement, got %T", stmt.Body[0]) + } + + _, ok = assign1.Name.(*parser.IndexExpression) + if !ok { + t.Fatalf("expected IndexExpression for assignment target, got %T", assign1.Name) + } + + // Second assignment: obj.nested.count = obj.nested.count + 1 + assign2, ok := stmt.Body[1].(*parser.AssignStatement) + if !ok { + t.Fatalf("expected AssignStatement, got %T", stmt.Body[1]) + } + + _, ok = assign2.Name.(*parser.DotExpression) + if !ok { + t.Fatalf("expected DotExpression for assignment target, got %T", assign2.Name) + } +} + func TestConditionalExpressions(t *testing.T) { input := `if 1 + 2 then x = 3 * 4 diff --git a/parser/tests/parser_test.go b/parser/tests/parser_test.go index 5fb894c..fd5a355 100644 --- a/parser/tests/parser_test.go +++ b/parser/tests/parser_test.go @@ -27,8 +27,13 @@ z = true + false` t.Fatalf("statement %d: expected AssignStatement, got %T", i, program.Statements[i]) } - if stmt.Name.Value != expectedIdent { - t.Errorf("statement %d: expected identifier %s, got %s", i, expectedIdent, stmt.Name.Value) + ident, ok := stmt.Name.(*parser.Identifier) + if !ok { + t.Fatalf("statement %d: expected Identifier for Name, got %T", i, stmt.Name) + } + + if ident.Value != expectedIdent { + t.Errorf("statement %d: expected identifier %s, got %s", i, expectedIdent, ident.Value) } } } @@ -57,8 +62,12 @@ arr = {a = 1, b = 2}` if !ok { t.Fatalf("statement 0: expected AssignStatement, got %T", program.Statements[0]) } - if stmt1.Name.Value != "x" { - t.Errorf("expected identifier 'x', got %s", stmt1.Name.Value) + ident1, ok := stmt1.Name.(*parser.Identifier) + if !ok { + t.Fatalf("expected Identifier for Name, got %T", stmt1.Name) + } + if ident1.Value != "x" { + t.Errorf("expected identifier 'x', got %s", ident1.Value) } // Second statement: if statement @@ -84,6 +93,63 @@ arr = {a = 1, b = 2}` } } +func TestMemberAccessProgram(t *testing.T) { + input := `table = {key = "value", nested = {inner = 42}} +table.key = "new value" +table["key"] = "bracket syntax" +table.nested.inner = 100 +echo table[table.key]` + + l := parser.NewLexer(input) + p := parser.NewParser(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + if len(program.Statements) != 5 { + t.Fatalf("expected 5 statements, got %d", len(program.Statements)) + } + + // Second statement: dot assignment + stmt2, ok := program.Statements[1].(*parser.AssignStatement) + if !ok { + t.Fatalf("statement 1: expected AssignStatement, got %T", program.Statements[1]) + } + _, ok = stmt2.Name.(*parser.DotExpression) + if !ok { + t.Fatalf("expected DotExpression for assignment target, got %T", stmt2.Name) + } + + // Third statement: bracket assignment + stmt3, ok := program.Statements[2].(*parser.AssignStatement) + if !ok { + t.Fatalf("statement 2: expected AssignStatement, got %T", program.Statements[2]) + } + _, ok = stmt3.Name.(*parser.IndexExpression) + if !ok { + t.Fatalf("expected IndexExpression for assignment target, got %T", stmt3.Name) + } + + // Fourth statement: chained dot assignment + stmt4, ok := program.Statements[3].(*parser.AssignStatement) + if !ok { + t.Fatalf("statement 3: expected AssignStatement, got %T", program.Statements[3]) + } + _, ok = stmt4.Name.(*parser.DotExpression) + if !ok { + t.Fatalf("expected DotExpression for assignment target, got %T", stmt4.Name) + } + + // Fifth statement: echo with nested access + stmt5, ok := program.Statements[4].(*parser.EchoStatement) + if !ok { + t.Fatalf("statement 4: expected EchoStatement, got %T", program.Statements[4]) + } + _, ok = stmt5.Value.(*parser.IndexExpression) + if !ok { + t.Fatalf("expected IndexExpression in echo, got %T", stmt5.Value) + } +} + func TestNestedConditionals(t *testing.T) { input := `if a then if b then diff --git a/parser/token.go b/parser/token.go index 87a1fc0..a65f523 100644 --- a/parser/token.go +++ b/parser/token.go @@ -18,13 +18,16 @@ const ( MINUS // - STAR // * SLASH // / + DOT // . // Delimiters - LPAREN // ( - RPAREN // ) - LBRACE // { - RBRACE // } - COMMA // , + LPAREN // ( + RPAREN // ) + LBRACE // { + RBRACE // } + LBRACKET // [ + RBRACKET // ] + COMMA // , // Keywords VAR @@ -56,16 +59,19 @@ const ( LOWEST SUM // + PRODUCT // * + MEMBER // table[key], table.key PREFIX // -x, !x CALL // function() ) // precedences maps token types to their precedence levels var precedences = map[TokenType]Precedence{ - PLUS: SUM, - MINUS: SUM, - SLASH: PRODUCT, - STAR: PRODUCT, + PLUS: SUM, + MINUS: SUM, + SLASH: PRODUCT, + STAR: PRODUCT, + DOT: MEMBER, + LBRACKET: MEMBER, } // lookupIdent checks if an identifier is a keyword