From 6c7802d40412df58d393c9dc5829e0d7b7fa7331 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 13 Jun 2017 08:53:33 -0700 Subject: [PATCH] zclsyntax: Parsing of ForExpr The ForExpr is essentially a list/map comprehension, allowing projecting one expression into another. From a syntactic standpoint it's the most complex structure we've dealt with so far, with many separate parts. The tests introduced here are not exhaustive but illustrate that the basic mechanism is working. --- zcl/zclsyntax/parser.go | 227 ++++++++++++++++++++++++++++++++++- zcl/zclsyntax/parser_test.go | 193 +++++++++++++++++++++++++++++ zcl/zclsyntax/variables.go | 5 + 3 files changed, 420 insertions(+), 5 deletions(-) diff --git a/zcl/zclsyntax/parser.go b/zcl/zclsyntax/parser.go index f6b54f4..306b47a 100644 --- a/zcl/zclsyntax/parser.go +++ b/zcl/zclsyntax/parser.go @@ -849,11 +849,15 @@ func (p *parser) parseTupleCons() (Expression, zcl.Diagnostics) { panic("parseTupleCons called without peeker pointing to open bracket") } - var close Token - p.PushIncludeNewlines(false) defer p.PopIncludeNewlines() + if forKeyword.TokenMatches(p.Peek()) { + return p.finishParsingForExpr(open) + } + + var close Token + var diags zcl.Diagnostics var exprs []Expression @@ -915,14 +919,18 @@ func (p *parser) parseObjectCons() (Expression, zcl.Diagnostics) { panic("parseObjectCons called without peeker pointing to open brace") } + p.PushIncludeNewlines(true) + defer p.PopIncludeNewlines() + + if forKeyword.TokenMatches(p.Peek()) { + return p.finishParsingForExpr(open) + } + var close Token var diags zcl.Diagnostics var items []ObjectConsItem - p.PushIncludeNewlines(true) - defer p.PopIncludeNewlines() - for { next := p.Peek() if next.Type == TokenNewline { @@ -1037,6 +1045,215 @@ func (p *parser) parseObjectCons() (Expression, zcl.Diagnostics) { }, diags } +func (p *parser) finishParsingForExpr(open Token) (Expression, zcl.Diagnostics) { + introducer := p.Read() + if !forKeyword.TokenMatches(introducer) { + // Should never happen if callers are behaving + panic("finishParsingForExpr called without peeker pointing to 'for' identifier") + } + + var makeObj bool + var closeType TokenType + switch open.Type { + case TokenOBrace: + makeObj = true + closeType = TokenCBrace + case TokenOBrack: + makeObj = false // making a tuple + closeType = TokenCBrack + default: + // Should never happen if callers are behaving + panic("finishParsingForExpr called with invalid open token") + } + + var diags zcl.Diagnostics + var keyName, valName string + + if p.Peek().Type != TokenIdent { + if !p.recovery { + diags = append(diags, &zcl.Diagnostic{ + Severity: zcl.DiagError, + Summary: "Invalid 'for' expression", + Detail: "For expression requires variable name after 'for'.", + Subject: p.Peek().Range.Ptr(), + Context: zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(), + }) + } + close := p.recover(closeType) + return &LiteralValueExpr{ + Val: cty.DynamicVal, + SrcRange: zcl.RangeBetween(open.Range, close.Range), + }, diags + } + + valName = string(p.Read().Bytes) + + if p.Peek().Type == TokenComma { + // What we just read was actually the key, then. + keyName = valName + p.Read() // eat comma + + if p.Peek().Type != TokenIdent { + if !p.recovery { + diags = append(diags, &zcl.Diagnostic{ + Severity: zcl.DiagError, + Summary: "Invalid 'for' expression", + Detail: "For expression requires value variable name after comma.", + Subject: p.Peek().Range.Ptr(), + Context: zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(), + }) + } + close := p.recover(closeType) + return &LiteralValueExpr{ + Val: cty.DynamicVal, + SrcRange: zcl.RangeBetween(open.Range, close.Range), + }, diags + } + + valName = string(p.Read().Bytes) + } + + if !inKeyword.TokenMatches(p.Peek()) { + if !p.recovery { + diags = append(diags, &zcl.Diagnostic{ + Severity: zcl.DiagError, + Summary: "Invalid 'for' expression", + Detail: "For expression requires 'in' keyword after names.", + Subject: p.Peek().Range.Ptr(), + Context: zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(), + }) + } + close := p.recover(closeType) + return &LiteralValueExpr{ + Val: cty.DynamicVal, + SrcRange: zcl.RangeBetween(open.Range, close.Range), + }, diags + } + p.Read() // eat 'in' keyword + + collExpr, collDiags := p.ParseExpression() + diags = append(diags, collDiags...) + if p.recovery && collDiags.HasErrors() { + close := p.recover(closeType) + return &LiteralValueExpr{ + Val: cty.DynamicVal, + SrcRange: zcl.RangeBetween(open.Range, close.Range), + }, diags + } + + if p.Peek().Type != TokenColon { + if !p.recovery { + diags = append(diags, &zcl.Diagnostic{ + Severity: zcl.DiagError, + Summary: "Invalid 'for' expression", + Detail: "For expression requires colon after collection expression.", + Subject: p.Peek().Range.Ptr(), + Context: zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(), + }) + } + close := p.recover(closeType) + return &LiteralValueExpr{ + Val: cty.DynamicVal, + SrcRange: zcl.RangeBetween(open.Range, close.Range), + }, diags + } + p.Read() // eat colon + + var keyExpr, valExpr Expression + var keyDiags, valDiags zcl.Diagnostics + valExpr, valDiags = p.ParseExpression() + if p.Peek().Type == TokenFatArrow { + // What we just parsed was actually keyExpr + p.Read() // eat the fat arrow + keyExpr, keyDiags = valExpr, valDiags + + valExpr, valDiags = p.ParseExpression() + } + diags = append(diags, keyDiags...) + diags = append(diags, valDiags...) + if p.recovery && (keyDiags.HasErrors() || valDiags.HasErrors()) { + close := p.recover(closeType) + return &LiteralValueExpr{ + Val: cty.DynamicVal, + SrcRange: zcl.RangeBetween(open.Range, close.Range), + }, diags + } + + group := false + var ellipsis Token + if p.Peek().Type == TokenEllipsis { + ellipsis = p.Read() + group = true + } + + var condExpr Expression + var condDiags zcl.Diagnostics + if ifKeyword.TokenMatches(p.Peek()) { + p.Read() // eat "if" + condExpr, condDiags = p.ParseExpression() + diags = append(diags, condDiags...) + if p.recovery && condDiags.HasErrors() { + close := p.recover(p.oppositeBracket(open.Type)) + return &LiteralValueExpr{ + Val: cty.DynamicVal, + SrcRange: zcl.RangeBetween(open.Range, close.Range), + }, diags + } + } + + var close Token + if p.Peek().Type == closeType { + close = p.Read() + } else { + if !p.recovery { + diags = append(diags, &zcl.Diagnostic{ + Severity: zcl.DiagError, + Summary: "Invalid 'for' expression", + Detail: "Extra characters after the end of the 'for' expression.", + Subject: p.Peek().Range.Ptr(), + Context: zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(), + }) + } + close = p.recover(closeType) + } + + if !makeObj { + if keyExpr != nil { + diags = append(diags, &zcl.Diagnostic{ + Severity: zcl.DiagError, + Summary: "Invalid 'for' expression", + Detail: "Key expression is not valid when building a tuple.", + Subject: keyExpr.Range().Ptr(), + Context: zcl.RangeBetween(open.Range, close.Range).Ptr(), + }) + } + + if group { + diags = append(diags, &zcl.Diagnostic{ + Severity: zcl.DiagError, + Summary: "Invalid 'for' expression", + Detail: "Grouping ellipsis (...) cannot be used when building a tuple.", + Subject: &ellipsis.Range, + Context: zcl.RangeBetween(open.Range, close.Range).Ptr(), + }) + } + } + + return &ForExpr{ + KeyVar: keyName, + ValVar: valName, + CollExpr: collExpr, + KeyExpr: keyExpr, + ValExpr: valExpr, + CondExpr: condExpr, + Group: group, + + SrcRange: zcl.RangeBetween(open.Range, close.Range), + OpenRange: open.Range, + CloseRange: close.Range, + }, diags +} + func (p *parser) ParseTemplate() (Expression, zcl.Diagnostics) { startRange := p.NextRange() parts, unwrap, diags := p.parseTemplateParts(TokenEOF) diff --git a/zcl/zclsyntax/parser_test.go b/zcl/zclsyntax/parser_test.go index 7113c51..5fc00c6 100644 --- a/zcl/zclsyntax/parser_test.go +++ b/zcl/zclsyntax/parser_test.go @@ -566,6 +566,199 @@ block "valid" {} }, }, + { + "a = [for k, v in foo: v if true]\n", + 0, + &Body{ + Attributes: Attributes{ + "a": { + Name: "a", + Expr: &ForExpr{ + KeyVar: "k", + ValVar: "v", + + CollExpr: &ScopeTraversalExpr{ + Traversal: zcl.Traversal{ + zcl.TraverseRoot{ + Name: "foo", + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 18, Byte: 17}, + End: zcl.Pos{Line: 1, Column: 21, Byte: 20}, + }, + }, + }, + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 18, Byte: 17}, + End: zcl.Pos{Line: 1, Column: 21, Byte: 20}, + }, + }, + ValExpr: &ScopeTraversalExpr{ + Traversal: zcl.Traversal{ + zcl.TraverseRoot{ + Name: "v", + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 23, Byte: 22}, + End: zcl.Pos{Line: 1, Column: 24, Byte: 23}, + }, + }, + }, + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 23, Byte: 22}, + End: zcl.Pos{Line: 1, Column: 24, Byte: 23}, + }, + }, + CondExpr: &LiteralValueExpr{ + Val: cty.True, + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 28, Byte: 27}, + End: zcl.Pos{Line: 1, Column: 32, Byte: 31}, + }, + }, + + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 5, Byte: 4}, + End: zcl.Pos{Line: 1, Column: 33, Byte: 32}, + }, + OpenRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 5, Byte: 4}, + End: zcl.Pos{Line: 1, Column: 6, Byte: 5}, + }, + CloseRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 32, Byte: 31}, + End: zcl.Pos{Line: 1, Column: 33, Byte: 32}, + }, + }, + + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: zcl.Pos{Line: 1, Column: 33, Byte: 32}, + }, + NameRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: zcl.Pos{Line: 1, Column: 2, Byte: 1}, + }, + EqualsRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 3, Byte: 2}, + End: zcl.Pos{Line: 1, Column: 4, Byte: 3}, + }, + }, + }, + Blocks: Blocks{}, + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: zcl.Pos{Line: 2, Column: 1, Byte: 33}, + }, + EndRange: zcl.Range{ + Start: zcl.Pos{Line: 2, Column: 1, Byte: 33}, + End: zcl.Pos{Line: 2, Column: 1, Byte: 33}, + }, + }, + }, + { + "a = [for k, v in foo: k => v... if true]\n", + 2, // can't use => or ... in a tuple for + &Body{ + Attributes: Attributes{ + "a": { + Name: "a", + Expr: &ForExpr{ + KeyVar: "k", + ValVar: "v", + + CollExpr: &ScopeTraversalExpr{ + Traversal: zcl.Traversal{ + zcl.TraverseRoot{ + Name: "foo", + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 18, Byte: 17}, + End: zcl.Pos{Line: 1, Column: 21, Byte: 20}, + }, + }, + }, + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 18, Byte: 17}, + End: zcl.Pos{Line: 1, Column: 21, Byte: 20}, + }, + }, + KeyExpr: &ScopeTraversalExpr{ + Traversal: zcl.Traversal{ + zcl.TraverseRoot{ + Name: "k", + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 23, Byte: 22}, + End: zcl.Pos{Line: 1, Column: 24, Byte: 23}, + }, + }, + }, + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 23, Byte: 22}, + End: zcl.Pos{Line: 1, Column: 24, Byte: 23}, + }, + }, + ValExpr: &ScopeTraversalExpr{ + Traversal: zcl.Traversal{ + zcl.TraverseRoot{ + Name: "v", + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 28, Byte: 27}, + End: zcl.Pos{Line: 1, Column: 29, Byte: 28}, + }, + }, + }, + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 28, Byte: 27}, + End: zcl.Pos{Line: 1, Column: 29, Byte: 28}, + }, + }, + CondExpr: &LiteralValueExpr{ + Val: cty.True, + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 36, Byte: 35}, + End: zcl.Pos{Line: 1, Column: 40, Byte: 39}, + }, + }, + Group: true, + + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 5, Byte: 4}, + End: zcl.Pos{Line: 1, Column: 41, Byte: 40}, + }, + OpenRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 5, Byte: 4}, + End: zcl.Pos{Line: 1, Column: 6, Byte: 5}, + }, + CloseRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 40, Byte: 39}, + End: zcl.Pos{Line: 1, Column: 41, Byte: 40}, + }, + }, + + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: zcl.Pos{Line: 1, Column: 41, Byte: 40}, + }, + NameRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: zcl.Pos{Line: 1, Column: 2, Byte: 1}, + }, + EqualsRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 3, Byte: 2}, + End: zcl.Pos{Line: 1, Column: 4, Byte: 3}, + }, + }, + }, + Blocks: Blocks{}, + SrcRange: zcl.Range{ + Start: zcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: zcl.Pos{Line: 2, Column: 1, Byte: 41}, + }, + EndRange: zcl.Range{ + Start: zcl.Pos{Line: 2, Column: 1, Byte: 41}, + End: zcl.Pos{Line: 2, Column: 1, Byte: 41}, + }, + }, + }, + { ` `, 2, // tabs not allowed, and body item is required here diff --git a/zcl/zclsyntax/variables.go b/zcl/zclsyntax/variables.go index 04de6a0..d18d044 100644 --- a/zcl/zclsyntax/variables.go +++ b/zcl/zclsyntax/variables.go @@ -10,6 +10,11 @@ import ( // expression. func Variables(expr Expression) []zcl.Traversal { var vars []zcl.Traversal + + // TODO: When traversing into ForExpr, filter out references to + // the iterator variables, since they are references into the child + // scope, and thus not interesting to the caller. + VisitAll(expr, func(n Node) zcl.Diagnostics { if ste, ok := n.(*ScopeTraversalExpr); ok { vars = append(vars, ste.Traversal)