zclsyntax: Parsing of ForExpr

The ForExpr is essentially a list/map comprehension, allowing projecting
one expression into another. From a syntactic standpoint it's the most
complex structure we've dealt with so far, with many separate parts.

The tests introduced here are not exhaustive but illustrate that the
basic mechanism is working.

zcl/zclsyntax/parser.go

@@ -849,11 +849,15 @@ func (p *parser) parseTupleCons() (Expression, zcl.Diagnostics) {
 		panic("parseTupleCons called without peeker pointing to open bracket")
 	}
 
-	var close Token
-
 	p.PushIncludeNewlines(false)
 	defer p.PopIncludeNewlines()
 
+	if forKeyword.TokenMatches(p.Peek()) {
+		return p.finishParsingForExpr(open)
+	}
+
+	var close Token
+
 	var diags zcl.Diagnostics
 	var exprs []Expression
 
@@ -915,14 +919,18 @@ func (p *parser) parseObjectCons() (Expression, zcl.Diagnostics) {
 		panic("parseObjectCons called without peeker pointing to open brace")
 	}
 
+	p.PushIncludeNewlines(true)
+	defer p.PopIncludeNewlines()
+
+	if forKeyword.TokenMatches(p.Peek()) {
+		return p.finishParsingForExpr(open)
+	}
+
 	var close Token
 
 	var diags zcl.Diagnostics
 	var items []ObjectConsItem
 
-	p.PushIncludeNewlines(true)
-	defer p.PopIncludeNewlines()
-
 	for {
 		next := p.Peek()
 		if next.Type == TokenNewline {
@@ -1037,6 +1045,215 @@ func (p *parser) parseObjectCons() (Expression, zcl.Diagnostics) {
 	}, diags
 }
 
+func (p *parser) finishParsingForExpr(open Token) (Expression, zcl.Diagnostics) {
+	introducer := p.Read()
+	if !forKeyword.TokenMatches(introducer) {
+		// Should never happen if callers are behaving
+		panic("finishParsingForExpr called without peeker pointing to 'for' identifier")
+	}
+
+	var makeObj bool
+	var closeType TokenType
+	switch open.Type {
+	case TokenOBrace:
+		makeObj = true
+		closeType = TokenCBrace
+	case TokenOBrack:
+		makeObj = false // making a tuple
+		closeType = TokenCBrack
+	default:
+		// Should never happen if callers are behaving
+		panic("finishParsingForExpr called with invalid open token")
+	}
+
+	var diags zcl.Diagnostics
+	var keyName, valName string
+
+	if p.Peek().Type != TokenIdent {
+		if !p.recovery {
+			diags = append(diags, &zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  "Invalid 'for' expression",
+				Detail:   "For expression requires variable name after 'for'.",
+				Subject:  p.Peek().Range.Ptr(),
+				Context:  zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
+			})
+		}
+		close := p.recover(closeType)
+		return &LiteralValueExpr{
+			Val:      cty.DynamicVal,
+			SrcRange: zcl.RangeBetween(open.Range, close.Range),
+		}, diags
+	}
+
+	valName = string(p.Read().Bytes)
+
+	if p.Peek().Type == TokenComma {
+		// What we just read was actually the key, then.
+		keyName = valName
+		p.Read() // eat comma
+
+		if p.Peek().Type != TokenIdent {
+			if !p.recovery {
+				diags = append(diags, &zcl.Diagnostic{
+					Severity: zcl.DiagError,
+					Summary:  "Invalid 'for' expression",
+					Detail:   "For expression requires value variable name after comma.",
+					Subject:  p.Peek().Range.Ptr(),
+					Context:  zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
+				})
+			}
+			close := p.recover(closeType)
+			return &LiteralValueExpr{
+				Val:      cty.DynamicVal,
+				SrcRange: zcl.RangeBetween(open.Range, close.Range),
+			}, diags
+		}
+
+		valName = string(p.Read().Bytes)
+	}
+
+	if !inKeyword.TokenMatches(p.Peek()) {
+		if !p.recovery {
+			diags = append(diags, &zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  "Invalid 'for' expression",
+				Detail:   "For expression requires 'in' keyword after names.",
+				Subject:  p.Peek().Range.Ptr(),
+				Context:  zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
+			})
+		}
+		close := p.recover(closeType)
+		return &LiteralValueExpr{
+			Val:      cty.DynamicVal,
+			SrcRange: zcl.RangeBetween(open.Range, close.Range),
+		}, diags
+	}
+	p.Read() // eat 'in' keyword
+
+	collExpr, collDiags := p.ParseExpression()
+	diags = append(diags, collDiags...)
+	if p.recovery && collDiags.HasErrors() {
+		close := p.recover(closeType)
+		return &LiteralValueExpr{
+			Val:      cty.DynamicVal,
+			SrcRange: zcl.RangeBetween(open.Range, close.Range),
+		}, diags
+	}
+
+	if p.Peek().Type != TokenColon {
+		if !p.recovery {
+			diags = append(diags, &zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  "Invalid 'for' expression",
+				Detail:   "For expression requires colon after collection expression.",
+				Subject:  p.Peek().Range.Ptr(),
+				Context:  zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
+			})
+		}
+		close := p.recover(closeType)
+		return &LiteralValueExpr{
+			Val:      cty.DynamicVal,
+			SrcRange: zcl.RangeBetween(open.Range, close.Range),
+		}, diags
+	}
+	p.Read() // eat colon
+
+	var keyExpr, valExpr Expression
+	var keyDiags, valDiags zcl.Diagnostics
+	valExpr, valDiags = p.ParseExpression()
+	if p.Peek().Type == TokenFatArrow {
+		// What we just parsed was actually keyExpr
+		p.Read() // eat the fat arrow
+		keyExpr, keyDiags = valExpr, valDiags
+
+		valExpr, valDiags = p.ParseExpression()
+	}
+	diags = append(diags, keyDiags...)
+	diags = append(diags, valDiags...)
+	if p.recovery && (keyDiags.HasErrors() || valDiags.HasErrors()) {
+		close := p.recover(closeType)
+		return &LiteralValueExpr{
+			Val:      cty.DynamicVal,
+			SrcRange: zcl.RangeBetween(open.Range, close.Range),
+		}, diags
+	}
+
+	group := false
+	var ellipsis Token
+	if p.Peek().Type == TokenEllipsis {
+		ellipsis = p.Read()
+		group = true
+	}
+
+	var condExpr Expression
+	var condDiags zcl.Diagnostics
+	if ifKeyword.TokenMatches(p.Peek()) {
+		p.Read() // eat "if"
+		condExpr, condDiags = p.ParseExpression()
+		diags = append(diags, condDiags...)
+		if p.recovery && condDiags.HasErrors() {
+			close := p.recover(p.oppositeBracket(open.Type))
+			return &LiteralValueExpr{
+				Val:      cty.DynamicVal,
+				SrcRange: zcl.RangeBetween(open.Range, close.Range),
+			}, diags
+		}
+	}
+
+	var close Token
+	if p.Peek().Type == closeType {
+		close = p.Read()
+	} else {
+		if !p.recovery {
+			diags = append(diags, &zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  "Invalid 'for' expression",
+				Detail:   "Extra characters after the end of the 'for' expression.",
+				Subject:  p.Peek().Range.Ptr(),
+				Context:  zcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
+			})
+		}
+		close = p.recover(closeType)
+	}
+
+	if !makeObj {
+		if keyExpr != nil {
+			diags = append(diags, &zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  "Invalid 'for' expression",
+				Detail:   "Key expression is not valid when building a tuple.",
+				Subject:  keyExpr.Range().Ptr(),
+				Context:  zcl.RangeBetween(open.Range, close.Range).Ptr(),
+			})
+		}
+
+		if group {
+			diags = append(diags, &zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  "Invalid 'for' expression",
+				Detail:   "Grouping ellipsis (...) cannot be used when building a tuple.",
+				Subject:  &ellipsis.Range,
+				Context:  zcl.RangeBetween(open.Range, close.Range).Ptr(),
+			})
+		}
+	}
+
+	return &ForExpr{
+		KeyVar:   keyName,
+		ValVar:   valName,
+		CollExpr: collExpr,
+		KeyExpr:  keyExpr,
+		ValExpr:  valExpr,
+		CondExpr: condExpr,
+		Group:    group,
+
+		SrcRange:   zcl.RangeBetween(open.Range, close.Range),
+		OpenRange:  open.Range,
+		CloseRange: close.Range,
+	}, diags
+}
+
 func (p *parser) ParseTemplate() (Expression, zcl.Diagnostics) {
 	startRange := p.NextRange()
 	parts, unwrap, diags := p.parseTemplateParts(TokenEOF)

zcl/zclsyntax/parser_test.go

@@ -566,6 +566,199 @@ block "valid" {}
 			},
 		},
 
+		{
+			"a = [for k, v in foo: v if true]\n",
+			0,
+			&Body{
+				Attributes: Attributes{
+					"a": {
+						Name: "a",
+						Expr: &ForExpr{
+							KeyVar: "k",
+							ValVar: "v",
+
+							CollExpr: &ScopeTraversalExpr{
+								Traversal: zcl.Traversal{
+									zcl.TraverseRoot{
+										Name: "foo",
+										SrcRange: zcl.Range{
+											Start: zcl.Pos{Line: 1, Column: 18, Byte: 17},
+											End:   zcl.Pos{Line: 1, Column: 21, Byte: 20},
+										},
+									},
+								},
+								SrcRange: zcl.Range{
+									Start: zcl.Pos{Line: 1, Column: 18, Byte: 17},
+									End:   zcl.Pos{Line: 1, Column: 21, Byte: 20},
+								},
+							},
+							ValExpr: &ScopeTraversalExpr{
+								Traversal: zcl.Traversal{
+									zcl.TraverseRoot{
+										Name: "v",
+										SrcRange: zcl.Range{
+											Start: zcl.Pos{Line: 1, Column: 23, Byte: 22},
+											End:   zcl.Pos{Line: 1, Column: 24, Byte: 23},
+										},
+									},
+								},
+								SrcRange: zcl.Range{
+									Start: zcl.Pos{Line: 1, Column: 23, Byte: 22},
+									End:   zcl.Pos{Line: 1, Column: 24, Byte: 23},
+								},
+							},
+							CondExpr: &LiteralValueExpr{
+								Val: cty.True,
+								SrcRange: zcl.Range{
+									Start: zcl.Pos{Line: 1, Column: 28, Byte: 27},
+									End:   zcl.Pos{Line: 1, Column: 32, Byte: 31},
+								},
+							},
+
+							SrcRange: zcl.Range{
+								Start: zcl.Pos{Line: 1, Column: 5, Byte: 4},
+								End:   zcl.Pos{Line: 1, Column: 33, Byte: 32},
+							},
+							OpenRange: zcl.Range{
+								Start: zcl.Pos{Line: 1, Column: 5, Byte: 4},
+								End:   zcl.Pos{Line: 1, Column: 6, Byte: 5},
+							},
+							CloseRange: zcl.Range{
+								Start: zcl.Pos{Line: 1, Column: 32, Byte: 31},
+								End:   zcl.Pos{Line: 1, Column: 33, Byte: 32},
+							},
+						},
+
+						SrcRange: zcl.Range{
+							Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+							End:   zcl.Pos{Line: 1, Column: 33, Byte: 32},
+						},
+						NameRange: zcl.Range{
+							Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+							End:   zcl.Pos{Line: 1, Column: 2, Byte: 1},
+						},
+						EqualsRange: zcl.Range{
+							Start: zcl.Pos{Line: 1, Column: 3, Byte: 2},
+							End:   zcl.Pos{Line: 1, Column: 4, Byte: 3},
+						},
+					},
+				},
+				Blocks: Blocks{},
+				SrcRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 2, Column: 1, Byte: 33},
+				},
+				EndRange: zcl.Range{
+					Start: zcl.Pos{Line: 2, Column: 1, Byte: 33},
+					End:   zcl.Pos{Line: 2, Column: 1, Byte: 33},
+				},
+			},
+		},
+		{
+			"a = [for k, v in foo: k => v... if true]\n",
+			2, // can't use => or ... in a tuple for
+			&Body{
+				Attributes: Attributes{
+					"a": {
+						Name: "a",
+						Expr: &ForExpr{
+							KeyVar: "k",
+							ValVar: "v",
+
+							CollExpr: &ScopeTraversalExpr{
+								Traversal: zcl.Traversal{
+									zcl.TraverseRoot{
+										Name: "foo",
+										SrcRange: zcl.Range{
+											Start: zcl.Pos{Line: 1, Column: 18, Byte: 17},
+											End:   zcl.Pos{Line: 1, Column: 21, Byte: 20},
+										},
+									},
+								},
+								SrcRange: zcl.Range{
+									Start: zcl.Pos{Line: 1, Column: 18, Byte: 17},
+									End:   zcl.Pos{Line: 1, Column: 21, Byte: 20},
+								},
+							},
+							KeyExpr: &ScopeTraversalExpr{
+								Traversal: zcl.Traversal{
+									zcl.TraverseRoot{
+										Name: "k",
+										SrcRange: zcl.Range{
+											Start: zcl.Pos{Line: 1, Column: 23, Byte: 22},
+											End:   zcl.Pos{Line: 1, Column: 24, Byte: 23},
+										},
+									},
+								},
+								SrcRange: zcl.Range{
+									Start: zcl.Pos{Line: 1, Column: 23, Byte: 22},
+									End:   zcl.Pos{Line: 1, Column: 24, Byte: 23},
+								},
+							},
+							ValExpr: &ScopeTraversalExpr{
+								Traversal: zcl.Traversal{
+									zcl.TraverseRoot{
+										Name: "v",
+										SrcRange: zcl.Range{
+											Start: zcl.Pos{Line: 1, Column: 28, Byte: 27},
+											End:   zcl.Pos{Line: 1, Column: 29, Byte: 28},
+										},
+									},
+								},
+								SrcRange: zcl.Range{
+									Start: zcl.Pos{Line: 1, Column: 28, Byte: 27},
+									End:   zcl.Pos{Line: 1, Column: 29, Byte: 28},
+								},
+							},
+							CondExpr: &LiteralValueExpr{
+								Val: cty.True,
+								SrcRange: zcl.Range{
+									Start: zcl.Pos{Line: 1, Column: 36, Byte: 35},
+									End:   zcl.Pos{Line: 1, Column: 40, Byte: 39},
+								},
+							},
+							Group: true,
+
+							SrcRange: zcl.Range{
+								Start: zcl.Pos{Line: 1, Column: 5, Byte: 4},
+								End:   zcl.Pos{Line: 1, Column: 41, Byte: 40},
+							},
+							OpenRange: zcl.Range{
+								Start: zcl.Pos{Line: 1, Column: 5, Byte: 4},
+								End:   zcl.Pos{Line: 1, Column: 6, Byte: 5},
+							},
+							CloseRange: zcl.Range{
+								Start: zcl.Pos{Line: 1, Column: 40, Byte: 39},
+								End:   zcl.Pos{Line: 1, Column: 41, Byte: 40},
+							},
+						},
+
+						SrcRange: zcl.Range{
+							Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+							End:   zcl.Pos{Line: 1, Column: 41, Byte: 40},
+						},
+						NameRange: zcl.Range{
+							Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+							End:   zcl.Pos{Line: 1, Column: 2, Byte: 1},
+						},
+						EqualsRange: zcl.Range{
+							Start: zcl.Pos{Line: 1, Column: 3, Byte: 2},
+							End:   zcl.Pos{Line: 1, Column: 4, Byte: 3},
+						},
+					},
+				},
+				Blocks: Blocks{},
+				SrcRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 2, Column: 1, Byte: 41},
+				},
+				EndRange: zcl.Range{
+					Start: zcl.Pos{Line: 2, Column: 1, Byte: 41},
+					End:   zcl.Pos{Line: 2, Column: 1, Byte: 41},
+				},
+			},
+		},
+
 		{
 			`	`,
 			2, // tabs not allowed, and body item is required here

zcl/zclsyntax/variables.go

@@ -10,6 +10,11 @@ import (
 // expression.
 func Variables(expr Expression) []zcl.Traversal {
 	var vars []zcl.Traversal
+
+	// TODO: When traversing into ForExpr, filter out references to
+	// the iterator variables, since they are references into the child
+	// scope, and thus not interesting to the caller.
+
 	VisitAll(expr, func(n Node) zcl.Diagnostics {
 		if ste, ok := n.(*ScopeTraversalExpr); ok {
 			vars = append(vars, ste.Traversal)