hcl: Annotate diagnostics with expression EvalContext

When we're evaluating expressions, we may end up evaluating the same
source-level expression a number of times in different contexts, such as
in a 'for' expression, where each one may produce a different set of
diagnostic messages.

Now we'll attach the EvalContext to each expression diagnostic so that
a diagnostic renderer can potentially show additional information to help
distinguish the different iterations in rendered diagnostics.

ext/dynblock/expand_spec.go

@@ -47,6 +47,7 @@ func (b *expandBody) decodeSpec(blockS *hcl.BlockHeaderSchema, rawSpec *hcl.Bloc
 			Summary:     "Invalid dynamic for_each value",
 			Detail:      fmt.Sprintf("Cannot use a value of type %s in for_each. An iterable collection is required.", eachVal.Type()),
 			Subject:     eachAttr.Expr.Range().Ptr(),
+			EvalContext: b.forEachCtx,
 		})
 		return nil, diags
 	}
@@ -56,6 +57,7 @@ func (b *expandBody) decodeSpec(blockS *hcl.BlockHeaderSchema, rawSpec *hcl.Bloc
 			Summary:     "Invalid dynamic for_each value",
 			Detail:      "Cannot use a null value in for_each.",
 			Subject:     eachAttr.Expr.Range().Ptr(),
+			EvalContext: b.forEachCtx,
 		})
 		return nil, diags
 	}
@@ -163,6 +165,7 @@ func (s *expandSpec) newBlock(i *iteration, ctx *hcl.EvalContext) (*hcl.Block, h
 				Summary:     "Invalid dynamic block label",
 				Detail:      fmt.Sprintf("Cannot use this value as a dynamic block label: %s.", convErr),
 				Subject:     labelExpr.Range().Ptr(),
+				EvalContext: lCtx,
 			})
 			return nil, diags
 		}
@@ -172,6 +175,7 @@ func (s *expandSpec) newBlock(i *iteration, ctx *hcl.EvalContext) (*hcl.Block, h
 				Summary:     "Invalid dynamic block label",
 				Detail:      "Cannot use a null value as a dynamic block label.",
 				Subject:     labelExpr.Range().Ptr(),
+				EvalContext: lCtx,
 			})
 			return nil, diags
 		}
@@ -181,6 +185,7 @@ func (s *expandSpec) newBlock(i *iteration, ctx *hcl.EvalContext) (*hcl.Block, h
 				Summary:     "Invalid dynamic block label",
 				Detail:      "This value is not yet known. Dynamic block labels must be immediately-known values.",
 				Subject:     labelExpr.Range().Ptr(),
+				EvalContext: lCtx,
 			})
 			return nil, diags
 		}

hcl/diagnostic.go

@@ -26,14 +26,43 @@ const (
 type Diagnostic struct {
 	Severity DiagnosticSeverity
 
-	// Summary and detail contain the English-language description of the
+	// Summary and Detail contain the English-language description of the
 	// problem. Summary is a terse description of the general problem and
 	// detail is a more elaborate, often-multi-sentence description of
 	// the probem and what might be done to solve it.
 	Summary string
 	Detail  string
+
+	// Subject and Context are both source ranges relating to the diagnostic.
+	//
+	// Subject is a tight range referring to exactly the construct that
+	// is problematic, while Context is an optional broader range (which should
+	// fully contain Subject) that ought to be shown around Subject when
+	// generating isolated source-code snippets in diagnostic messages.
+	// If Context is nil, the Subject is also the Context.
+	//
+	// Some diagnostics have no source ranges at all. If Context is set then
+	// Subject should always also be set.
 	Subject *Range
 	Context *Range
+
+	// For diagnostics that occur when evaluating an expression, EvalContext
+	// may point to the EvalContext that was active when evaluating that
+	// expression, which may allow for the inclusion of additional useful
+	// information when rendering a diagnostic message to the user.
+	//
+	// It is not always possible to select a single EvalContext for a
+	// diagnostic, and so in some cases this field may be nil even when an
+	// expression causes a problem. Therefore it is not valid to use the
+	// nil-ness of this field to definitively decide whether a diagnostic
+	// relates to an expression.
+	//
+	// EvalContexts form a tree, so the given EvalContext may refer to a parent
+	// which in turn refers to another parent, etc. For a full picture of all
+	// of the active variables and functions the caller must walk up this
+	// chain, preferring definitions that are "closer" to the expression in
+	// case of colliding names.
+	EvalContext *EvalContext
 }
 
 // Diagnostics is a list of Diagnostic instances.

hcl/hclsyntax/diagnostics.go

@@ -0,0 +1,22 @@
+package hclsyntax
+
+import (
+	"github.com/hashicorp/hcl2/hcl"
+)
+
+// setDiagEvalContext is an internal helper that will impose a particular
+// EvalContext on a set of diagnostics in-place, for any diagnostic that
+// does not already have an EvalContext set.
+//
+// We generally expect diagnostics to be immutable, but this is safe to use
+// on any Diagnostics where none of the contained Diagnostic objects have yet
+// been seen by a caller. Its purpose is to apply additional context to a
+// set of diagnostics produced by a "deeper" component as the stack unwinds
+// during expression evaluation.
+func setDiagEvalContext(diags hcl.Diagnostics, ctx *hcl.EvalContext) {
+	for _, diag := range diags {
+		if diag.EvalContext == nil {
+			diag.EvalContext = ctx
+		}
+	}
+}

hcl/hclsyntax/expression.go

@@ -105,7 +105,9 @@ func (e *ScopeTraversalExpr) walkChildNodes(w internalWalkFunc) {
 }
 
 func (e *ScopeTraversalExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
-	return e.Traversal.TraverseAbs(ctx)
+	val, diags := e.Traversal.TraverseAbs(ctx)
+	setDiagEvalContext(diags, ctx)
+	return val, diags
 }
 
 func (e *ScopeTraversalExpr) Range() hcl.Range {
@@ -136,6 +138,7 @@ func (e *RelativeTraversalExpr) walkChildNodes(w internalWalkFunc) {
 func (e *RelativeTraversalExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 	src, diags := e.Source.Value(ctx)
 	ret, travDiags := e.Traversal.TraverseRel(src)
+	setDiagEvalContext(travDiags, ctx)
 	diags = append(diags, travDiags...)
 	return ret, diags
 }
@@ -211,6 +214,7 @@ func (e *FunctionCallExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 					Summary:     "Function calls not allowed",
 					Detail:      "Functions may not be called here.",
 					Subject:     e.Range().Ptr(),
+					EvalContext: ctx,
 				},
 			}
 		}
@@ -231,6 +235,7 @@ func (e *FunctionCallExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 				Detail:      fmt.Sprintf("There is no function named %q.%s", e.Name, suggestion),
 				Subject:     &e.NameRange,
 				Context:     e.Range().Ptr(),
+				EvalContext: ctx,
 			},
 		}
 	}
@@ -260,6 +265,7 @@ func (e *FunctionCallExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 					Detail:      "The expanding argument (indicated by ...) must not be null.",
 					Context:     expandExpr.Range().Ptr(),
 					Subject:     e.Range().Ptr(),
+					EvalContext: ctx,
 				})
 				return cty.DynamicVal, diags
 			}
@@ -285,6 +291,7 @@ func (e *FunctionCallExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 				Detail:      "The expanding argument (indicated by ...) must be of a tuple, list, or set type.",
 				Context:     expandExpr.Range().Ptr(),
 				Subject:     e.Range().Ptr(),
+				EvalContext: ctx,
 			})
 			return cty.DynamicVal, diags
 		}
@@ -306,6 +313,7 @@ func (e *FunctionCallExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 				),
 				Subject:     &e.CloseParenRange,
 				Context:     e.Range().Ptr(),
+				EvalContext: ctx,
 			},
 		}
 	}
@@ -321,6 +329,7 @@ func (e *FunctionCallExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 				),
 				Subject:     args[len(params)].StartRange().Ptr(),
 				Context:     e.Range().Ptr(),
+				EvalContext: ctx,
 			},
 		}
 	}
@@ -352,6 +361,7 @@ func (e *FunctionCallExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 				),
 				Subject:     argExpr.StartRange().Ptr(),
 				Context:     e.Range().Ptr(),
+				EvalContext: ctx,
 			})
 		}
 
@@ -389,6 +399,7 @@ func (e *FunctionCallExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 				),
 				Subject:     argExpr.StartRange().Ptr(),
 				Context:     e.Range().Ptr(),
+				EvalContext: ctx,
 			})
 
 		default:
@@ -401,6 +412,7 @@ func (e *FunctionCallExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 				),
 				Subject:     e.StartRange().Ptr(),
 				Context:     e.Range().Ptr(),
+				EvalContext: ctx,
 			})
 		}
 
@@ -469,6 +481,7 @@ func (e *ConditionalExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostic
 				),
 				Subject:     hcl.RangeBetween(e.TrueResult.Range(), e.FalseResult.Range()).Ptr(),
 				Context:     &e.SrcRange,
+				EvalContext: ctx,
 			},
 		}
 	}
@@ -482,6 +495,7 @@ func (e *ConditionalExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostic
 			Detail:      "The condition value is null. Conditions must either be true or false.",
 			Subject:     e.Condition.Range().Ptr(),
 			Context:     &e.SrcRange,
+			EvalContext: ctx,
 		})
 		return cty.UnknownVal(resultType), diags
 	}
@@ -496,6 +510,7 @@ func (e *ConditionalExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostic
 			Detail:      fmt.Sprintf("The condition expression must be of type bool."),
 			Subject:     e.Condition.Range().Ptr(),
 			Context:     &e.SrcRange,
+			EvalContext: ctx,
 		})
 		return cty.UnknownVal(resultType), diags
 	}
@@ -516,6 +531,7 @@ func (e *ConditionalExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostic
 					),
 					Subject:     e.TrueResult.Range().Ptr(),
 					Context:     &e.SrcRange,
+					EvalContext: ctx,
 				})
 				trueResult = cty.UnknownVal(resultType)
 			}
@@ -537,6 +553,7 @@ func (e *ConditionalExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostic
 					),
 					Subject:     e.TrueResult.Range().Ptr(),
 					Context:     &e.SrcRange,
+					EvalContext: ctx,
 				})
 				falseResult = cty.UnknownVal(resultType)
 			}
@@ -680,6 +697,7 @@ func (e *ObjectConsExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics
 				Summary:     "Null value as key",
 				Detail:      "Can't use a null value as a key.",
 				Subject:     item.ValueExpr.Range().Ptr(),
+				EvalContext: ctx,
 			})
 			known = false
 			continue
@@ -693,6 +711,7 @@ func (e *ObjectConsExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics
 				Summary:     "Incorrect key type",
 				Detail:      fmt.Sprintf("Can't use this value as a key: %s.", err.Error()),
 				Subject:     item.ValueExpr.Range().Ptr(),
+				EvalContext: ctx,
 			})
 			known = false
 			continue
@@ -824,6 +843,7 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 			Detail:      "A null value cannot be used as the collection in a 'for' expression.",
 			Subject:     e.CollExpr.Range().Ptr(),
 			Context:     &e.SrcRange,
+			EvalContext: ctx,
 		})
 		return cty.DynamicVal, diags
 	}
@@ -840,6 +860,7 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 			),
 			Subject:     e.CollExpr.Range().Ptr(),
 			Context:     &e.SrcRange,
+			EvalContext: ctx,
 		})
 		return cty.DynamicVal, diags
 	}
@@ -869,6 +890,7 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 				Detail:      "The value of the 'if' clause must not be null.",
 				Subject:     e.CondExpr.Range().Ptr(),
 				Context:     &e.SrcRange,
+				EvalContext: ctx,
 			})
 			return cty.DynamicVal, diags
 		}
@@ -880,6 +902,7 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 				Detail:      fmt.Sprintf("The 'if' clause value is invalid: %s.", err.Error()),
 				Subject:     e.CondExpr.Range().Ptr(),
 				Context:     &e.SrcRange,
+				EvalContext: ctx,
 			})
 			return cty.DynamicVal, diags
 		}
@@ -915,10 +938,11 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 					if known {
 						diags = append(diags, &hcl.Diagnostic{
 							Severity:    hcl.DiagError,
-							Summary:  "Condition is null",
+							Summary:     "Invalid 'for' condition",
 							Detail:      "The value of the 'if' clause must not be null.",
 							Subject:     e.CondExpr.Range().Ptr(),
 							Context:     &e.SrcRange,
+							EvalContext: ctx,
 						})
 					}
 					known = false
@@ -933,6 +957,7 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 							Detail:      fmt.Sprintf("The 'if' clause value is invalid: %s.", err.Error()),
 							Subject:     e.CondExpr.Range().Ptr(),
 							Context:     &e.SrcRange,
+							EvalContext: ctx,
 						})
 					}
 					known = false
@@ -959,6 +984,7 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 						Detail:      "Key expression in 'for' expression must not produce a null value.",
 						Subject:     e.KeyExpr.Range().Ptr(),
 						Context:     &e.SrcRange,
+						EvalContext: ctx,
 					})
 				}
 				known = false
@@ -978,6 +1004,7 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 						Detail:      fmt.Sprintf("The key expression produced an invalid result: %s.", err.Error()),
 						Subject:     e.KeyExpr.Range().Ptr(),
 						Context:     &e.SrcRange,
+						EvalContext: ctx,
 					})
 				}
 				known = false
@@ -997,11 +1024,12 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 						Severity: hcl.DiagError,
 						Summary:  "Duplicate object key",
 						Detail: fmt.Sprintf(
-							"Two different items produced the key %q in this for expression. If duplicates are expected, use the ellipsis (...) after the value expression to enable grouping by key.",
+							"Two different items produced the key %q in this 'for' expression. If duplicates are expected, use the ellipsis (...) after the value expression to enable grouping by key.",
 							k,
 						),
 						Subject:     e.KeyExpr.Range().Ptr(),
 						Context:     &e.SrcRange,
+						EvalContext: ctx,
 					})
 				} else {
 					vals[key.AsString()] = val
@@ -1043,10 +1071,11 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 					if known {
 						diags = append(diags, &hcl.Diagnostic{
 							Severity:    hcl.DiagError,
-							Summary:  "Condition is null",
+							Summary:     "Invalid 'for' condition",
 							Detail:      "The value of the 'if' clause must not be null.",
 							Subject:     e.CondExpr.Range().Ptr(),
 							Context:     &e.SrcRange,
+							EvalContext: ctx,
 						})
 					}
 					known = false
@@ -1069,6 +1098,7 @@ func (e *ForExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 							Detail:      fmt.Sprintf("The 'if' clause value is invalid: %s.", err.Error()),
 							Subject:     e.CondExpr.Range().Ptr(),
 							Context:     &e.SrcRange,
+							EvalContext: ctx,
 						})
 					}
 					known = false
@@ -1159,6 +1189,7 @@ func (e *SplatExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 			Detail:      "Splat expressions (with the * symbol) cannot be applied to null values.",
 			Subject:     e.Source.Range().Ptr(),
 			Context:     hcl.RangeBetween(e.Source.Range(), e.MarkerRange).Ptr(),
+			EvalContext: ctx,
 		})
 		return cty.DynamicVal, diags
 	}

hcl/hclsyntax/expression_ops.go

@@ -154,6 +154,7 @@ func (e *BinaryOpExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics)
 			Detail:      fmt.Sprintf("Unsuitable value for left operand: %s.", err),
 			Subject:     e.LHS.Range().Ptr(),
 			Context:     &e.SrcRange,
+			EvalContext: ctx,
 		})
 	}
 	rhsVal, err := convert.Convert(givenRHSVal, rhsParam.Type)
@@ -164,6 +165,7 @@ func (e *BinaryOpExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics)
 			Detail:      fmt.Sprintf("Unsuitable value for right operand: %s.", err),
 			Subject:     e.RHS.Range().Ptr(),
 			Context:     &e.SrcRange,
+			EvalContext: ctx,
 		})
 	}
 
@@ -182,6 +184,7 @@ func (e *BinaryOpExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics)
 			Summary:     "Operation failed",
 			Detail:      fmt.Sprintf("Error during operation: %s.", err),
 			Subject:     &e.SrcRange,
+			EvalContext: ctx,
 		})
 		return cty.UnknownVal(e.Op.Type), diags
 	}
@@ -224,6 +227,7 @@ func (e *UnaryOpExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 			Detail:      fmt.Sprintf("Unsuitable value for unary operand: %s.", err),
 			Subject:     e.Val.Range().Ptr(),
 			Context:     &e.SrcRange,
+			EvalContext: ctx,
 		})
 	}
 
@@ -242,6 +246,7 @@ func (e *UnaryOpExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 			Summary:     "Operation failed",
 			Detail:      fmt.Sprintf("Error during operation: %s.", err),
 			Subject:     &e.SrcRange,
+			EvalContext: ctx,
 		})
 		return cty.UnknownVal(e.Op.Type), diags
 	}

hcl/hclsyntax/expression_template.go

@@ -39,6 +39,7 @@ func (e *TemplateExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics)
 				),
 				Subject:     part.Range().Ptr(),
 				Context:     &e.SrcRange,
+				EvalContext: ctx,
 			})
 			continue
 		}
@@ -63,6 +64,7 @@ func (e *TemplateExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics)
 				),
 				Subject:     part.Range().Ptr(),
 				Context:     &e.SrcRange,
+				EvalContext: ctx,
 			})
 			continue
 		}
@@ -128,6 +130,7 @@ func (e *TemplateJoinExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 					"An iteration result is null. Cannot include a null value in a string template.",
 				),
 				Subject:     e.Range().Ptr(),
+				EvalContext: ctx,
 			})
 			continue
 		}
@@ -144,6 +147,7 @@ func (e *TemplateJoinExpr) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnosti
 					err.Error(),
 				),
 				Subject:     e.Range().Ptr(),
+				EvalContext: ctx,
 			})
 			continue
 		}

hcl/json/structure.go

@@ -444,6 +444,7 @@ func (e *expression) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 					Summary:     "Invalid object key expression",
 					Detail:      fmt.Sprintf("Cannot use this expression as an object key: %s.", err),
 					Subject:     &jsonAttr.NameRange,
+					EvalContext: ctx,
 				})
 				continue
 			}
@@ -453,6 +454,7 @@ func (e *expression) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 					Summary:     "Invalid object key expression",
 					Detail:      "Cannot use null value as an object key.",
 					Subject:     &jsonAttr.NameRange,
+					EvalContext: ctx,
 				})
 				continue
 			}
@@ -475,6 +477,7 @@ func (e *expression) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
 					Summary:     "Duplicate object attribute",
 					Detail:      fmt.Sprintf("An attribute named %q was already defined at %s.", nameStr, attrRanges[nameStr]),
 					Subject:     &jsonAttr.NameRange,
+					EvalContext: ctx,
 				})
 				continue
 			}