cmd/hclspecsuite: Check for traversals when requested

cmd/hclspecsuite/runner.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -123,7 +124,8 @@ func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile)
 	// though it'll actually be parsed by the hcldec child process, since that
 	// way we can produce nice diagnostic messages if hcldec fails to process
 	// the input file.
-	if src, err := ioutil.ReadFile(inputFilename); err == nil {
+	src, err := ioutil.ReadFile(inputFilename)
+	if err == nil {
 		r.parser.AddFile(inputFilename, &hcl.File{
 			Bytes: src,
 		})
@@ -131,6 +133,42 @@ func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile)
 
 	var diags hcl.Diagnostics
 
+	if tf.ChecksTraversals {
+		gotTraversals, moreDiags := r.hcldecVariables(specFilename, inputFilename)
+		diags = append(diags, moreDiags...)
+		if !moreDiags.HasErrors() {
+			expected := tf.ExpectedTraversals
+			for _, got := range gotTraversals {
+				e := findTraversalSpec(got, expected)
+				rng := got.SourceRange()
+				if e == nil {
+					diags = append(diags, &hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Unexpected traversal",
+						Detail:   "Detected traversal that is not indicated as expected in the test file.",
+						Subject:  &rng,
+					})
+				} else {
+					moreDiags := checkTraversalsMatch(got, inputFilename, e)
+					diags = append(diags, moreDiags...)
+				}
+			}
+
+			// Look for any traversals that didn't show up at all.
+			for _, e := range expected {
+				if t := findTraversalForSpec(e, gotTraversals); t == nil {
+					diags = append(diags, &hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Missing expected traversal",
+						Detail:   "This expected traversal was not detected.",
+						Subject:  e.Traversal.SourceRange().Ptr(),
+					})
+				}
+			}
+		}
+
+	}
+
 	val, moreDiags := r.hcldecTransform(specFilename, inputFilename)
 	diags = append(diags, moreDiags...)
 	if moreDiags.HasErrors() {
@@ -226,6 +264,152 @@ func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Dia
 	}
 }
 
+func (r *Runner) hcldecVariables(specFile, inputFile string) ([]hcl.Traversal, hcl.Diagnostics) {
+	var diags hcl.Diagnostics
+	var outBuffer bytes.Buffer
+	var errBuffer bytes.Buffer
+
+	cmd := &exec.Cmd{
+		Path: r.hcldecPath,
+		Args: []string{
+			r.hcldecPath,
+			"--spec=" + specFile,
+			"--diags=json",
+			"--var-refs",
+			inputFile,
+		},
+		Stdout: &outBuffer,
+		Stderr: &errBuffer,
+	}
+	err := cmd.Run()
+	if err != nil {
+		if _, isExit := err.(*exec.ExitError); !isExit {
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to run hcldec",
+				Detail:   fmt.Sprintf("Sub-program hcldec (evaluating input) failed to start: %s.", err),
+			})
+			return nil, diags
+		}
+
+		// If we exited unsuccessfully then we'll expect diagnostics on stderr
+		moreDiags := decodeJSONDiagnostics(errBuffer.Bytes())
+		diags = append(diags, moreDiags...)
+		return nil, diags
+	} else {
+		// Otherwise, we expect a JSON description of the traversals on stdout.
+		type PosJSON struct {
+			Line   int `json:"line"`
+			Column int `json:"column"`
+			Byte   int `json:"byte"`
+		}
+		type RangeJSON struct {
+			Filename string  `json:"filename"`
+			Start    PosJSON `json:"start"`
+			End      PosJSON `json:"end"`
+		}
+		type StepJSON struct {
+			Kind  string          `json:"kind"`
+			Name  string          `json:"name,omitempty"`
+			Key   json.RawMessage `json:"key,omitempty"`
+			Range RangeJSON       `json:"range"`
+		}
+		type TraversalJSON struct {
+			Steps []StepJSON `json:"steps"`
+		}
+
+		var raw []TraversalJSON
+		err := json.Unmarshal(outBuffer.Bytes(), &raw)
+		if err != nil {
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to parse hcldec result",
+				Detail:   fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: %s.", err),
+			})
+			return nil, diags
+		}
+
+		var ret []hcl.Traversal
+		if len(raw) == 0 {
+			return ret, diags
+		}
+
+		ret = make([]hcl.Traversal, 0, len(raw))
+		for _, rawT := range raw {
+			traversal := make(hcl.Traversal, 0, len(rawT.Steps))
+			for _, rawS := range rawT.Steps {
+				rng := hcl.Range{
+					Filename: rawS.Range.Filename,
+					Start: hcl.Pos{
+						Line:   rawS.Range.Start.Line,
+						Column: rawS.Range.Start.Column,
+						Byte:   rawS.Range.Start.Byte,
+					},
+					End: hcl.Pos{
+						Line:   rawS.Range.End.Line,
+						Column: rawS.Range.End.Column,
+						Byte:   rawS.Range.End.Byte,
+					},
+				}
+
+				switch rawS.Kind {
+
+				case "root":
+					traversal = append(traversal, hcl.TraverseRoot{
+						Name:     rawS.Name,
+						SrcRange: rng,
+					})
+
+				case "attr":
+					traversal = append(traversal, hcl.TraverseAttr{
+						Name:     rawS.Name,
+						SrcRange: rng,
+					})
+
+				case "index":
+					ty, err := ctyjson.ImpliedType([]byte(rawS.Key))
+					if err != nil {
+						diags = append(diags, &hcl.Diagnostic{
+							Severity: hcl.DiagError,
+							Summary:  "Failed to parse hcldec result",
+							Detail:   fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step has invalid index key %s.", rawS.Key),
+						})
+						return nil, diags
+					}
+					keyVal, err := ctyjson.Unmarshal([]byte(rawS.Key), ty)
+					if err != nil {
+						diags = append(diags, &hcl.Diagnostic{
+							Severity: hcl.DiagError,
+							Summary:  "Failed to parse hcldec result",
+							Detail:   fmt.Sprintf("Sub-program hcldec (with --var-refs) produced a result with an invalid index key %s: %s.", rawS.Key, err),
+						})
+						return nil, diags
+					}
+
+					traversal = append(traversal, hcl.TraverseIndex{
+						Key:      keyVal,
+						SrcRange: rng,
+					})
+
+				default:
+					// Should never happen since the above cases are exhaustive,
+					// but we'll catch it gracefully since this is coming from
+					// a possibly-buggy hcldec implementation that we're testing.
+					diags = append(diags, &hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Failed to parse hcldec result",
+						Detail:   fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step of unsupported kind %q.", rawS.Kind),
+					})
+					return nil, diags
+				}
+			}
+
+			ret = append(ret, traversal)
+		}
+		return ret, diags
+	}
+}
+
 func (r *Runner) prettyDirName(dir string) string {
 	rel, err := filepath.Rel(r.baseDir, dir)
 	if err != nil {

cmd/hclspecsuite/test_file.go

@@ -27,11 +27,13 @@ type TestFile struct {
 type TestFileExpectTraversal struct {
 	Traversal hcl.Traversal
 	Range     hcl.Range
+	DeclRange hcl.Range
 }
 
 type TestFileExpectDiag struct {
 	Severity  hcl.DiagnosticSeverity
 	Range     hcl.Range
+	DeclRange hcl.Range
 }
 
 func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) {
@@ -181,6 +183,7 @@ func (r *Runner) decodeTraversalExpectBlock(block *hcl.Block) (*TestFileExpectTr
 	return &TestFileExpectTraversal{
 		Traversal: traversal,
 		Range:     rng,
+		DeclRange: block.DefRange,
 	}, diags
 }
 
@@ -228,6 +231,7 @@ func (r *Runner) decodeDiagnosticsBlock(block *hcl.Block) ([]*TestFileExpectDiag
 		ret = append(ret, &TestFileExpectDiag{
 			Severity:  severity,
 			Range:     rng,
+			DeclRange: block.TypeRange,
 		})
 	}
 	return ret, diags
@@ -254,13 +258,13 @@ func (r *Runner) decodeRangeFromBody(body hcl.Body) (hcl.Range, hcl.Body, hcl.Di
 		// path we pass to hcldec.
 		Start: hcl.Pos{
 			Line:   raw.From.Line,
-			Column: raw.From.Line,
+			Column: raw.From.Column,
-			Byte:   raw.From.Line,
+			Byte:   raw.From.Byte,
 		},
 		End: hcl.Pos{
 			Line:   raw.To.Line,
-			Column: raw.To.Line,
+			Column: raw.To.Column,
-			Byte:   raw.To.Line,
+			Byte:   raw.To.Byte,
 		},
 	}, raw.Remain, diags
 }

cmd/hclspecsuite/traversals.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/hashicorp/hcl2/hcl"
+)
+
+func findTraversalSpec(got hcl.Traversal, candidates []*TestFileExpectTraversal) *TestFileExpectTraversal {
+	for _, candidate := range candidates {
+		if traversalsAreEquivalent(candidate.Traversal, got) {
+			return candidate
+		}
+	}
+	return nil
+}
+
+func findTraversalForSpec(want *TestFileExpectTraversal, have []hcl.Traversal) hcl.Traversal {
+	for _, candidate := range have {
+		if traversalsAreEquivalent(candidate, want.Traversal) {
+			return candidate
+		}
+	}
+	return nil
+}
+
+func traversalsAreEquivalent(a, b hcl.Traversal) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		aStep := a[i]
+		bStep := b[i]
+
+		if reflect.TypeOf(aStep) != reflect.TypeOf(bStep) {
+			return false
+		}
+
+		// We can now assume that both are of the same type.
+		switch ts := aStep.(type) {
+
+		case hcl.TraverseRoot:
+			if bStep.(hcl.TraverseRoot).Name != ts.Name {
+				return false
+			}
+
+		case hcl.TraverseAttr:
+			if bStep.(hcl.TraverseAttr).Name != ts.Name {
+				return false
+			}
+
+		case hcl.TraverseIndex:
+			if !bStep.(hcl.TraverseIndex).Key.RawEquals(ts.Key) {
+				return false
+			}
+
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+// checkTraversalsMatch determines if a given traversal matches the given
+// expectation, which must've been produced by an earlier call to
+// findTraversalSpec for the same traversal.
+func checkTraversalsMatch(got hcl.Traversal, filename string, match *TestFileExpectTraversal) hcl.Diagnostics {
+	var diags hcl.Diagnostics
+
+	gotRng := got.SourceRange()
+	wantRng := match.Range
+
+	if got, want := gotRng.Filename, filename; got != want {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Incorrect filename in detected traversal",
+			Detail: fmt.Sprintf(
+				"Filename was reported as %q, but was expecting %q.",
+				got, want,
+			),
+			Subject: match.Traversal.SourceRange().Ptr(),
+		})
+		return diags
+	}
+
+	// If we have the expected filename then we'll use that to construct the
+	// full "want range" here so that we can use it to point to the appropriate
+	// location in the remaining diagnostics.
+	wantRng.Filename = filename
+
+	if got, want := gotRng.Start, wantRng.Start; got != want {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Incorrect start position in detected traversal",
+			Detail: fmt.Sprintf(
+				"Start position was reported as line %d column %d byte %d, but was expecting line %d column %d byte %d.",
+				got.Line, got.Column, got.Byte,
+				want.Line, want.Column, want.Byte,
+			),
+			Subject: &wantRng,
+		})
+	}
+	if got, want := gotRng.End, wantRng.End; got != want {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Incorrect end position in detected traversal",
+			Detail: fmt.Sprintf(
+				"End position was reported as line %d column %d byte %d, but was expecting line %d column %d byte %d.",
+				got.Line, got.Column, got.Byte,
+				want.Line, want.Column, want.Byte,
+			),
+			Subject: &wantRng,
+		})
+	}
+	return diags
+}