diff --git a/cmd/hclspecsuite/runner.go b/cmd/hclspecsuite/runner.go index 1570e67..977a3d7 100644 --- a/cmd/hclspecsuite/runner.go +++ b/cmd/hclspecsuite/runner.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "fmt" "io/ioutil" "os" @@ -123,7 +124,8 @@ func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile) // though it'll actually be parsed by the hcldec child process, since that // way we can produce nice diagnostic messages if hcldec fails to process // the input file. - if src, err := ioutil.ReadFile(inputFilename); err == nil { + src, err := ioutil.ReadFile(inputFilename) + if err == nil { r.parser.AddFile(inputFilename, &hcl.File{ Bytes: src, }) @@ -131,6 +133,42 @@ func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile) var diags hcl.Diagnostics + if tf.ChecksTraversals { + gotTraversals, moreDiags := r.hcldecVariables(specFilename, inputFilename) + diags = append(diags, moreDiags...) + if !moreDiags.HasErrors() { + expected := tf.ExpectedTraversals + for _, got := range gotTraversals { + e := findTraversalSpec(got, expected) + rng := got.SourceRange() + if e == nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unexpected traversal", + Detail: "Detected traversal that is not indicated as expected in the test file.", + Subject: &rng, + }) + } else { + moreDiags := checkTraversalsMatch(got, inputFilename, e) + diags = append(diags, moreDiags...) + } + } + + // Look for any traversals that didn't show up at all. + for _, e := range expected { + if t := findTraversalForSpec(e, gotTraversals); t == nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing expected traversal", + Detail: "This expected traversal was not detected.", + Subject: e.Traversal.SourceRange().Ptr(), + }) + } + } + } + + } + val, moreDiags := r.hcldecTransform(specFilename, inputFilename) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { @@ -226,6 +264,152 @@ func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Dia } } +func (r *Runner) hcldecVariables(specFile, inputFile string) ([]hcl.Traversal, hcl.Diagnostics) { + var diags hcl.Diagnostics + var outBuffer bytes.Buffer + var errBuffer bytes.Buffer + + cmd := &exec.Cmd{ + Path: r.hcldecPath, + Args: []string{ + r.hcldecPath, + "--spec=" + specFile, + "--diags=json", + "--var-refs", + inputFile, + }, + Stdout: &outBuffer, + Stderr: &errBuffer, + } + err := cmd.Run() + if err != nil { + if _, isExit := err.(*exec.ExitError); !isExit { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to run hcldec", + Detail: fmt.Sprintf("Sub-program hcldec (evaluating input) failed to start: %s.", err), + }) + return nil, diags + } + + // If we exited unsuccessfully then we'll expect diagnostics on stderr + moreDiags := decodeJSONDiagnostics(errBuffer.Bytes()) + diags = append(diags, moreDiags...) + return nil, diags + } else { + // Otherwise, we expect a JSON description of the traversals on stdout. + type PosJSON struct { + Line int `json:"line"` + Column int `json:"column"` + Byte int `json:"byte"` + } + type RangeJSON struct { + Filename string `json:"filename"` + Start PosJSON `json:"start"` + End PosJSON `json:"end"` + } + type StepJSON struct { + Kind string `json:"kind"` + Name string `json:"name,omitempty"` + Key json.RawMessage `json:"key,omitempty"` + Range RangeJSON `json:"range"` + } + type TraversalJSON struct { + Steps []StepJSON `json:"steps"` + } + + var raw []TraversalJSON + err := json.Unmarshal(outBuffer.Bytes(), &raw) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to parse hcldec result", + Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: %s.", err), + }) + return nil, diags + } + + var ret []hcl.Traversal + if len(raw) == 0 { + return ret, diags + } + + ret = make([]hcl.Traversal, 0, len(raw)) + for _, rawT := range raw { + traversal := make(hcl.Traversal, 0, len(rawT.Steps)) + for _, rawS := range rawT.Steps { + rng := hcl.Range{ + Filename: rawS.Range.Filename, + Start: hcl.Pos{ + Line: rawS.Range.Start.Line, + Column: rawS.Range.Start.Column, + Byte: rawS.Range.Start.Byte, + }, + End: hcl.Pos{ + Line: rawS.Range.End.Line, + Column: rawS.Range.End.Column, + Byte: rawS.Range.End.Byte, + }, + } + + switch rawS.Kind { + + case "root": + traversal = append(traversal, hcl.TraverseRoot{ + Name: rawS.Name, + SrcRange: rng, + }) + + case "attr": + traversal = append(traversal, hcl.TraverseAttr{ + Name: rawS.Name, + SrcRange: rng, + }) + + case "index": + ty, err := ctyjson.ImpliedType([]byte(rawS.Key)) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to parse hcldec result", + Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step has invalid index key %s.", rawS.Key), + }) + return nil, diags + } + keyVal, err := ctyjson.Unmarshal([]byte(rawS.Key), ty) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to parse hcldec result", + Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced a result with an invalid index key %s: %s.", rawS.Key, err), + }) + return nil, diags + } + + traversal = append(traversal, hcl.TraverseIndex{ + Key: keyVal, + SrcRange: rng, + }) + + default: + // Should never happen since the above cases are exhaustive, + // but we'll catch it gracefully since this is coming from + // a possibly-buggy hcldec implementation that we're testing. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to parse hcldec result", + Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step of unsupported kind %q.", rawS.Kind), + }) + return nil, diags + } + } + + ret = append(ret, traversal) + } + return ret, diags + } +} + func (r *Runner) prettyDirName(dir string) string { rel, err := filepath.Rel(r.baseDir, dir) if err != nil { diff --git a/cmd/hclspecsuite/test_file.go b/cmd/hclspecsuite/test_file.go index 2577328..cdb384a 100644 --- a/cmd/hclspecsuite/test_file.go +++ b/cmd/hclspecsuite/test_file.go @@ -27,11 +27,13 @@ type TestFile struct { type TestFileExpectTraversal struct { Traversal hcl.Traversal Range hcl.Range + DeclRange hcl.Range } type TestFileExpectDiag struct { - Severity hcl.DiagnosticSeverity - Range hcl.Range + Severity hcl.DiagnosticSeverity + Range hcl.Range + DeclRange hcl.Range } func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) { @@ -181,6 +183,7 @@ func (r *Runner) decodeTraversalExpectBlock(block *hcl.Block) (*TestFileExpectTr return &TestFileExpectTraversal{ Traversal: traversal, Range: rng, + DeclRange: block.DefRange, }, diags } @@ -226,8 +229,9 @@ func (r *Runner) decodeDiagnosticsBlock(block *hcl.Block) ([]*TestFileExpectDiag } ret = append(ret, &TestFileExpectDiag{ - Severity: severity, - Range: rng, + Severity: severity, + Range: rng, + DeclRange: block.TypeRange, }) } return ret, diags @@ -254,13 +258,13 @@ func (r *Runner) decodeRangeFromBody(body hcl.Body) (hcl.Range, hcl.Body, hcl.Di // path we pass to hcldec. Start: hcl.Pos{ Line: raw.From.Line, - Column: raw.From.Line, - Byte: raw.From.Line, + Column: raw.From.Column, + Byte: raw.From.Byte, }, End: hcl.Pos{ Line: raw.To.Line, - Column: raw.To.Line, - Byte: raw.To.Line, + Column: raw.To.Column, + Byte: raw.To.Byte, }, }, raw.Remain, diags } diff --git a/cmd/hclspecsuite/traversals.go b/cmd/hclspecsuite/traversals.go new file mode 100644 index 0000000..160d5b7 --- /dev/null +++ b/cmd/hclspecsuite/traversals.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "reflect" + + "github.com/hashicorp/hcl2/hcl" +) + +func findTraversalSpec(got hcl.Traversal, candidates []*TestFileExpectTraversal) *TestFileExpectTraversal { + for _, candidate := range candidates { + if traversalsAreEquivalent(candidate.Traversal, got) { + return candidate + } + } + return nil +} + +func findTraversalForSpec(want *TestFileExpectTraversal, have []hcl.Traversal) hcl.Traversal { + for _, candidate := range have { + if traversalsAreEquivalent(candidate, want.Traversal) { + return candidate + } + } + return nil +} + +func traversalsAreEquivalent(a, b hcl.Traversal) bool { + if len(a) != len(b) { + return false + } + for i := range a { + aStep := a[i] + bStep := b[i] + + if reflect.TypeOf(aStep) != reflect.TypeOf(bStep) { + return false + } + + // We can now assume that both are of the same type. + switch ts := aStep.(type) { + + case hcl.TraverseRoot: + if bStep.(hcl.TraverseRoot).Name != ts.Name { + return false + } + + case hcl.TraverseAttr: + if bStep.(hcl.TraverseAttr).Name != ts.Name { + return false + } + + case hcl.TraverseIndex: + if !bStep.(hcl.TraverseIndex).Key.RawEquals(ts.Key) { + return false + } + + default: + return false + } + } + return true +} + +// checkTraversalsMatch determines if a given traversal matches the given +// expectation, which must've been produced by an earlier call to +// findTraversalSpec for the same traversal. +func checkTraversalsMatch(got hcl.Traversal, filename string, match *TestFileExpectTraversal) hcl.Diagnostics { + var diags hcl.Diagnostics + + gotRng := got.SourceRange() + wantRng := match.Range + + if got, want := gotRng.Filename, filename; got != want { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Incorrect filename in detected traversal", + Detail: fmt.Sprintf( + "Filename was reported as %q, but was expecting %q.", + got, want, + ), + Subject: match.Traversal.SourceRange().Ptr(), + }) + return diags + } + + // If we have the expected filename then we'll use that to construct the + // full "want range" here so that we can use it to point to the appropriate + // location in the remaining diagnostics. + wantRng.Filename = filename + + if got, want := gotRng.Start, wantRng.Start; got != want { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Incorrect start position in detected traversal", + Detail: fmt.Sprintf( + "Start position was reported as line %d column %d byte %d, but was expecting line %d column %d byte %d.", + got.Line, got.Column, got.Byte, + want.Line, want.Column, want.Byte, + ), + Subject: &wantRng, + }) + } + if got, want := gotRng.End, wantRng.End; got != want { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Incorrect end position in detected traversal", + Detail: fmt.Sprintf( + "End position was reported as line %d column %d byte %d, but was expecting line %d column %d byte %d.", + got.Line, got.Column, got.Byte, + want.Line, want.Column, want.Byte, + ), + Subject: &wantRng, + }) + } + return diags +}