7887810587
Lately we've made some changes that have affected the behavior of the specsuite tests, causing them to fail. Much of this was caused by changes to the harness itself (based on hcldec), although one break in particular here was an intentional change to the implementation of modulo in upstream cty to make it produce a more accurate result when used with a fractional divisor.
525 lines
14 KiB
Go
525 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/zclconf/go-cty-debug/ctydebug"
|
|
"github.com/zclconf/go-cty/cty"
|
|
"github.com/zclconf/go-cty/cty/convert"
|
|
ctyjson "github.com/zclconf/go-cty/cty/json"
|
|
|
|
"github.com/hashicorp/hcl/v2"
|
|
"github.com/hashicorp/hcl/v2/ext/typeexpr"
|
|
"github.com/hashicorp/hcl/v2/hclparse"
|
|
)
|
|
|
|
type Runner struct {
|
|
parser *hclparse.Parser
|
|
hcldecPath string
|
|
baseDir string
|
|
logBegin LogBeginCallback
|
|
logProblems LogProblemsCallback
|
|
}
|
|
|
|
func (r *Runner) Run() hcl.Diagnostics {
|
|
return r.runDir(r.baseDir)
|
|
}
|
|
|
|
func (r *Runner) runDir(dir string) hcl.Diagnostics {
|
|
var diags hcl.Diagnostics
|
|
|
|
infos, err := ioutil.ReadDir(dir)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to read test directory",
|
|
Detail: fmt.Sprintf("The directory %q could not be opened: %s.", dir, err),
|
|
})
|
|
return diags
|
|
}
|
|
|
|
var tests []string
|
|
var subDirs []string
|
|
for _, info := range infos {
|
|
name := info.Name()
|
|
if strings.HasPrefix(name, ".") {
|
|
continue
|
|
}
|
|
|
|
if info.IsDir() {
|
|
subDirs = append(subDirs, name)
|
|
}
|
|
if strings.HasSuffix(name, ".t") {
|
|
tests = append(tests, name)
|
|
}
|
|
}
|
|
sort.Strings(tests)
|
|
sort.Strings(subDirs)
|
|
|
|
for _, filename := range tests {
|
|
filename = filepath.Join(dir, filename)
|
|
testDiags := r.runTest(filename)
|
|
diags = append(diags, testDiags...)
|
|
}
|
|
|
|
for _, dirName := range subDirs {
|
|
dir := filepath.Join(dir, dirName)
|
|
dirDiags := r.runDir(dir)
|
|
diags = append(diags, dirDiags...)
|
|
}
|
|
|
|
return diags
|
|
}
|
|
|
|
func (r *Runner) runTest(filename string) hcl.Diagnostics {
|
|
prettyName := r.prettyTestName(filename)
|
|
tf, diags := r.LoadTestFile(filename)
|
|
if diags.HasErrors() {
|
|
// We'll still log, so it's clearer which test the diagnostics belong to.
|
|
if r.logBegin != nil {
|
|
r.logBegin(prettyName, nil)
|
|
}
|
|
if r.logProblems != nil {
|
|
r.logProblems(prettyName, nil, diags)
|
|
return nil // don't duplicate the diagnostics we already reported
|
|
}
|
|
return diags
|
|
}
|
|
|
|
if r.logBegin != nil {
|
|
r.logBegin(prettyName, tf)
|
|
}
|
|
|
|
basePath := filename[:len(filename)-2]
|
|
specFilename := basePath + ".hcldec"
|
|
nativeFilename := basePath + ".hcl"
|
|
jsonFilename := basePath + ".hcl.json"
|
|
|
|
// We'll add the source code of the spec file to our own parser, even
|
|
// though it'll actually be parsed by the hcldec child process, since that
|
|
// way we can produce nice diagnostic messages if hcldec fails to process
|
|
// the spec file.
|
|
src, err := ioutil.ReadFile(specFilename)
|
|
if err == nil {
|
|
r.parser.AddFile(specFilename, &hcl.File{
|
|
Bytes: src,
|
|
})
|
|
}
|
|
|
|
if _, err := os.Stat(specFilename); err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Missing .hcldec file",
|
|
Detail: fmt.Sprintf("No specification file for test %s: %s.", prettyName, err),
|
|
})
|
|
return diags
|
|
}
|
|
|
|
if _, err := os.Stat(nativeFilename); err == nil {
|
|
moreDiags := r.runTestInput(specFilename, nativeFilename, tf)
|
|
diags = append(diags, moreDiags...)
|
|
}
|
|
|
|
if _, err := os.Stat(jsonFilename); err == nil {
|
|
moreDiags := r.runTestInput(specFilename, jsonFilename, tf)
|
|
diags = append(diags, moreDiags...)
|
|
}
|
|
|
|
if r.logProblems != nil {
|
|
r.logProblems(prettyName, nil, diags)
|
|
return nil // don't duplicate the diagnostics we already reported
|
|
}
|
|
|
|
return diags
|
|
}
|
|
|
|
func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile) hcl.Diagnostics {
|
|
// We'll add the source code of the input file to our own parser, even
|
|
// though it'll actually be parsed by the hcldec child process, since that
|
|
// way we can produce nice diagnostic messages if hcldec fails to process
|
|
// the input file.
|
|
src, err := ioutil.ReadFile(inputFilename)
|
|
if err == nil {
|
|
r.parser.AddFile(inputFilename, &hcl.File{
|
|
Bytes: src,
|
|
})
|
|
}
|
|
|
|
var diags hcl.Diagnostics
|
|
|
|
if tf.ChecksTraversals {
|
|
gotTraversals, moreDiags := r.hcldecVariables(specFilename, inputFilename)
|
|
diags = append(diags, moreDiags...)
|
|
if !moreDiags.HasErrors() {
|
|
expected := tf.ExpectedTraversals
|
|
for _, got := range gotTraversals {
|
|
e := findTraversalSpec(got, expected)
|
|
rng := got.SourceRange()
|
|
if e == nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Unexpected traversal",
|
|
Detail: "Detected traversal that is not indicated as expected in the test file.",
|
|
Subject: &rng,
|
|
})
|
|
} else {
|
|
moreDiags := checkTraversalsMatch(got, inputFilename, e)
|
|
diags = append(diags, moreDiags...)
|
|
}
|
|
}
|
|
|
|
// Look for any traversals that didn't show up at all.
|
|
for _, e := range expected {
|
|
if t := findTraversalForSpec(e, gotTraversals); t == nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Missing expected traversal",
|
|
Detail: "This expected traversal was not detected.",
|
|
Subject: e.Traversal.SourceRange().Ptr(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
val, transformDiags := r.hcldecTransform(specFilename, inputFilename)
|
|
if len(tf.ExpectedDiags) == 0 {
|
|
diags = append(diags, transformDiags...)
|
|
if transformDiags.HasErrors() {
|
|
// If hcldec failed then there's no point in continuing.
|
|
return diags
|
|
}
|
|
|
|
if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Incorrect result type",
|
|
Detail: fmt.Sprintf(
|
|
"Input file %s produced %s, but was expecting %s.",
|
|
inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType),
|
|
),
|
|
})
|
|
}
|
|
|
|
if tf.Result != cty.NilVal {
|
|
cmpVal, err := convert.Convert(tf.Result, tf.ResultType)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Incorrect type for result value",
|
|
Detail: fmt.Sprintf(
|
|
"Result does not conform to the given result type: %s.", err,
|
|
),
|
|
Subject: &tf.ResultRange,
|
|
})
|
|
} else {
|
|
if !val.RawEquals(cmpVal) {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Incorrect result value",
|
|
Detail: fmt.Sprintf(
|
|
"Input file %s produced %#v, but was expecting %#v.\n\n%s",
|
|
inputFilename, val, tf.Result,
|
|
ctydebug.DiffValues(tf.Result, val),
|
|
),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// We're expecting diagnostics, and so we'll need to correlate the
|
|
// severities and source ranges of our actual diagnostics against
|
|
// what we were expecting.
|
|
type DiagnosticEntry struct {
|
|
Severity hcl.DiagnosticSeverity
|
|
Range hcl.Range
|
|
}
|
|
got := make(map[DiagnosticEntry]*hcl.Diagnostic)
|
|
want := make(map[DiagnosticEntry]hcl.Range)
|
|
for _, diag := range transformDiags {
|
|
if diag.Subject == nil {
|
|
// Sourceless diagnostics can never be expected, so we'll just
|
|
// pass these through as-is and assume they are hcldec
|
|
// operational errors.
|
|
diags = append(diags, diag)
|
|
continue
|
|
}
|
|
if diag.Subject.Filename != inputFilename {
|
|
// If the problem is for something other than the input file
|
|
// then it can't be expected.
|
|
diags = append(diags, diag)
|
|
continue
|
|
}
|
|
entry := DiagnosticEntry{
|
|
Severity: diag.Severity,
|
|
Range: *diag.Subject,
|
|
}
|
|
got[entry] = diag
|
|
}
|
|
for _, e := range tf.ExpectedDiags {
|
|
e.Range.Filename = inputFilename // assumed here, since we don't allow any other filename to be expected
|
|
entry := DiagnosticEntry{
|
|
Severity: e.Severity,
|
|
Range: e.Range,
|
|
}
|
|
want[entry] = e.DeclRange
|
|
}
|
|
|
|
for gotEntry, diag := range got {
|
|
if _, wanted := want[gotEntry]; !wanted {
|
|
// Pass through the diagnostic itself so the user can see what happened
|
|
diags = append(diags, diag)
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Unexpected diagnostic",
|
|
Detail: fmt.Sprintf(
|
|
"No %s diagnostic was expected %s. The unexpected diagnostic was shown above.",
|
|
severityString(gotEntry.Severity), rangeString(gotEntry.Range),
|
|
),
|
|
Subject: gotEntry.Range.Ptr(),
|
|
})
|
|
}
|
|
}
|
|
|
|
for wantEntry, declRange := range want {
|
|
if _, gotted := got[wantEntry]; !gotted {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Missing expected diagnostic",
|
|
Detail: fmt.Sprintf(
|
|
"No %s diagnostic was generated %s.",
|
|
severityString(wantEntry.Severity), rangeString(wantEntry.Range),
|
|
),
|
|
Subject: declRange.Ptr(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
return diags
|
|
}
|
|
|
|
func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Diagnostics) {
|
|
var diags hcl.Diagnostics
|
|
var outBuffer bytes.Buffer
|
|
var errBuffer bytes.Buffer
|
|
|
|
cmd := &exec.Cmd{
|
|
Path: r.hcldecPath,
|
|
Args: []string{
|
|
r.hcldecPath,
|
|
"--spec=" + specFile,
|
|
"--diags=json",
|
|
"--with-type",
|
|
"--keep-nulls",
|
|
inputFile,
|
|
},
|
|
Stdout: &outBuffer,
|
|
Stderr: &errBuffer,
|
|
}
|
|
err := cmd.Run()
|
|
if err != nil {
|
|
if _, isExit := err.(*exec.ExitError); !isExit {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to run hcldec",
|
|
Detail: fmt.Sprintf("Sub-program hcldec failed to start: %s.", err),
|
|
})
|
|
return cty.DynamicVal, diags
|
|
}
|
|
|
|
// If we exited unsuccessfully then we'll expect diagnostics on stderr
|
|
moreDiags := decodeJSONDiagnostics(errBuffer.Bytes())
|
|
diags = append(diags, moreDiags...)
|
|
return cty.DynamicVal, diags
|
|
} else {
|
|
// Otherwise, we expect a JSON result value on stdout. Since we used
|
|
// --with-type above, we can decode as DynamicPseudoType to recover
|
|
// exactly the type that was saved, without the usual JSON lossiness.
|
|
val, err := ctyjson.Unmarshal(outBuffer.Bytes(), cty.DynamicPseudoType)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec produced an invalid result: %s.", err),
|
|
})
|
|
return cty.DynamicVal, diags
|
|
}
|
|
return val, diags
|
|
}
|
|
}
|
|
|
|
func (r *Runner) hcldecVariables(specFile, inputFile string) ([]hcl.Traversal, hcl.Diagnostics) {
|
|
var diags hcl.Diagnostics
|
|
var outBuffer bytes.Buffer
|
|
var errBuffer bytes.Buffer
|
|
|
|
cmd := &exec.Cmd{
|
|
Path: r.hcldecPath,
|
|
Args: []string{
|
|
r.hcldecPath,
|
|
"--spec=" + specFile,
|
|
"--diags=json",
|
|
"--var-refs",
|
|
inputFile,
|
|
},
|
|
Stdout: &outBuffer,
|
|
Stderr: &errBuffer,
|
|
}
|
|
err := cmd.Run()
|
|
if err != nil {
|
|
if _, isExit := err.(*exec.ExitError); !isExit {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to run hcldec",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (evaluating input) failed to start: %s.", err),
|
|
})
|
|
return nil, diags
|
|
}
|
|
|
|
// If we exited unsuccessfully then we'll expect diagnostics on stderr
|
|
moreDiags := decodeJSONDiagnostics(errBuffer.Bytes())
|
|
diags = append(diags, moreDiags...)
|
|
return nil, diags
|
|
} else {
|
|
// Otherwise, we expect a JSON description of the traversals on stdout.
|
|
type PosJSON struct {
|
|
Line int `json:"line"`
|
|
Column int `json:"column"`
|
|
Byte int `json:"byte"`
|
|
}
|
|
type RangeJSON struct {
|
|
Filename string `json:"filename"`
|
|
Start PosJSON `json:"start"`
|
|
End PosJSON `json:"end"`
|
|
}
|
|
type StepJSON struct {
|
|
Kind string `json:"kind"`
|
|
Name string `json:"name,omitempty"`
|
|
Key json.RawMessage `json:"key,omitempty"`
|
|
Range RangeJSON `json:"range"`
|
|
}
|
|
type TraversalJSON struct {
|
|
Steps []StepJSON `json:"steps"`
|
|
}
|
|
|
|
var raw []TraversalJSON
|
|
err := json.Unmarshal(outBuffer.Bytes(), &raw)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: %s.", err),
|
|
})
|
|
return nil, diags
|
|
}
|
|
|
|
var ret []hcl.Traversal
|
|
if len(raw) == 0 {
|
|
return ret, diags
|
|
}
|
|
|
|
ret = make([]hcl.Traversal, 0, len(raw))
|
|
for _, rawT := range raw {
|
|
traversal := make(hcl.Traversal, 0, len(rawT.Steps))
|
|
for _, rawS := range rawT.Steps {
|
|
rng := hcl.Range{
|
|
Filename: rawS.Range.Filename,
|
|
Start: hcl.Pos{
|
|
Line: rawS.Range.Start.Line,
|
|
Column: rawS.Range.Start.Column,
|
|
Byte: rawS.Range.Start.Byte,
|
|
},
|
|
End: hcl.Pos{
|
|
Line: rawS.Range.End.Line,
|
|
Column: rawS.Range.End.Column,
|
|
Byte: rawS.Range.End.Byte,
|
|
},
|
|
}
|
|
|
|
switch rawS.Kind {
|
|
|
|
case "root":
|
|
traversal = append(traversal, hcl.TraverseRoot{
|
|
Name: rawS.Name,
|
|
SrcRange: rng,
|
|
})
|
|
|
|
case "attr":
|
|
traversal = append(traversal, hcl.TraverseAttr{
|
|
Name: rawS.Name,
|
|
SrcRange: rng,
|
|
})
|
|
|
|
case "index":
|
|
ty, err := ctyjson.ImpliedType([]byte(rawS.Key))
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step has invalid index key %s.", rawS.Key),
|
|
})
|
|
return nil, diags
|
|
}
|
|
keyVal, err := ctyjson.Unmarshal([]byte(rawS.Key), ty)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced a result with an invalid index key %s: %s.", rawS.Key, err),
|
|
})
|
|
return nil, diags
|
|
}
|
|
|
|
traversal = append(traversal, hcl.TraverseIndex{
|
|
Key: keyVal,
|
|
SrcRange: rng,
|
|
})
|
|
|
|
default:
|
|
// Should never happen since the above cases are exhaustive,
|
|
// but we'll catch it gracefully since this is coming from
|
|
// a possibly-buggy hcldec implementation that we're testing.
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step of unsupported kind %q.", rawS.Kind),
|
|
})
|
|
return nil, diags
|
|
}
|
|
}
|
|
|
|
ret = append(ret, traversal)
|
|
}
|
|
return ret, diags
|
|
}
|
|
}
|
|
|
|
func (r *Runner) prettyDirName(dir string) string {
|
|
rel, err := filepath.Rel(r.baseDir, dir)
|
|
if err != nil {
|
|
return filepath.ToSlash(dir)
|
|
}
|
|
return filepath.ToSlash(rel)
|
|
}
|
|
|
|
func (r *Runner) prettyTestName(filename string) string {
|
|
dir := filepath.Dir(filename)
|
|
dirName := r.prettyDirName(dir)
|
|
filename = filepath.Base(filename)
|
|
testName := filename[:len(filename)-2]
|
|
if dirName == "." {
|
|
return testName
|
|
}
|
|
return fmt.Sprintf("%s/%s", dirName, testName)
|
|
}
|