522 lines
14 KiB
Go
522 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/zclconf/go-cty/cty"
|
|
"github.com/zclconf/go-cty/cty/convert"
|
|
ctyjson "github.com/zclconf/go-cty/cty/json"
|
|
|
|
"github.com/hashicorp/hcl/v2/ext/typeexpr"
|
|
"github.com/hashicorp/hcl/v2"
|
|
"github.com/hashicorp/hcl/v2/hclparse"
|
|
)
|
|
|
|
type Runner struct {
|
|
parser *hclparse.Parser
|
|
hcldecPath string
|
|
baseDir string
|
|
logBegin LogBeginCallback
|
|
logProblems LogProblemsCallback
|
|
}
|
|
|
|
func (r *Runner) Run() hcl.Diagnostics {
|
|
return r.runDir(r.baseDir)
|
|
}
|
|
|
|
func (r *Runner) runDir(dir string) hcl.Diagnostics {
|
|
var diags hcl.Diagnostics
|
|
|
|
infos, err := ioutil.ReadDir(dir)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to read test directory",
|
|
Detail: fmt.Sprintf("The directory %q could not be opened: %s.", dir, err),
|
|
})
|
|
return diags
|
|
}
|
|
|
|
var tests []string
|
|
var subDirs []string
|
|
for _, info := range infos {
|
|
name := info.Name()
|
|
if strings.HasPrefix(name, ".") {
|
|
continue
|
|
}
|
|
|
|
if info.IsDir() {
|
|
subDirs = append(subDirs, name)
|
|
}
|
|
if strings.HasSuffix(name, ".t") {
|
|
tests = append(tests, name)
|
|
}
|
|
}
|
|
sort.Strings(tests)
|
|
sort.Strings(subDirs)
|
|
|
|
for _, filename := range tests {
|
|
filename = filepath.Join(dir, filename)
|
|
testDiags := r.runTest(filename)
|
|
diags = append(diags, testDiags...)
|
|
}
|
|
|
|
for _, dirName := range subDirs {
|
|
dir := filepath.Join(dir, dirName)
|
|
dirDiags := r.runDir(dir)
|
|
diags = append(diags, dirDiags...)
|
|
}
|
|
|
|
return diags
|
|
}
|
|
|
|
func (r *Runner) runTest(filename string) hcl.Diagnostics {
|
|
prettyName := r.prettyTestName(filename)
|
|
tf, diags := r.LoadTestFile(filename)
|
|
if diags.HasErrors() {
|
|
// We'll still log, so it's clearer which test the diagnostics belong to.
|
|
if r.logBegin != nil {
|
|
r.logBegin(prettyName, nil)
|
|
}
|
|
if r.logProblems != nil {
|
|
r.logProblems(prettyName, nil, diags)
|
|
return nil // don't duplicate the diagnostics we already reported
|
|
}
|
|
return diags
|
|
}
|
|
|
|
if r.logBegin != nil {
|
|
r.logBegin(prettyName, tf)
|
|
}
|
|
|
|
basePath := filename[:len(filename)-2]
|
|
specFilename := basePath + ".hcldec"
|
|
nativeFilename := basePath + ".hcl"
|
|
jsonFilename := basePath + ".hcl.json"
|
|
|
|
// We'll add the source code of the spec file to our own parser, even
|
|
// though it'll actually be parsed by the hcldec child process, since that
|
|
// way we can produce nice diagnostic messages if hcldec fails to process
|
|
// the spec file.
|
|
src, err := ioutil.ReadFile(specFilename)
|
|
if err == nil {
|
|
r.parser.AddFile(specFilename, &hcl.File{
|
|
Bytes: src,
|
|
})
|
|
}
|
|
|
|
if _, err := os.Stat(specFilename); err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Missing .hcldec file",
|
|
Detail: fmt.Sprintf("No specification file for test %s: %s.", prettyName, err),
|
|
})
|
|
return diags
|
|
}
|
|
|
|
if _, err := os.Stat(nativeFilename); err == nil {
|
|
moreDiags := r.runTestInput(specFilename, nativeFilename, tf)
|
|
diags = append(diags, moreDiags...)
|
|
}
|
|
|
|
if _, err := os.Stat(jsonFilename); err == nil {
|
|
moreDiags := r.runTestInput(specFilename, jsonFilename, tf)
|
|
diags = append(diags, moreDiags...)
|
|
}
|
|
|
|
if r.logProblems != nil {
|
|
r.logProblems(prettyName, nil, diags)
|
|
return nil // don't duplicate the diagnostics we already reported
|
|
}
|
|
|
|
return diags
|
|
}
|
|
|
|
func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile) hcl.Diagnostics {
|
|
// We'll add the source code of the input file to our own parser, even
|
|
// though it'll actually be parsed by the hcldec child process, since that
|
|
// way we can produce nice diagnostic messages if hcldec fails to process
|
|
// the input file.
|
|
src, err := ioutil.ReadFile(inputFilename)
|
|
if err == nil {
|
|
r.parser.AddFile(inputFilename, &hcl.File{
|
|
Bytes: src,
|
|
})
|
|
}
|
|
|
|
var diags hcl.Diagnostics
|
|
|
|
if tf.ChecksTraversals {
|
|
gotTraversals, moreDiags := r.hcldecVariables(specFilename, inputFilename)
|
|
diags = append(diags, moreDiags...)
|
|
if !moreDiags.HasErrors() {
|
|
expected := tf.ExpectedTraversals
|
|
for _, got := range gotTraversals {
|
|
e := findTraversalSpec(got, expected)
|
|
rng := got.SourceRange()
|
|
if e == nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Unexpected traversal",
|
|
Detail: "Detected traversal that is not indicated as expected in the test file.",
|
|
Subject: &rng,
|
|
})
|
|
} else {
|
|
moreDiags := checkTraversalsMatch(got, inputFilename, e)
|
|
diags = append(diags, moreDiags...)
|
|
}
|
|
}
|
|
|
|
// Look for any traversals that didn't show up at all.
|
|
for _, e := range expected {
|
|
if t := findTraversalForSpec(e, gotTraversals); t == nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Missing expected traversal",
|
|
Detail: "This expected traversal was not detected.",
|
|
Subject: e.Traversal.SourceRange().Ptr(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
val, transformDiags := r.hcldecTransform(specFilename, inputFilename)
|
|
if len(tf.ExpectedDiags) == 0 {
|
|
diags = append(diags, transformDiags...)
|
|
if transformDiags.HasErrors() {
|
|
// If hcldec failed then there's no point in continuing.
|
|
return diags
|
|
}
|
|
|
|
if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Incorrect result type",
|
|
Detail: fmt.Sprintf(
|
|
"Input file %s produced %s, but was expecting %s.",
|
|
inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType),
|
|
),
|
|
})
|
|
}
|
|
|
|
if tf.Result != cty.NilVal {
|
|
cmpVal, err := convert.Convert(tf.Result, tf.ResultType)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Incorrect type for result value",
|
|
Detail: fmt.Sprintf(
|
|
"Result does not conform to the given result type: %s.", err,
|
|
),
|
|
Subject: &tf.ResultRange,
|
|
})
|
|
} else {
|
|
if !val.RawEquals(cmpVal) {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Incorrect result value",
|
|
Detail: fmt.Sprintf(
|
|
"Input file %s produced %#v, but was expecting %#v.",
|
|
inputFilename, val, tf.Result,
|
|
),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// We're expecting diagnostics, and so we'll need to correlate the
|
|
// severities and source ranges of our actual diagnostics against
|
|
// what we were expecting.
|
|
type DiagnosticEntry struct {
|
|
Severity hcl.DiagnosticSeverity
|
|
Range hcl.Range
|
|
}
|
|
got := make(map[DiagnosticEntry]*hcl.Diagnostic)
|
|
want := make(map[DiagnosticEntry]hcl.Range)
|
|
for _, diag := range transformDiags {
|
|
if diag.Subject == nil {
|
|
// Sourceless diagnostics can never be expected, so we'll just
|
|
// pass these through as-is and assume they are hcldec
|
|
// operational errors.
|
|
diags = append(diags, diag)
|
|
continue
|
|
}
|
|
if diag.Subject.Filename != inputFilename {
|
|
// If the problem is for something other than the input file
|
|
// then it can't be expected.
|
|
diags = append(diags, diag)
|
|
continue
|
|
}
|
|
entry := DiagnosticEntry{
|
|
Severity: diag.Severity,
|
|
Range: *diag.Subject,
|
|
}
|
|
got[entry] = diag
|
|
}
|
|
for _, e := range tf.ExpectedDiags {
|
|
e.Range.Filename = inputFilename // assumed here, since we don't allow any other filename to be expected
|
|
entry := DiagnosticEntry{
|
|
Severity: e.Severity,
|
|
Range: e.Range,
|
|
}
|
|
want[entry] = e.DeclRange
|
|
}
|
|
|
|
for gotEntry, diag := range got {
|
|
if _, wanted := want[gotEntry]; !wanted {
|
|
// Pass through the diagnostic itself so the user can see what happened
|
|
diags = append(diags, diag)
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Unexpected diagnostic",
|
|
Detail: fmt.Sprintf(
|
|
"No %s diagnostic was expected %s. The unexpected diagnostic was shown above.",
|
|
severityString(gotEntry.Severity), rangeString(gotEntry.Range),
|
|
),
|
|
Subject: gotEntry.Range.Ptr(),
|
|
})
|
|
}
|
|
}
|
|
|
|
for wantEntry, declRange := range want {
|
|
if _, gotted := got[wantEntry]; !gotted {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Missing expected diagnostic",
|
|
Detail: fmt.Sprintf(
|
|
"No %s diagnostic was generated %s.",
|
|
severityString(wantEntry.Severity), rangeString(wantEntry.Range),
|
|
),
|
|
Subject: declRange.Ptr(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
return diags
|
|
}
|
|
|
|
func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Diagnostics) {
|
|
var diags hcl.Diagnostics
|
|
var outBuffer bytes.Buffer
|
|
var errBuffer bytes.Buffer
|
|
|
|
cmd := &exec.Cmd{
|
|
Path: r.hcldecPath,
|
|
Args: []string{
|
|
r.hcldecPath,
|
|
"--spec=" + specFile,
|
|
"--diags=json",
|
|
"--with-type",
|
|
inputFile,
|
|
},
|
|
Stdout: &outBuffer,
|
|
Stderr: &errBuffer,
|
|
}
|
|
err := cmd.Run()
|
|
if err != nil {
|
|
if _, isExit := err.(*exec.ExitError); !isExit {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to run hcldec",
|
|
Detail: fmt.Sprintf("Sub-program hcldec failed to start: %s.", err),
|
|
})
|
|
return cty.DynamicVal, diags
|
|
}
|
|
|
|
// If we exited unsuccessfully then we'll expect diagnostics on stderr
|
|
moreDiags := decodeJSONDiagnostics(errBuffer.Bytes())
|
|
diags = append(diags, moreDiags...)
|
|
return cty.DynamicVal, diags
|
|
} else {
|
|
// Otherwise, we expect a JSON result value on stdout. Since we used
|
|
// --with-type above, we can decode as DynamicPseudoType to recover
|
|
// exactly the type that was saved, without the usual JSON lossiness.
|
|
val, err := ctyjson.Unmarshal(outBuffer.Bytes(), cty.DynamicPseudoType)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec produced an invalid result: %s.", err),
|
|
})
|
|
return cty.DynamicVal, diags
|
|
}
|
|
return val, diags
|
|
}
|
|
}
|
|
|
|
func (r *Runner) hcldecVariables(specFile, inputFile string) ([]hcl.Traversal, hcl.Diagnostics) {
|
|
var diags hcl.Diagnostics
|
|
var outBuffer bytes.Buffer
|
|
var errBuffer bytes.Buffer
|
|
|
|
cmd := &exec.Cmd{
|
|
Path: r.hcldecPath,
|
|
Args: []string{
|
|
r.hcldecPath,
|
|
"--spec=" + specFile,
|
|
"--diags=json",
|
|
"--var-refs",
|
|
inputFile,
|
|
},
|
|
Stdout: &outBuffer,
|
|
Stderr: &errBuffer,
|
|
}
|
|
err := cmd.Run()
|
|
if err != nil {
|
|
if _, isExit := err.(*exec.ExitError); !isExit {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to run hcldec",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (evaluating input) failed to start: %s.", err),
|
|
})
|
|
return nil, diags
|
|
}
|
|
|
|
// If we exited unsuccessfully then we'll expect diagnostics on stderr
|
|
moreDiags := decodeJSONDiagnostics(errBuffer.Bytes())
|
|
diags = append(diags, moreDiags...)
|
|
return nil, diags
|
|
} else {
|
|
// Otherwise, we expect a JSON description of the traversals on stdout.
|
|
type PosJSON struct {
|
|
Line int `json:"line"`
|
|
Column int `json:"column"`
|
|
Byte int `json:"byte"`
|
|
}
|
|
type RangeJSON struct {
|
|
Filename string `json:"filename"`
|
|
Start PosJSON `json:"start"`
|
|
End PosJSON `json:"end"`
|
|
}
|
|
type StepJSON struct {
|
|
Kind string `json:"kind"`
|
|
Name string `json:"name,omitempty"`
|
|
Key json.RawMessage `json:"key,omitempty"`
|
|
Range RangeJSON `json:"range"`
|
|
}
|
|
type TraversalJSON struct {
|
|
Steps []StepJSON `json:"steps"`
|
|
}
|
|
|
|
var raw []TraversalJSON
|
|
err := json.Unmarshal(outBuffer.Bytes(), &raw)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: %s.", err),
|
|
})
|
|
return nil, diags
|
|
}
|
|
|
|
var ret []hcl.Traversal
|
|
if len(raw) == 0 {
|
|
return ret, diags
|
|
}
|
|
|
|
ret = make([]hcl.Traversal, 0, len(raw))
|
|
for _, rawT := range raw {
|
|
traversal := make(hcl.Traversal, 0, len(rawT.Steps))
|
|
for _, rawS := range rawT.Steps {
|
|
rng := hcl.Range{
|
|
Filename: rawS.Range.Filename,
|
|
Start: hcl.Pos{
|
|
Line: rawS.Range.Start.Line,
|
|
Column: rawS.Range.Start.Column,
|
|
Byte: rawS.Range.Start.Byte,
|
|
},
|
|
End: hcl.Pos{
|
|
Line: rawS.Range.End.Line,
|
|
Column: rawS.Range.End.Column,
|
|
Byte: rawS.Range.End.Byte,
|
|
},
|
|
}
|
|
|
|
switch rawS.Kind {
|
|
|
|
case "root":
|
|
traversal = append(traversal, hcl.TraverseRoot{
|
|
Name: rawS.Name,
|
|
SrcRange: rng,
|
|
})
|
|
|
|
case "attr":
|
|
traversal = append(traversal, hcl.TraverseAttr{
|
|
Name: rawS.Name,
|
|
SrcRange: rng,
|
|
})
|
|
|
|
case "index":
|
|
ty, err := ctyjson.ImpliedType([]byte(rawS.Key))
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step has invalid index key %s.", rawS.Key),
|
|
})
|
|
return nil, diags
|
|
}
|
|
keyVal, err := ctyjson.Unmarshal([]byte(rawS.Key), ty)
|
|
if err != nil {
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced a result with an invalid index key %s: %s.", rawS.Key, err),
|
|
})
|
|
return nil, diags
|
|
}
|
|
|
|
traversal = append(traversal, hcl.TraverseIndex{
|
|
Key: keyVal,
|
|
SrcRange: rng,
|
|
})
|
|
|
|
default:
|
|
// Should never happen since the above cases are exhaustive,
|
|
// but we'll catch it gracefully since this is coming from
|
|
// a possibly-buggy hcldec implementation that we're testing.
|
|
diags = append(diags, &hcl.Diagnostic{
|
|
Severity: hcl.DiagError,
|
|
Summary: "Failed to parse hcldec result",
|
|
Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step of unsupported kind %q.", rawS.Kind),
|
|
})
|
|
return nil, diags
|
|
}
|
|
}
|
|
|
|
ret = append(ret, traversal)
|
|
}
|
|
return ret, diags
|
|
}
|
|
}
|
|
|
|
func (r *Runner) prettyDirName(dir string) string {
|
|
rel, err := filepath.Rel(r.baseDir, dir)
|
|
if err != nil {
|
|
return filepath.ToSlash(dir)
|
|
}
|
|
return filepath.ToSlash(rel)
|
|
}
|
|
|
|
func (r *Runner) prettyTestName(filename string) string {
|
|
dir := filepath.Dir(filename)
|
|
dirName := r.prettyDirName(dir)
|
|
filename = filepath.Base(filename)
|
|
testName := filename[:len(filename)-2]
|
|
if dirName == "." {
|
|
return testName
|
|
}
|
|
return fmt.Sprintf("%s/%s", dirName, testName)
|
|
}
|