From 48039c0368f075eb495ed52dbf05ef2b1e3d7854 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 10 Aug 2018 08:49:43 -0700 Subject: [PATCH] cmd/hclspecsuite: basic runner functionality for successful cases The harness can now run tests that decode successfully and compare the result with a given value. Further work is required in later commits to deal with other cases, such as tests that intentionally produce errors. --- cmd/hcldec/main.go | 17 ++++- cmd/hclspecsuite/diagnostics.go | 89 +++++++++++++++++++++++ cmd/hclspecsuite/runner.go | 120 +++++++++++++++++++++++++++----- cmd/hclspecsuite/test_file.go | 5 ++ 4 files changed, 210 insertions(+), 21 deletions(-) create mode 100644 cmd/hclspecsuite/diagnostics.go diff --git a/cmd/hcldec/main.go b/cmd/hcldec/main.go index 4f14226..16dbb03 100644 --- a/cmd/hcldec/main.go +++ b/cmd/hcldec/main.go @@ -28,6 +28,7 @@ var ( outputFile = flag.StringP("out", "o", "", "write to the given file, instead of stdout") diagsFormat = flag.StringP("diags", "", "", "format any returned diagnostics in the given format; currently only \"json\" is accepted") showVarRefs = flag.BoolP("var-refs", "", false, "rather than decoding input, produce a JSON description of the variables referenced by it") + withType = flag.BoolP("with-type", "", false, "include an additional object level at the top describing the HCL-oriented type of the result value") showVersion = flag.BoolP("version", "v", false, "show the version number and immediately exit") ) @@ -146,7 +147,13 @@ func realmain(args []string) error { } } else { for _, filename := range args { - f, fDiags := parser.ParseHCLFile(filename) + var f *hcl.File + var fDiags hcl.Diagnostics + if strings.HasSuffix(filename, ".json") { + f, fDiags = parser.ParseJSONFile(filename) + } else { + f, fDiags = parser.ParseHCLFile(filename) + } diags = append(diags, fDiags...) if !fDiags.HasErrors() { bodies = append(bodies, f.Body) @@ -185,7 +192,13 @@ func realmain(args []string) error { os.Exit(2) } - out, err := ctyjson.Marshal(val, val.Type()) + wantType := val.Type() + if *withType { + // We'll instead ask to encode as dynamic, which will make the + // marshaler include type information. + wantType = cty.DynamicPseudoType + } + out, err := ctyjson.Marshal(val, wantType) if err != nil { return err } diff --git a/cmd/hclspecsuite/diagnostics.go b/cmd/hclspecsuite/diagnostics.go new file mode 100644 index 0000000..802c0cb --- /dev/null +++ b/cmd/hclspecsuite/diagnostics.go @@ -0,0 +1,89 @@ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/hashicorp/hcl2/hcl" +) + +func decodeJSONDiagnostics(src []byte) hcl.Diagnostics { + type PosJSON struct { + Line int `json:"line"` + Column int `json:"column"` + Byte int `json:"byte"` + } + type RangeJSON struct { + Filename string `json:"filename"` + Start PosJSON `json:"start"` + End PosJSON `json:"end"` + } + type DiagnosticJSON struct { + Severity string `json:"severity"` + Summary string `json:"summary"` + Detail string `json:"detail,omitempty"` + Subject *RangeJSON `json:"subject,omitempty"` + } + type DiagnosticsJSON struct { + Diagnostics []DiagnosticJSON `json:"diagnostics"` + } + + var raw DiagnosticsJSON + var diags hcl.Diagnostics + err := json.Unmarshal(src, &raw) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to parse hcldec diagnostics result", + Detail: fmt.Sprintf("Sub-program hcldec produced invalid diagnostics: %s.", err), + }) + return diags + } + + if len(raw.Diagnostics) == 0 { + return nil + } + + diags = make(hcl.Diagnostics, 0, len(raw.Diagnostics)) + for _, rawDiag := range raw.Diagnostics { + var severity hcl.DiagnosticSeverity + switch rawDiag.Severity { + case "error": + severity = hcl.DiagError + case "warning": + severity = hcl.DiagWarning + default: + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to parse hcldec diagnostics result", + Detail: fmt.Sprintf("Diagnostic has unsupported severity %q.", rawDiag.Severity), + }) + continue + } + + diag := &hcl.Diagnostic{ + Severity: severity, + Summary: rawDiag.Summary, + Detail: rawDiag.Detail, + } + if rawDiag.Subject != nil { + rawRange := rawDiag.Subject + diag.Subject = &hcl.Range{ + Filename: rawRange.Filename, + Start: hcl.Pos{ + Line: rawRange.Start.Line, + Column: rawRange.Start.Column, + Byte: rawRange.Start.Byte, + }, + End: hcl.Pos{ + Line: rawRange.End.Line, + Column: rawRange.End.Column, + Byte: rawRange.End.Byte, + }, + } + } + diags = append(diags, diag) + } + + return diags +} diff --git a/cmd/hclspecsuite/runner.go b/cmd/hclspecsuite/runner.go index 0be1822..97b8fbb 100644 --- a/cmd/hclspecsuite/runner.go +++ b/cmd/hclspecsuite/runner.go @@ -10,9 +10,13 @@ import ( "sort" "strings" + "github.com/hashicorp/hcl2/ext/typeexpr" + "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hclparse" "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" + ctyjson "github.com/zclconf/go-cty/cty/json" ) type Runner struct { @@ -58,13 +62,13 @@ func (r *Runner) runDir(dir string) hcl.Diagnostics { sort.Strings(subDirs) for _, filename := range tests { - filename = filepath.Join(r.baseDir, filename) + filename = filepath.Join(dir, filename) testDiags := r.runTest(filename) diags = append(diags, testDiags...) } for _, dirName := range subDirs { - dir := filepath.Join(r.baseDir, dirName) + dir := filepath.Join(dir, dirName) dirDiags := r.runDir(dir) diags = append(diags, dirDiags...) } @@ -90,7 +94,7 @@ func (r *Runner) runTest(filename string) hcl.Diagnostics { basePath := filename[:len(filename)-2] specFilename := basePath + ".hcldec" nativeFilename := basePath + ".hcl" - //jsonFilename := basePath + ".hcl.json" + jsonFilename := basePath + ".hcl.json" if _, err := os.Stat(specFilename); err != nil { diags = append(diags, &hcl.Diagnostic{ @@ -102,7 +106,72 @@ func (r *Runner) runTest(filename string) hcl.Diagnostics { } if _, err := os.Stat(nativeFilename); err == nil { + moreDiags := r.runTestInput(specFilename, nativeFilename, tf) + diags = append(diags, moreDiags...) + } + if _, err := os.Stat(jsonFilename); err == nil { + moreDiags := r.runTestInput(specFilename, jsonFilename, tf) + diags = append(diags, moreDiags...) + } + + return diags +} + +func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile) hcl.Diagnostics { + // We'll add the source code of the input file to our own parser, even + // though it'll actually be parsed by the hcldec child process, since that + // way we can produce nice diagnostic messages if hcldec fails to process + // the input file. + if src, err := ioutil.ReadFile(inputFilename); err == nil { + r.parser.AddFile(inputFilename, &hcl.File{ + Bytes: src, + }) + } + + var diags hcl.Diagnostics + + val, moreDiags := r.hcldecTransform(specFilename, inputFilename) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + // If hcldec failed then there's no point in continuing. + return diags + } + + if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Incorrect result type", + Detail: fmt.Sprintf( + "Input file %s produced %s, but was expecting %s.", + inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType), + ), + }) + } + + if tf.Result != cty.NilVal { + cmpVal, err := convert.Convert(tf.Result, tf.ResultType) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Incorrect type for result value", + Detail: fmt.Sprintf( + "Result does not conform to the given result type: %s.", err, + ), + Subject: &tf.ResultRange, + }) + } else { + if !val.RawEquals(cmpVal) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Incorrect result value", + Detail: fmt.Sprintf( + "Input file %s produced %#v, but was expecting %#v.", + inputFilename, val, tf.Result, + ), + }) + } + } } return diags @@ -116,32 +185,45 @@ func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Dia cmd := &exec.Cmd{ Path: r.hcldecPath, Args: []string{ + r.hcldecPath, "--spec=" + specFile, "--diags=json", + "--with-type", inputFile, }, Stdout: &outBuffer, Stderr: &errBuffer, } err := cmd.Run() - if _, isExit := err.(*exec.ExitError); !isExit { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to run hcldec", - Detail: fmt.Sprintf("Sub-program hcldec failed to start: %s.", err), - }) - return cty.DynamicVal, diags - } - if err != nil { - // If we exited unsuccessfully then we'll expect diagnostics on stderr - // TODO: implement that - } else { - // Otherwise, we expect a JSON result value on stdout - // TODO: implement that - } + if _, isExit := err.(*exec.ExitError); !isExit { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to run hcldec", + Detail: fmt.Sprintf("Sub-program hcldec failed to start: %s.", err), + }) + return cty.DynamicVal, diags + } - return cty.DynamicVal, diags + // If we exited unsuccessfully then we'll expect diagnostics on stderr + moreDiags := decodeJSONDiagnostics(errBuffer.Bytes()) + diags = append(diags, moreDiags...) + return cty.DynamicVal, diags + } else { + // Otherwise, we expect a JSON result value on stdout. Since we used + // --with-type above, we can decode as DynamicPseudoType to recover + // exactly the type that was saved, without the usual JSON lossiness. + val, err := ctyjson.Unmarshal(outBuffer.Bytes(), cty.DynamicPseudoType) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to parse hcldec result", + Detail: fmt.Sprintf("Sub-program hcldec produced an invalid result: %s.", err), + }) + return cty.DynamicVal, diags + } + return val, diags + } } func (r *Runner) prettyDirName(dir string) string { diff --git a/cmd/hclspecsuite/test_file.go b/cmd/hclspecsuite/test_file.go index 98371ee..e6f2191 100644 --- a/cmd/hclspecsuite/test_file.go +++ b/cmd/hclspecsuite/test_file.go @@ -14,6 +14,9 @@ type TestFile struct { ResultType cty.Type Traversals []hcl.Traversal + + ResultRange hcl.Range + ResultTypeRange hcl.Range } func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) { @@ -38,6 +41,7 @@ func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) { if !moreDiags.HasErrors() { ret.ResultType = ty } + ret.ResultTypeRange = typeAttr.Expr.Range() } if resultAttr, exists := content.Attributes["result"]; exists { @@ -56,6 +60,7 @@ func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) { ret.Result = resultVal } } + ret.ResultRange = resultAttr.Expr.Range() } return ret, diags