cmd/hclspecsuite: basic runner functionality for successful cases

The harness can now run tests that decode successfully and compare the
result with a given value. Further work is required in later commits to
deal with other cases, such as tests that intentionally produce errors.

cmd/hcldec/main.go

@@ -28,6 +28,7 @@ var (
 	outputFile  = flag.StringP("out", "o", "", "write to the given file, instead of stdout")
 	diagsFormat = flag.StringP("diags", "", "", "format any returned diagnostics in the given format; currently only \"json\" is accepted")
 	showVarRefs = flag.BoolP("var-refs", "", false, "rather than decoding input, produce a JSON description of the variables referenced by it")
+	withType    = flag.BoolP("with-type", "", false, "include an additional object level at the top describing the HCL-oriented type of the result value")
 	showVersion = flag.BoolP("version", "v", false, "show the version number and immediately exit")
 )
 
@@ -146,7 +147,13 @@ func realmain(args []string) error {
 		}
 	} else {
 		for _, filename := range args {
-			f, fDiags := parser.ParseHCLFile(filename)
+			var f *hcl.File
+			var fDiags hcl.Diagnostics
+			if strings.HasSuffix(filename, ".json") {
+				f, fDiags = parser.ParseJSONFile(filename)
+			} else {
+				f, fDiags = parser.ParseHCLFile(filename)
+			}
 			diags = append(diags, fDiags...)
 			if !fDiags.HasErrors() {
 				bodies = append(bodies, f.Body)
@@ -185,7 +192,13 @@ func realmain(args []string) error {
 		os.Exit(2)
 	}
 
-	out, err := ctyjson.Marshal(val, val.Type())
+	wantType := val.Type()
+	if *withType {
+		// We'll instead ask to encode as dynamic, which will make the
+		// marshaler include type information.
+		wantType = cty.DynamicPseudoType
+	}
+	out, err := ctyjson.Marshal(val, wantType)
 	if err != nil {
 		return err
 	}

cmd/hclspecsuite/diagnostics.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/hashicorp/hcl2/hcl"
+)
+
+func decodeJSONDiagnostics(src []byte) hcl.Diagnostics {
+	type PosJSON struct {
+		Line   int `json:"line"`
+		Column int `json:"column"`
+		Byte   int `json:"byte"`
+	}
+	type RangeJSON struct {
+		Filename string  `json:"filename"`
+		Start    PosJSON `json:"start"`
+		End      PosJSON `json:"end"`
+	}
+	type DiagnosticJSON struct {
+		Severity string     `json:"severity"`
+		Summary  string     `json:"summary"`
+		Detail   string     `json:"detail,omitempty"`
+		Subject  *RangeJSON `json:"subject,omitempty"`
+	}
+	type DiagnosticsJSON struct {
+		Diagnostics []DiagnosticJSON `json:"diagnostics"`
+	}
+
+	var raw DiagnosticsJSON
+	var diags hcl.Diagnostics
+	err := json.Unmarshal(src, &raw)
+	if err != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Failed to parse hcldec diagnostics result",
+			Detail:   fmt.Sprintf("Sub-program hcldec produced invalid diagnostics: %s.", err),
+		})
+		return diags
+	}
+
+	if len(raw.Diagnostics) == 0 {
+		return nil
+	}
+
+	diags = make(hcl.Diagnostics, 0, len(raw.Diagnostics))
+	for _, rawDiag := range raw.Diagnostics {
+		var severity hcl.DiagnosticSeverity
+		switch rawDiag.Severity {
+		case "error":
+			severity = hcl.DiagError
+		case "warning":
+			severity = hcl.DiagWarning
+		default:
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to parse hcldec diagnostics result",
+				Detail:   fmt.Sprintf("Diagnostic has unsupported severity %q.", rawDiag.Severity),
+			})
+			continue
+		}
+
+		diag := &hcl.Diagnostic{
+			Severity: severity,
+			Summary:  rawDiag.Summary,
+			Detail:   rawDiag.Detail,
+		}
+		if rawDiag.Subject != nil {
+			rawRange := rawDiag.Subject
+			diag.Subject = &hcl.Range{
+				Filename: rawRange.Filename,
+				Start: hcl.Pos{
+					Line:   rawRange.Start.Line,
+					Column: rawRange.Start.Column,
+					Byte:   rawRange.Start.Byte,
+				},
+				End: hcl.Pos{
+					Line:   rawRange.End.Line,
+					Column: rawRange.End.Column,
+					Byte:   rawRange.End.Byte,
+				},
+			}
+		}
+		diags = append(diags, diag)
+	}
+
+	return diags
+}

cmd/hclspecsuite/runner.go

@@ -10,9 +10,13 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/hashicorp/hcl2/ext/typeexpr"
+
 	"github.com/hashicorp/hcl2/hcl"
 	"github.com/hashicorp/hcl2/hclparse"
 	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/convert"
+	ctyjson "github.com/zclconf/go-cty/cty/json"
 )
 
 type Runner struct {
@@ -58,13 +62,13 @@ func (r *Runner) runDir(dir string) hcl.Diagnostics {
 	sort.Strings(subDirs)
 
 	for _, filename := range tests {
-		filename = filepath.Join(r.baseDir, filename)
+		filename = filepath.Join(dir, filename)
 		testDiags := r.runTest(filename)
 		diags = append(diags, testDiags...)
 	}
 
 	for _, dirName := range subDirs {
-		dir := filepath.Join(r.baseDir, dirName)
+		dir := filepath.Join(dir, dirName)
 		dirDiags := r.runDir(dir)
 		diags = append(diags, dirDiags...)
 	}
@@ -90,7 +94,7 @@ func (r *Runner) runTest(filename string) hcl.Diagnostics {
 	basePath := filename[:len(filename)-2]
 	specFilename := basePath + ".hcldec"
 	nativeFilename := basePath + ".hcl"
-	//jsonFilename := basePath + ".hcl.json"
+	jsonFilename := basePath + ".hcl.json"
 
 	if _, err := os.Stat(specFilename); err != nil {
 		diags = append(diags, &hcl.Diagnostic{
@@ -102,7 +106,72 @@ func (r *Runner) runTest(filename string) hcl.Diagnostics {
 	}
 
 	if _, err := os.Stat(nativeFilename); err == nil {
+		moreDiags := r.runTestInput(specFilename, nativeFilename, tf)
+		diags = append(diags, moreDiags...)
+	}
 
+	if _, err := os.Stat(jsonFilename); err == nil {
+		moreDiags := r.runTestInput(specFilename, jsonFilename, tf)
+		diags = append(diags, moreDiags...)
+	}
+
+	return diags
+}
+
+func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile) hcl.Diagnostics {
+	// We'll add the source code of the input file to our own parser, even
+	// though it'll actually be parsed by the hcldec child process, since that
+	// way we can produce nice diagnostic messages if hcldec fails to process
+	// the input file.
+	if src, err := ioutil.ReadFile(inputFilename); err == nil {
+		r.parser.AddFile(inputFilename, &hcl.File{
+			Bytes: src,
+		})
+	}
+
+	var diags hcl.Diagnostics
+
+	val, moreDiags := r.hcldecTransform(specFilename, inputFilename)
+	diags = append(diags, moreDiags...)
+	if moreDiags.HasErrors() {
+		// If hcldec failed then there's no point in continuing.
+		return diags
+	}
+
+	if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Incorrect result type",
+			Detail: fmt.Sprintf(
+				"Input file %s produced %s, but was expecting %s.",
+				inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType),
+			),
+		})
+	}
+
+	if tf.Result != cty.NilVal {
+		cmpVal, err := convert.Convert(tf.Result, tf.ResultType)
+		if err != nil {
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Incorrect type for result value",
+				Detail: fmt.Sprintf(
+					"Result does not conform to the given result type: %s.", err,
+				),
+				Subject: &tf.ResultRange,
+			})
+		} else {
+			if !val.RawEquals(cmpVal) {
+				diags = append(diags, &hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Incorrect result value",
+					Detail: fmt.Sprintf(
+						"Input file %s produced %#v, but was expecting %#v.",
+						inputFilename, val, tf.Result,
+					),
+				})
+			}
+		}
 	}
 
 	return diags
@@ -116,32 +185,45 @@ func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Dia
 	cmd := &exec.Cmd{
 		Path: r.hcldecPath,
 		Args: []string{
+			r.hcldecPath,
 			"--spec=" + specFile,
 			"--diags=json",
+			"--with-type",
 			inputFile,
 		},
 		Stdout: &outBuffer,
 		Stderr: &errBuffer,
 	}
 	err := cmd.Run()
-	if _, isExit := err.(*exec.ExitError); !isExit {
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Failed to run hcldec",
-			Detail:   fmt.Sprintf("Sub-program hcldec failed to start: %s.", err),
-		})
-		return cty.DynamicVal, diags
-	}
-
 	if err != nil {
-		// If we exited unsuccessfully then we'll expect diagnostics on stderr
+		if _, isExit := err.(*exec.ExitError); !isExit {
-		// TODO: implement that
+			diags = append(diags, &hcl.Diagnostic{
-	} else {
+				Severity: hcl.DiagError,
-		// Otherwise, we expect a JSON result value on stdout
+				Summary:  "Failed to run hcldec",
-		// TODO: implement that
+				Detail:   fmt.Sprintf("Sub-program hcldec failed to start: %s.", err),
-	}
+			})
+			return cty.DynamicVal, diags
+		}
 
-	return cty.DynamicVal, diags
+		// If we exited unsuccessfully then we'll expect diagnostics on stderr
+		moreDiags := decodeJSONDiagnostics(errBuffer.Bytes())
+		diags = append(diags, moreDiags...)
+		return cty.DynamicVal, diags
+	} else {
+		// Otherwise, we expect a JSON result value on stdout. Since we used
+		// --with-type above, we can decode as DynamicPseudoType to recover
+		// exactly the type that was saved, without the usual JSON lossiness.
+		val, err := ctyjson.Unmarshal(outBuffer.Bytes(), cty.DynamicPseudoType)
+		if err != nil {
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to parse hcldec result",
+				Detail:   fmt.Sprintf("Sub-program hcldec produced an invalid result: %s.", err),
+			})
+			return cty.DynamicVal, diags
+		}
+		return val, diags
+	}
 }
 
 func (r *Runner) prettyDirName(dir string) string {

cmd/hclspecsuite/test_file.go

@@ -14,6 +14,9 @@ type TestFile struct {
 	ResultType cty.Type
 
 	Traversals []hcl.Traversal
+
+	ResultRange     hcl.Range
+	ResultTypeRange hcl.Range
 }
 
 func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) {
@@ -38,6 +41,7 @@ func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) {
 		if !moreDiags.HasErrors() {
 			ret.ResultType = ty
 		}
+		ret.ResultTypeRange = typeAttr.Expr.Range()
 	}
 
 	if resultAttr, exists := content.Attributes["result"]; exists {
@@ -56,6 +60,7 @@ func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) {
 				ret.Result = resultVal
 			}
 		}
+		ret.ResultRange = resultAttr.Expr.Range()
 	}
 
 	return ret, diags