specsuite: Start of the harness for the specification test suite

cmd/hclspecsuite/README.md

@@ -0,0 +1,4 @@
+# `hclspecsuite`
+
+`hclspecsuite` is the test harness for
+[the HCL specification test suite](../../specsuite/README.md).

cmd/hclspecsuite/log.go

@@ -0,0 +1,3 @@
+package main
+
+type LogCallback func(testName string, testFile *TestFile)

cmd/hclspecsuite/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/hashicorp/hcl2/hclparse"
+
+	"github.com/hashicorp/hcl2/hcl"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+func main() {
+	os.Exit(realMain(os.Args[1:]))
+}
+
+func realMain(args []string) int {
+	if len(args) != 2 {
+		fmt.Fprintf(os.Stderr, "Usage: hclspecsuite <tests-dir> <hcldec-file>\n")
+		return 2
+	}
+
+	testsDir := args[0]
+	hcldecPath := args[1]
+
+	hcldecPath, err := exec.LookPath(hcldecPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		return 2
+	}
+
+	parser := hclparse.NewParser()
+
+	runner := &Runner{
+		parser:     parser,
+		hcldecPath: hcldecPath,
+		baseDir:    testsDir,
+		log: func(name string, file *TestFile) {
+			fmt.Printf("- %s\n", name)
+		},
+	}
+	diags := runner.Run()
+
+	if len(diags) != 0 {
+		os.Stderr.WriteString("\n")
+		color := terminal.IsTerminal(int(os.Stderr.Fd()))
+		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+		if err != nil {
+			w = 80
+		}
+		diagWr := hcl.NewDiagnosticTextWriter(os.Stderr, parser.Files(), uint(w), color)
+		diagWr.WriteDiagnostics(diags)
+		return 2
+	}
+
+	return 0
+}

cmd/hclspecsuite/runner.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/hcl2/hclparse"
+	"github.com/zclconf/go-cty/cty"
+)
+
+type Runner struct {
+	parser     *hclparse.Parser
+	hcldecPath string
+	baseDir    string
+	log        LogCallback
+}
+
+func (r *Runner) Run() hcl.Diagnostics {
+	return r.runDir(r.baseDir)
+}
+
+func (r *Runner) runDir(dir string) hcl.Diagnostics {
+	var diags hcl.Diagnostics
+
+	infos, err := ioutil.ReadDir(dir)
+	if err != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Failed to read test directory",
+			Detail:   fmt.Sprintf("The directory %q could not be opened: %s.", dir, err),
+		})
+		return diags
+	}
+
+	var tests []string
+	var subDirs []string
+	for _, info := range infos {
+		name := info.Name()
+		if strings.HasPrefix(name, ".") {
+			continue
+		}
+
+		if info.IsDir() {
+			subDirs = append(subDirs, name)
+		}
+		if strings.HasSuffix(name, ".t") {
+			tests = append(tests, name)
+		}
+	}
+	sort.Strings(tests)
+	sort.Strings(subDirs)
+
+	for _, filename := range tests {
+		filename = filepath.Join(r.baseDir, filename)
+		testDiags := r.runTest(filename)
+		diags = append(diags, testDiags...)
+	}
+
+	for _, dirName := range subDirs {
+		dir := filepath.Join(r.baseDir, dirName)
+		dirDiags := r.runDir(dir)
+		diags = append(diags, dirDiags...)
+	}
+
+	return diags
+}
+
+func (r *Runner) runTest(filename string) hcl.Diagnostics {
+	prettyName := r.prettyTestName(filename)
+	tf, diags := r.LoadTestFile(filename)
+	if diags.HasErrors() {
+		// We'll still log, so it's clearer which test the diagnostics belong to.
+		if r.log != nil {
+			r.log(prettyName, nil)
+		}
+		return diags
+	}
+
+	if r.log != nil {
+		r.log(prettyName, tf)
+	}
+
+	basePath := filename[:len(filename)-2]
+	specFilename := basePath + ".hcldec"
+	nativeFilename := basePath + ".hcl"
+	//jsonFilename := basePath + ".hcl.json"
+
+	if _, err := os.Stat(specFilename); err != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Missing .hcldec file",
+			Detail:   fmt.Sprintf("No specification file for test %s: %s.", prettyName, err),
+		})
+		return diags
+	}
+
+	if _, err := os.Stat(nativeFilename); err == nil {
+
+	}
+
+	return diags
+}
+
+func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Diagnostics) {
+	var diags hcl.Diagnostics
+	var outBuffer bytes.Buffer
+	var errBuffer bytes.Buffer
+
+	cmd := &exec.Cmd{
+		Path: r.hcldecPath,
+		Args: []string{
+			"--spec=" + specFile,
+			"--diags=json",
+			inputFile,
+		},
+		Stdout: &outBuffer,
+		Stderr: &errBuffer,
+	}
+	err := cmd.Run()
+	if _, isExit := err.(*exec.ExitError); !isExit {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Failed to run hcldec",
+			Detail:   fmt.Sprintf("Sub-program hcldec failed to start: %s.", err),
+		})
+		return cty.DynamicVal, diags
+	}
+
+	if err != nil {
+		// If we exited unsuccessfully then we'll expect diagnostics on stderr
+		// TODO: implement that
+	} else {
+		// Otherwise, we expect a JSON result value on stdout
+		// TODO: implement that
+	}
+
+	return cty.DynamicVal, diags
+}
+
+func (r *Runner) prettyDirName(dir string) string {
+	rel, err := filepath.Rel(r.baseDir, dir)
+	if err != nil {
+		return filepath.ToSlash(dir)
+	}
+	return filepath.ToSlash(rel)
+}
+
+func (r *Runner) prettyTestName(filename string) string {
+	dir := filepath.Dir(filename)
+	dirName := r.prettyDirName(dir)
+	filename = filepath.Base(filename)
+	testName := filename[:len(filename)-2]
+	if dirName == "." {
+		return testName
+	}
+	return fmt.Sprintf("%s/%s", dirName, testName)
+}

cmd/hclspecsuite/test_file.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/hcl2/ext/typeexpr"
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/convert"
+)
+
+type TestFile struct {
+	Result     cty.Value
+	ResultType cty.Type
+
+	Traversals []hcl.Traversal
+}
+
+func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) {
+	f, diags := r.parser.ParseHCLFile(filename)
+	if diags.HasErrors() {
+		return nil, diags
+	}
+
+	content, moreDiags := f.Body.Content(testFileSchema)
+	diags = append(diags, moreDiags...)
+	if moreDiags.HasErrors() {
+		return nil, diags
+	}
+
+	ret := &TestFile{
+		ResultType: cty.DynamicPseudoType,
+	}
+
+	if typeAttr, exists := content.Attributes["result_type"]; exists {
+		ty, moreDiags := typeexpr.TypeConstraint(typeAttr.Expr)
+		diags = append(diags, moreDiags...)
+		if !moreDiags.HasErrors() {
+			ret.ResultType = ty
+		}
+	}
+
+	if resultAttr, exists := content.Attributes["result"]; exists {
+		resultVal, moreDiags := resultAttr.Expr.Value(nil)
+		diags = append(diags, moreDiags...)
+		if !moreDiags.HasErrors() {
+			resultVal, err := convert.Convert(resultVal, ret.ResultType)
+			if err != nil {
+				diags = diags.Append(&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid result value",
+					Detail:   fmt.Sprintf("The result value does not conform to the given result type: %s.", err),
+					Subject:  resultAttr.Expr.Range().Ptr(),
+				})
+			} else {
+				ret.Result = resultVal
+			}
+		}
+	}
+
+	return ret, diags
+}
+
+var testFileSchema = &hcl.BodySchema{
+	Attributes: []hcl.AttributeSchema{
+		{
+			Name: "result",
+		},
+		{
+			Name: "result_type",
+		},
+	},
+	Blocks: []hcl.BlockHeaderSchema{
+		{
+			Type: "traversals",
+		},
+	},
+}

specsuite/README.md

@@ -0,0 +1,37 @@
+# HCL Language Test Suite
+
+This directory contains an implementation-agnostic test suite that can be used
+to verify the correct behavior not only of the HCL implementation in _this_
+repository but also of possible other implementations.
+
+The harness for running this suite -- a Go program in this directory -- uses
+the `hcldec` program as a level of abstraction to avoid depending directly on
+the Go implementation. As a result, other HCL implementations must also
+include a version of `hcldec` in order to run this spec.
+
+The tests defined in this suite each correspond to a detail of
+[the HCL spec](../hcl/spec.md). This suite is separate from and not a
+substitute for direct unit tests of a given implementation that would presumably
+also exercise that implementation's own programmatic API.
+
+To run the suite, first build the harness using Go:
+
+```
+go install github.com/hashicorp/hcl2/cmd/hclspecsuite
+```
+
+Then run it, passing it the directory containing the test definitions (the
+"tests" subdirectory of this directory) and the path to the `hcldec` executable
+to use.
+
+For example, if working in the root of this repository and using the `hcldec`
+implementation from here:
+
+```
+go install ./cmd/hcldec
+hclspecsuite ./specsuite/tests $GOPATH/bin/hcldec
+```
+
+For developers working on the Go implementation of HCL from this repository,
+please note that this spec suite is run as part of a normal `go test ./...`
+execution for this whole repository and so does not need to be run separately.

specsuite/tests/empty.hcl.json

@@ -0,0 +1 @@
+{}

specsuite/tests/empty.hcldec

@@ -0,0 +1,3 @@
+literal {
+  value = "ok"
+}

specsuite/tests/empty.t

@@ -0,0 +1,9 @@
+# This test ensures that we can successfully parse an empty file.
+# Since an empty file has no content, the hcldec spec for this test is
+# just a literal value, which we test below.
+
+result = "ok"
+
+traversals {
+  # Explicitly no traversals
+}