hcl/spectests: run the spec testsuite as part of "go test"

Although the spec testsuite and associated harness is designed to be
usable by other implementations of HCL not written in Go, it's convenient
to run it as part of our own "go test" test suite here so there isn't
an additional thing to run on each change.

To achieve this, the new package hcl/spectests will build both hcldec and
hclspecsuite from latest source and then run the latter to execute the
test suite, capturing the output and converting it (sloppily) into
testing.T method calls to produce something vaguely reasonable.

Other than the small amount of "parsing" to make it look in the output
like a normal Go test, there's nothing special going on here and so it's
still valid to run the spec suite manually with a build of hcldec from
this codebase, which should produce the same result.

cmd/hclspecsuite/log.go

@@ -1,3 +1,8 @@
 package main
 
-type LogCallback func(testName string, testFile *TestFile)
+import (
+	"github.com/hashicorp/hcl2/hcl"
+)
+
+type LogBeginCallback func(testName string, testFile *TestFile)
+type LogProblemsCallback func(testName string, testFile *TestFile, diags hcl.Diagnostics)

cmd/hclspecsuite/main.go

@@ -32,27 +32,40 @@ func realMain(args []string) int {
 
 	parser := hclparse.NewParser()
 
-	runner := &Runner{
-		parser:     parser,
-		hcldecPath: hcldecPath,
-		baseDir:    testsDir,
-		log: func(name string, file *TestFile) {
-			fmt.Printf("- %s\n", name)
-		},
-	}
-	diags := runner.Run()
-
-	if len(diags) != 0 {
-		os.Stderr.WriteString("\n")
 	color := terminal.IsTerminal(int(os.Stderr.Fd()))
 	w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
 	if err != nil {
 		w = 80
 	}
 	diagWr := hcl.NewDiagnosticTextWriter(os.Stderr, parser.Files(), uint(w), color)
+	var diagCount int
+
+	runner := &Runner{
+		parser:     parser,
+		hcldecPath: hcldecPath,
+		baseDir:    testsDir,
+		logBegin: func(name string, file *TestFile) {
+			fmt.Printf("- %s\n", name)
+		},
+		logProblems: func(name string, file *TestFile, diags hcl.Diagnostics) {
+			if len(diags) != 0 {
+				os.Stderr.WriteString("\n")
 				diagWr.WriteDiagnostics(diags)
-		return 2
+				diagCount += len(diags)
+			}
+			fmt.Printf("- %s\n", name)
+		},
+	}
+	diags := runner.Run()
+
+	if len(diags) != 0 {
+		os.Stderr.WriteString("\n\n\n== Test harness problems:\n\n")
+		diagWr.WriteDiagnostics(diags)
+		diagCount += len(diags)
 	}
 
+	if diagCount > 0 {
+		return 2
+	}
 	return 0
 }

cmd/hclspecsuite/runner.go

@@ -24,7 +24,8 @@ type Runner struct {
 	parser      *hclparse.Parser
 	hcldecPath  string
 	baseDir     string
-	log        LogCallback
+	logBegin    LogBeginCallback
+	logProblems LogProblemsCallback
 }
 
 func (r *Runner) Run() hcl.Diagnostics {
@@ -82,14 +83,18 @@ func (r *Runner) runTest(filename string) hcl.Diagnostics {
 	tf, diags := r.LoadTestFile(filename)
 	if diags.HasErrors() {
 		// We'll still log, so it's clearer which test the diagnostics belong to.
-		if r.log != nil {
+		if r.logBegin != nil {
-			r.log(prettyName, nil)
+			r.logBegin(prettyName, nil)
+		}
+		if r.logProblems != nil {
+			r.logProblems(prettyName, nil, diags)
+			return nil // don't duplicate the diagnostics we already reported
 		}
 		return diags
 	}
 
-	if r.log != nil {
+	if r.logBegin != nil {
-		r.log(prettyName, tf)
+		r.logBegin(prettyName, tf)
 	}
 
 	basePath := filename[:len(filename)-2]
@@ -127,6 +132,11 @@ func (r *Runner) runTest(filename string) hcl.Diagnostics {
 		diags = append(diags, moreDiags...)
 	}
 
+	if r.logProblems != nil {
+		r.logProblems(prettyName, nil, diags)
+		return nil // don't duplicate the diagnostics we already reported
+	}
+
 	return diags
 }
 

hcl/spectests/.gitignore

@@ -0,0 +1 @@
+tmp_*

hcl/spectests/spec_test.go

@@ -0,0 +1,100 @@
+package spectests
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	// The test harness is an external program that also expects to have
+	// hcldec built as an external program, so we'll build both into
+	// temporary files in our working directory before running our tests
+	// here, to ensure that we're always running a build of the latest code.
+	err := build()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+		os.Exit(1)
+	}
+
+	// Now we can run the tests
+	os.Exit(m.Run())
+}
+
+func build() error {
+	err := goBuild("github.com/hashicorp/hcl2/cmd/hcldec", "tmp_hcldec")
+	if err != nil {
+		return fmt.Errorf("error building hcldec: %s", err)
+	}
+
+	err = goBuild("github.com/hashicorp/hcl2/cmd/hclspecsuite", "tmp_hclspecsuite")
+	if err != nil {
+		return fmt.Errorf("error building hcldec: %s", err)
+	}
+
+	return nil
+}
+
+func TestSpec(t *testing.T) {
+	suiteDir := filepath.Clean("../../specsuite/tests")
+	harness := "./tmp_hclspecsuite"
+	hcldec := "./tmp_hcldec"
+
+	cmd := exec.Command(harness, suiteDir, hcldec)
+	out, err := cmd.CombinedOutput()
+	if _, isExit := err.(*exec.ExitError); err != nil && !isExit {
+		t.Errorf("failed to run harness: %s", err)
+	}
+	failed := err != nil
+
+	sc := bufio.NewScanner(bytes.NewReader(out))
+	var lines []string
+	for sc.Scan() {
+		lines = append(lines, sc.Text())
+	}
+
+	i := 0
+	for i < len(lines) {
+		cur := lines[i]
+		if strings.HasPrefix(cur, "- ") {
+			testName := cur[2:]
+			t.Run(testName, func(t *testing.T) {
+				i++
+				for i < len(lines) {
+					cur := lines[i]
+					if strings.HasPrefix(cur, "- ") || strings.HasPrefix(cur, "==") {
+						return
+					}
+					t.Error(cur)
+					i++
+				}
+			})
+		} else {
+			if !strings.HasPrefix(cur, "==") { // not the "test harness problems" report, then
+				t.Log(cur)
+			}
+			i++
+		}
+	}
+
+	if failed {
+		t.Error("specsuite failed")
+	}
+}
+
+func goBuild(pkg, outFile string) error {
+	if runtime.GOOS == "windows" {
+		outFile += ".exe"
+	}
+
+	cmd := exec.Command("go", "build", "-i", "-o", outFile, pkg)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+	return cmd.Run()
+}