From 0956c193b7144f92c97bb18d657fb4e066687eae Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 9 Aug 2018 19:29:32 -0700 Subject: [PATCH] specsuite: Start of the harness for the specification test suite --- cmd/hclspecsuite/README.md | 4 + cmd/hclspecsuite/log.go | 3 + cmd/hclspecsuite/main.go | 58 ++++++++++++ cmd/hclspecsuite/runner.go | 164 +++++++++++++++++++++++++++++++++ cmd/hclspecsuite/test_file.go | 78 ++++++++++++++++ specsuite/README.md | 37 ++++++++ specsuite/tests/empty.hcl | 0 specsuite/tests/empty.hcl.json | 1 + specsuite/tests/empty.hcldec | 3 + specsuite/tests/empty.t | 9 ++ 10 files changed, 357 insertions(+) create mode 100644 cmd/hclspecsuite/README.md create mode 100644 cmd/hclspecsuite/log.go create mode 100644 cmd/hclspecsuite/main.go create mode 100644 cmd/hclspecsuite/runner.go create mode 100644 cmd/hclspecsuite/test_file.go create mode 100644 specsuite/README.md create mode 100644 specsuite/tests/empty.hcl create mode 100644 specsuite/tests/empty.hcl.json create mode 100644 specsuite/tests/empty.hcldec create mode 100644 specsuite/tests/empty.t diff --git a/cmd/hclspecsuite/README.md b/cmd/hclspecsuite/README.md new file mode 100644 index 0000000..0f7badc --- /dev/null +++ b/cmd/hclspecsuite/README.md @@ -0,0 +1,4 @@ +# `hclspecsuite` + +`hclspecsuite` is the test harness for +[the HCL specification test suite](../../specsuite/README.md). diff --git a/cmd/hclspecsuite/log.go b/cmd/hclspecsuite/log.go new file mode 100644 index 0000000..f2af650 --- /dev/null +++ b/cmd/hclspecsuite/log.go @@ -0,0 +1,3 @@ +package main + +type LogCallback func(testName string, testFile *TestFile) diff --git a/cmd/hclspecsuite/main.go b/cmd/hclspecsuite/main.go new file mode 100644 index 0000000..e64a053 --- /dev/null +++ b/cmd/hclspecsuite/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + + "github.com/hashicorp/hcl2/hclparse" + + "github.com/hashicorp/hcl2/hcl" + "golang.org/x/crypto/ssh/terminal" +) + +func main() { + os.Exit(realMain(os.Args[1:])) +} + +func realMain(args []string) int { + if len(args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: hclspecsuite \n") + return 2 + } + + testsDir := args[0] + hcldecPath := args[1] + + hcldecPath, err := exec.LookPath(hcldecPath) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + return 2 + } + + parser := hclparse.NewParser() + + runner := &Runner{ + parser: parser, + hcldecPath: hcldecPath, + baseDir: testsDir, + log: func(name string, file *TestFile) { + fmt.Printf("- %s\n", name) + }, + } + diags := runner.Run() + + if len(diags) != 0 { + os.Stderr.WriteString("\n") + color := terminal.IsTerminal(int(os.Stderr.Fd())) + w, _, err := terminal.GetSize(int(os.Stdout.Fd())) + if err != nil { + w = 80 + } + diagWr := hcl.NewDiagnosticTextWriter(os.Stderr, parser.Files(), uint(w), color) + diagWr.WriteDiagnostics(diags) + return 2 + } + + return 0 +} diff --git a/cmd/hclspecsuite/runner.go b/cmd/hclspecsuite/runner.go new file mode 100644 index 0000000..0be1822 --- /dev/null +++ b/cmd/hclspecsuite/runner.go @@ -0,0 +1,164 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hclparse" + "github.com/zclconf/go-cty/cty" +) + +type Runner struct { + parser *hclparse.Parser + hcldecPath string + baseDir string + log LogCallback +} + +func (r *Runner) Run() hcl.Diagnostics { + return r.runDir(r.baseDir) +} + +func (r *Runner) runDir(dir string) hcl.Diagnostics { + var diags hcl.Diagnostics + + infos, err := ioutil.ReadDir(dir) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to read test directory", + Detail: fmt.Sprintf("The directory %q could not be opened: %s.", dir, err), + }) + return diags + } + + var tests []string + var subDirs []string + for _, info := range infos { + name := info.Name() + if strings.HasPrefix(name, ".") { + continue + } + + if info.IsDir() { + subDirs = append(subDirs, name) + } + if strings.HasSuffix(name, ".t") { + tests = append(tests, name) + } + } + sort.Strings(tests) + sort.Strings(subDirs) + + for _, filename := range tests { + filename = filepath.Join(r.baseDir, filename) + testDiags := r.runTest(filename) + diags = append(diags, testDiags...) + } + + for _, dirName := range subDirs { + dir := filepath.Join(r.baseDir, dirName) + dirDiags := r.runDir(dir) + diags = append(diags, dirDiags...) + } + + return diags +} + +func (r *Runner) runTest(filename string) hcl.Diagnostics { + prettyName := r.prettyTestName(filename) + tf, diags := r.LoadTestFile(filename) + if diags.HasErrors() { + // We'll still log, so it's clearer which test the diagnostics belong to. + if r.log != nil { + r.log(prettyName, nil) + } + return diags + } + + if r.log != nil { + r.log(prettyName, tf) + } + + basePath := filename[:len(filename)-2] + specFilename := basePath + ".hcldec" + nativeFilename := basePath + ".hcl" + //jsonFilename := basePath + ".hcl.json" + + if _, err := os.Stat(specFilename); err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing .hcldec file", + Detail: fmt.Sprintf("No specification file for test %s: %s.", prettyName, err), + }) + return diags + } + + if _, err := os.Stat(nativeFilename); err == nil { + + } + + return diags +} + +func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Diagnostics) { + var diags hcl.Diagnostics + var outBuffer bytes.Buffer + var errBuffer bytes.Buffer + + cmd := &exec.Cmd{ + Path: r.hcldecPath, + Args: []string{ + "--spec=" + specFile, + "--diags=json", + inputFile, + }, + Stdout: &outBuffer, + Stderr: &errBuffer, + } + err := cmd.Run() + if _, isExit := err.(*exec.ExitError); !isExit { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to run hcldec", + Detail: fmt.Sprintf("Sub-program hcldec failed to start: %s.", err), + }) + return cty.DynamicVal, diags + } + + if err != nil { + // If we exited unsuccessfully then we'll expect diagnostics on stderr + // TODO: implement that + } else { + // Otherwise, we expect a JSON result value on stdout + // TODO: implement that + } + + return cty.DynamicVal, diags +} + +func (r *Runner) prettyDirName(dir string) string { + rel, err := filepath.Rel(r.baseDir, dir) + if err != nil { + return filepath.ToSlash(dir) + } + return filepath.ToSlash(rel) +} + +func (r *Runner) prettyTestName(filename string) string { + dir := filepath.Dir(filename) + dirName := r.prettyDirName(dir) + filename = filepath.Base(filename) + testName := filename[:len(filename)-2] + if dirName == "." { + return testName + } + return fmt.Sprintf("%s/%s", dirName, testName) +} diff --git a/cmd/hclspecsuite/test_file.go b/cmd/hclspecsuite/test_file.go new file mode 100644 index 0000000..98371ee --- /dev/null +++ b/cmd/hclspecsuite/test_file.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + + "github.com/hashicorp/hcl2/ext/typeexpr" + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" +) + +type TestFile struct { + Result cty.Value + ResultType cty.Type + + Traversals []hcl.Traversal +} + +func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) { + f, diags := r.parser.ParseHCLFile(filename) + if diags.HasErrors() { + return nil, diags + } + + content, moreDiags := f.Body.Content(testFileSchema) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + return nil, diags + } + + ret := &TestFile{ + ResultType: cty.DynamicPseudoType, + } + + if typeAttr, exists := content.Attributes["result_type"]; exists { + ty, moreDiags := typeexpr.TypeConstraint(typeAttr.Expr) + diags = append(diags, moreDiags...) + if !moreDiags.HasErrors() { + ret.ResultType = ty + } + } + + if resultAttr, exists := content.Attributes["result"]; exists { + resultVal, moreDiags := resultAttr.Expr.Value(nil) + diags = append(diags, moreDiags...) + if !moreDiags.HasErrors() { + resultVal, err := convert.Convert(resultVal, ret.ResultType) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid result value", + Detail: fmt.Sprintf("The result value does not conform to the given result type: %s.", err), + Subject: resultAttr.Expr.Range().Ptr(), + }) + } else { + ret.Result = resultVal + } + } + } + + return ret, diags +} + +var testFileSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "result", + }, + { + Name: "result_type", + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "traversals", + }, + }, +} diff --git a/specsuite/README.md b/specsuite/README.md new file mode 100644 index 0000000..0ad305e --- /dev/null +++ b/specsuite/README.md @@ -0,0 +1,37 @@ +# HCL Language Test Suite + +This directory contains an implementation-agnostic test suite that can be used +to verify the correct behavior not only of the HCL implementation in _this_ +repository but also of possible other implementations. + +The harness for running this suite -- a Go program in this directory -- uses +the `hcldec` program as a level of abstraction to avoid depending directly on +the Go implementation. As a result, other HCL implementations must also +include a version of `hcldec` in order to run this spec. + +The tests defined in this suite each correspond to a detail of +[the HCL spec](../hcl/spec.md). This suite is separate from and not a +substitute for direct unit tests of a given implementation that would presumably +also exercise that implementation's own programmatic API. + +To run the suite, first build the harness using Go: + +``` +go install github.com/hashicorp/hcl2/cmd/hclspecsuite +``` + +Then run it, passing it the directory containing the test definitions (the +"tests" subdirectory of this directory) and the path to the `hcldec` executable +to use. + +For example, if working in the root of this repository and using the `hcldec` +implementation from here: + +``` +go install ./cmd/hcldec +hclspecsuite ./specsuite/tests $GOPATH/bin/hcldec +``` + +For developers working on the Go implementation of HCL from this repository, +please note that this spec suite is run as part of a normal `go test ./...` +execution for this whole repository and so does not need to be run separately. diff --git a/specsuite/tests/empty.hcl b/specsuite/tests/empty.hcl new file mode 100644 index 0000000..e69de29 diff --git a/specsuite/tests/empty.hcl.json b/specsuite/tests/empty.hcl.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/specsuite/tests/empty.hcl.json @@ -0,0 +1 @@ +{} diff --git a/specsuite/tests/empty.hcldec b/specsuite/tests/empty.hcldec new file mode 100644 index 0000000..0b26ee8 --- /dev/null +++ b/specsuite/tests/empty.hcldec @@ -0,0 +1,3 @@ +literal { + value = "ok" +} diff --git a/specsuite/tests/empty.t b/specsuite/tests/empty.t new file mode 100644 index 0000000..6b164b2 --- /dev/null +++ b/specsuite/tests/empty.t @@ -0,0 +1,9 @@ +# This test ensures that we can successfully parse an empty file. +# Since an empty file has no content, the hcldec spec for this test is +# just a literal value, which we test below. + +result = "ok" + +traversals { + # Explicitly no traversals +}