From ee147d9ee6291a6ae5a589c4a85f58f6e33d3a02 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 3 Feb 2018 15:37:11 -0800 Subject: [PATCH] cmd/hcldec: Command-line tool for converting HCL config to JSON This is essentially a CLI wrapper around the hcldec package, accepting a decoding specification via a HCL-based language and using it to translate input HCL files into JSON values while performing basic structural and type validation of the input files. --- cmd/hcldec/README.md | 100 ++++ .../examples/npm-package/example.npmhcl | 14 + cmd/hcldec/examples/npm-package/spec.hcldec | 136 +++++ .../examples/sh-config-file/example.conf | 10 + cmd/hcldec/examples/sh-config-file/example.sh | 26 + .../examples/sh-config-file/spec.hcldec | 23 + cmd/hcldec/main.go | 214 ++++++++ cmd/hcldec/spec-format.md | 339 +++++++++++++ cmd/hcldec/spec.go | 474 ++++++++++++++++++ cmd/hcldec/type_expr.go | 129 +++++ cmd/hcldec/vars.go | 74 +++ 11 files changed, 1539 insertions(+) create mode 100644 cmd/hcldec/README.md create mode 100644 cmd/hcldec/examples/npm-package/example.npmhcl create mode 100644 cmd/hcldec/examples/npm-package/spec.hcldec create mode 100644 cmd/hcldec/examples/sh-config-file/example.conf create mode 100755 cmd/hcldec/examples/sh-config-file/example.sh create mode 100644 cmd/hcldec/examples/sh-config-file/spec.hcldec create mode 100644 cmd/hcldec/main.go create mode 100644 cmd/hcldec/spec-format.md create mode 100644 cmd/hcldec/spec.go create mode 100644 cmd/hcldec/type_expr.go create mode 100644 cmd/hcldec/vars.go diff --git a/cmd/hcldec/README.md b/cmd/hcldec/README.md new file mode 100644 index 0000000..a037a4c --- /dev/null +++ b/cmd/hcldec/README.md @@ -0,0 +1,100 @@ +# hcldec + +`hcldec` is a command line tool that transforms HCL input into JSON output +using a decoding specification given by the user. + +This tool is intended as a "glue" tool, with use-cases like the following: + +* Define a HCL-based configuration format for a third-party tool that takes + JSON as input, and then translate the HCL configuration into JSON before + running the tool. (See [the `npm-package` example](examples/npm-package).) + +* Use HCL from languages where a HCL parser/decoder is not yet available. + At the time of writing, that's any language other than Go. + +* In particular, define a HCL-based configuration format for a shell script + and then use `jq` to load the result into environment variables for + further processing. (See [the `sh-config-file` example](examples/sh-config-file).) + +## Installation + +If you have a working Go development environment, you can install this tool +with `go get` in the usual way: + +``` +$ go get -u github.com/hashicorp/hcl2/cmd/hcldec +``` + +This will install `hcldec` in `$GOPATH/bin`, which usually places it into +your shell `PATH` so you can then run it as `hcldec`. + +## Usage + +``` +usage: hcldec --spec= [options] [hcl-file ...] + -o, --out string write to the given file, instead of stdout + -s, --spec string path to spec file (required) + -V, --vars json-or-file provide variables to the given configuration file(s) + -v, --version show the version number and immediately exit +``` + +The most important step in using `hcldec` is to write the specification that +defines how to interpret the given configuration files and translate them +into JSON. The following is a simple specification that creates a JSON +object from two top-level attributes in the input configuration: + +```hcl +object { + attr "name" { + type = string + required = true + } + attr "is_member" { + type = bool + } +} +``` + +Specification files are conventionally kept in files with a `.hcldec` +extension. We'll call this one `example.hcldec`. + +With the above specification, the following input file `example.conf` is +valid: + +```hcl +name = "Raul" +``` + +The spec and the input file can then be provided to `hcldec` to extract a +JSON representation: + +``` +$ hcldec --spec=example.hcldec example.conf +{"name": "Raul"} +``` + +The specification defines both how to map the input into a JSON data structure +and what input is valid. The `required = true` specified for the `name` +allows `hcldec` to detect and raise an error when an attribute of that name +is not provided: + +``` +$ hcldec --spec=example.hcldec typo.conf +Error: Unsupported attribute + + on example.conf line 1: + 1: namme = "Juan" + +An attribute named "namme" is not expected here. Did you mean "name"? + +Error: Missing required attribute + + on example.conf line 2: + +The attribute "name" is required, but no definition was found. +``` + +## Further Reading + +For more details on the `.hcldec` specification file format, see +[the spec file documentation](spec-format.md). diff --git a/cmd/hcldec/examples/npm-package/example.npmhcl b/cmd/hcldec/examples/npm-package/example.npmhcl new file mode 100644 index 0000000..445ba77 --- /dev/null +++ b/cmd/hcldec/examples/npm-package/example.npmhcl @@ -0,0 +1,14 @@ +name = "hello-world" +version = "v0.0.1" + +author { + name = "Иван Петрович Сидоров" +} + +contributor { + name = "Juan Pérez" +} + +dependencies = { + left-pad = "1.2.0" +} diff --git a/cmd/hcldec/examples/npm-package/spec.hcldec b/cmd/hcldec/examples/npm-package/spec.hcldec new file mode 100644 index 0000000..a15c187 --- /dev/null +++ b/cmd/hcldec/examples/npm-package/spec.hcldec @@ -0,0 +1,136 @@ +object { + attr "name" { + type = string + required = true + } + attr "version" { + type = string + required = true + } + attr "description" { + type = string + } + attr "keywords" { + type = list(string) + } + attr "homepage" { + # "homepage_url" in input file is translated to "homepage" in output + name = "homepage_url" + } + block "bugs" { + object { + attr "url" { + type = string + } + attr "email" { + type = string + } + } + } + attr "license" { + type = string + } + block "author" { + object { + attr "name" { + type = string + } + attr "email" { + type = string + } + attr "url" { + type = string + } + } + } + block_list "contributors" { + block_type = "contributor" + object { + attr "name" { + type = string + } + attr "email" { + type = string + } + attr "url" { + type = string + } + } + } + attr "files" { + type = list(string) + } + attr "main" { + type = string + } + attr "bin" { + type = map(string) + } + attr "man" { + type = list(string) + } + attr "directories" { + type = map(string) + } + block "repository" { + object { + attr "type" { + type = string + required = true + } + attr "url" { + type = string + required = true + } + } + } + attr "scripts" { + type = map(string) + } + attr "config" { + type = map(string) + } + attr "dependencies" { + type = map(string) + } + attr "devDependencies" { + name = "dev_dependencies" + type = map(string) + } + attr "peerDependencies" { + name = "peer_dependencies" + type = map(string) + } + attr "bundledDependencies" { + name = "bundled_dependencies" + type = map(string) + } + attr "optionalDependencies" { + name = "optional_dependencies" + type = map(string) + } + attr "engines" { + type = map(string) + } + attr "os" { + type = list(string) + } + attr "cpu" { + type = list(string) + } + attr "prefer_global" { + type = bool + } + default "private" { + attr { + name = "private" + type = bool + } + literal { + value = false + } + } + attr "publishConfig" { + type = map(any) + } +} diff --git a/cmd/hcldec/examples/sh-config-file/example.conf b/cmd/hcldec/examples/sh-config-file/example.conf new file mode 100644 index 0000000..c0d7705 --- /dev/null +++ b/cmd/hcldec/examples/sh-config-file/example.conf @@ -0,0 +1,10 @@ +name = "Juan" +friend { + name = "John" +} +friend { + name = "Yann" +} +friend { + name = "Ermintrude" +} diff --git a/cmd/hcldec/examples/sh-config-file/example.sh b/cmd/hcldec/examples/sh-config-file/example.sh new file mode 100755 index 0000000..95a0080 --- /dev/null +++ b/cmd/hcldec/examples/sh-config-file/example.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -euo pipefail + +# All paths from this point on are relative to the directory containing this +# script, for simplicity's sake. +cd "$( dirname "${BASH_SOURCE[0]}" )" + +# Read the config file using hcldec and then use jq to extract values in a +# shell-friendly form. jq will ensure that the values are properly quoted and +# escaped for consumption by the shell. +CONFIG_VARS="$(hcldec --spec=spec.hcldec example.conf | jq -r '@sh "NAME=\(.name) GREETING=\(.greeting) FRIENDS=(\(.friends))"')" +if [ $? != 0 ]; then + # If hcldec or jq failed then it has already printed out some error messages + # and so we can bail out. + exit $? +fi + +# Import our settings into our environment +eval "$CONFIG_VARS" + +# ...and now, some contrived usage of the settings we loaded: +echo "$GREETING $NAME!" +for name in ${FRIENDS[@]}; do + echo "$GREETING $name, too!" +done diff --git a/cmd/hcldec/examples/sh-config-file/spec.hcldec b/cmd/hcldec/examples/sh-config-file/spec.hcldec new file mode 100644 index 0000000..6b15fdc --- /dev/null +++ b/cmd/hcldec/examples/sh-config-file/spec.hcldec @@ -0,0 +1,23 @@ +object { + attr "name" { + type = string + required = true + } + default "greeting" { + attr { + name = "greeting" + type = string + } + literal { + value = "Hello" + } + } + block_list "friends" { + block_type = "friend" + attr { + name = "name" + type = string + required = true + } + } +} diff --git a/cmd/hcldec/main.go b/cmd/hcldec/main.go new file mode 100644 index 0000000..9ace52d --- /dev/null +++ b/cmd/hcldec/main.go @@ -0,0 +1,214 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcldec" + "github.com/hashicorp/hcl2/hclparse" + flag "github.com/spf13/pflag" + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" + "golang.org/x/crypto/ssh/terminal" +) + +const versionStr = "0.0.1-dev" + +// vars is populated from --vars arguments on the command line, via a flag +// registration in init() below. +var vars = &varSpecs{} + +var ( + specFile = flag.StringP("spec", "s", "", "path to spec file (required)") + outputFile = flag.StringP("out", "o", "", "write to the given file, instead of stdout") + showVersion = flag.BoolP("version", "v", false, "show the version number and immediately exit") +) + +var parser = hclparse.NewParser() +var diagWr hcl.DiagnosticWriter // initialized in init + +func init() { + flag.VarP(vars, "vars", "V", "provide variables to the given configuration file(s)") + + color := terminal.IsTerminal(int(os.Stderr.Fd())) + w, _, err := terminal.GetSize(int(os.Stdout.Fd())) + if err != nil { + w = 80 + } + diagWr = hcl.NewDiagnosticTextWriter(os.Stderr, parser.Files(), uint(w), color) +} + +func main() { + flag.Usage = usage + flag.Parse() + + if *showVersion { + fmt.Println(versionStr) + os.Exit(0) + } + + args := flag.Args() + + err := realmain(args) + + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n\n", err.Error()) + os.Exit(1) + } +} + +func realmain(args []string) error { + + if *specFile == "" { + return fmt.Errorf("the --spec=... argument is required") + } + + var diags hcl.Diagnostics + + spec, specDiags := loadSpecFile(*specFile) + diags = append(diags, specDiags...) + if specDiags.HasErrors() { + diagWr.WriteDiagnostics(diags) + os.Exit(2) + } + + var ctx *hcl.EvalContext + if len(*vars) != 0 { + ctx = &hcl.EvalContext{ + Variables: map[string]cty.Value{}, + } + for i, varsSpec := range *vars { + var vals map[string]cty.Value + var valsDiags hcl.Diagnostics + if strings.HasPrefix(strings.TrimSpace(varsSpec), "{") { + // literal JSON object on the command line + vals, valsDiags = parseVarsArg(varsSpec, i) + } else { + // path to a file containing either HCL or JSON (by file extension) + vals, valsDiags = parseVarsFile(varsSpec) + } + diags = append(diags, valsDiags...) + for k, v := range vals { + ctx.Variables[k] = v + } + } + } + + var bodies []hcl.Body + + if len(args) == 0 { + src, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("failed to read stdin: %s", err) + } + + f, fDiags := parser.ParseHCL(src, "") + diags = append(diags, fDiags...) + if !fDiags.HasErrors() { + bodies = append(bodies, f.Body) + } + } else { + for _, filename := range args { + f, fDiags := parser.ParseHCLFile(filename) + diags = append(diags, fDiags...) + if !fDiags.HasErrors() { + bodies = append(bodies, f.Body) + } + } + } + + if diags.HasErrors() { + diagWr.WriteDiagnostics(diags) + os.Exit(2) + } + + var body hcl.Body + switch len(bodies) { + case 0: + // should never happen, but... okay? + body = hcl.EmptyBody() + case 1: + body = bodies[0] + default: + body = hcl.MergeBodies(bodies) + } + + val, decDiags := hcldec.Decode(body, spec, ctx) + diags = append(diags, decDiags...) + + if diags.HasErrors() { + diagWr.WriteDiagnostics(diags) + os.Exit(2) + } + + out, err := ctyjson.Marshal(val, val.Type()) + if err != nil { + return err + } + + // hcldec will include explicit nulls where an ObjectSpec has a spec + // that refers to a missing item, but that'll probably be annoying for + // a consumer of our output to deal with so we'll just strip those + // out and reduce to only the non-null values. + out = stripJSONNullProperties(out) + + target := os.Stdout + if *outputFile != "" { + target, err = os.OpenFile(*outputFile, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + return fmt.Errorf("can't open %s for writing: %s", *outputFile, err) + } + } + + fmt.Fprintf(target, "%s\n", out) + + return nil +} + +func usage() { + fmt.Fprintf(os.Stderr, "usage: hcldec --spec= [options] [hcl-file ...]\n") + flag.PrintDefaults() + os.Exit(2) +} + +func stripJSONNullProperties(src []byte) []byte { + var v interface{} + err := json.Unmarshal(src, &v) + if err != nil { + // We expect valid JSON + panic(err) + } + + v = stripNullMapElements(v) + + new, err := json.Marshal(v) + if err != nil { + panic(err) + } + return new +} + +func stripNullMapElements(v interface{}) interface{} { + switch tv := v.(type) { + case map[string]interface{}: + for k, ev := range tv { + if ev == nil { + delete(tv, k) + } else { + tv[k] = stripNullMapElements(ev) + } + } + return v + case []interface{}: + for i, ev := range tv { + tv[i] = stripNullMapElements(ev) + } + return v + default: + return v + } +} diff --git a/cmd/hcldec/spec-format.md b/cmd/hcldec/spec-format.md new file mode 100644 index 0000000..7ba709b --- /dev/null +++ b/cmd/hcldec/spec-format.md @@ -0,0 +1,339 @@ +# `hcldec` spec format + +The `hcldec` spec format instructs [`hcldec`](README.md) on how to validate +one or more configuration files given in the HCL syntax and how to translate +the result into JSON format. + +The spec format is itself built from HCL syntax, with each HCL block serving +as a _spec_ whose block type and contents together describe a single mapping +action and, in most cases, a validation constraint. Each spec block produces +one JSON value. + +A spec _file_ must have a single top-level spec block that describes the +top-level JSON value `hcldec` will return, and that spec block may have other +nested spec blocks (depending on its type) that produce nested structures and +additional validation constraints. + +The most common usage of `hcldec` is to produce a JSON object whose properties +are derived from the top-level content of the input file. In this case, the +root of the given spec file will have an `object` spec block whose contents +describe how each of the object's properties are to be populated using +nested spec blocks. + +Each spec is evaluated in the context of an HCL _body_, which is the HCL +terminology for one level of nesting in a configuration file. The top-level +objects in a file all belong to the root body of that file, and then each +nested block has its own body containing the elements within that block. +Some spec types select a new body as the context for their nested specs, +allowing nested HCL structures to be decoded. + +## Spec Block Types + +The following sections describe the different block types that can be used to +define specs within a spec file. + +### `object` spec blocks + +The `object` spec type is the most commonly used at the root of a spec file. +Its result is a JSON object whose properties are set based on any nested +spec blocks: + +```hcl +object { + attr "name" { + type = "string" + } + block "address" { + object { + attr "street" { + type = "string" + } + # ... + } + } +} +``` + +Nested spec blocks inside `object` must always have an extra block label +`"name"`, `"address"` and `"street"` in the above example) that specifies +the name of the property that should be created in the JSON object result. +This label also acts as a default name selector for the nested spec, allowing +the `attr` blocks in the above example to omit the usually-required `name` +argument in cases where the HCL input name and JSON output name are the same. + +An `object` spec block creates no validation constraints, but it passes on +any validation constraints created by the nested specs. + +### `array` spec blocks + +The `array` spec type produces a JSON array whose elements are set based on +any nested spec blocks: + +```hcl +array { + attr { + name = "first_element" + type = "string" + } + attr { + name = "second_element" + type = "string" + } +} +``` + +An `array` spec block creates no validation constraints, but it passes on +any validation constraints created by the nested specs. + +### `attr` spec blocks + +The `attr` spec type reads the value of an attribute in the current body +and returns that value as its result. It also creates validation constraints +for the given attribute name and its value. + +```hcl +attr { + name = "document_root" + type = string + required = true +} +``` + +`attr` spec blocks accept the following arguments: + +* `name` (required) - The attribute name to expect within the HCL input file. + This may be omitted when a default name selector is created by a parent + `object` spec, if the input attribute name should match the output JSON + object property name. + +* `type` (optional) - A [type expression](#type-expressions) that the given + attribute value must conform to. If this argument is set, `hcldec` will + automatically convert the given input value to this type or produce an + error if that is not possible. + +* `required` (optional) - If set to `true`, `hcldec` will produce an error + if a value is not provided for the source attribute. + +`attr` is a leaf spec type, so no nested spec blocks are permitted. + +### `block` spec blocks + +The `block` spec type applies one nested spec block to the contents of a +block within the current body and returns the result of that spec. It also +creates validation constraints for the given block type name. + +```hcl +block { + block_type = "logging" + + object { + attr "level" { + type = string + } + attr "file" { + type = string + } + } +} +``` + +`block` spec blocks accept the following arguments: + +* `block_type` (required) - The block type name to expect within the HCL + input file. This may be omitted when a default name selector is created + by a parent `object` spec, if the input block type name should match the + output JSON object property name. + +* `required` (optional) - If set to `true`, `hcldec` will produce an error + if a block of the specified type is not present in the current body. + +`block` creates a validation constraint that there must be zero or one blocks +of the given type name, or exactly one if `required` is set. + +`block` expects a single nested spec block, which is applied to the body of +the block of the given type when it is present. + +### `block_list` spec blocks + +The `block_list` spec type is similar to `block`, but it accepts zero or +more blocks of a specified type rather than requiring zero or one. The +result is a JSON array with one entry per block of the given type. + +```hcl +block_list { + block_type = "log_file" + + object { + attr "level" { + type = string + } + attr "filename" { + type = string + required = true + } + } +} +``` + +`block_list` spec blocks accept the following arguments: + +* `block_type` (required) - The block type name to expect within the HCL + input file. This may be omitted when a default name selector is created + by a parent `object` spec, if the input block type name should match the + output JSON object property name. + +* `min_items` (optional) - If set to a number greater than zero, `hcldec` will + produce an error if fewer than the given number of blocks are present. + +* `max_items` (optional) - If set to a number greater than zero, `hcldec` will + produce an error if more than the given number of blocks are present. This + attribute must be greater than or equal to `min_items` if both are set. + +`block` creates a validation constraint on the number of blocks of the given +type that must be present. + +`block` expects a single nested spec block, which is applied to the body of +each matching block to produce the resulting list items. + +### `block_set` spec blocks + +The `block_set` spec type behaves the same as `block_list` except that +the result is in no specific order and any duplicate items are removed. + +```hcl +block_set { + block_type = "log_file" + + object { + attr "level" { + type = string + } + attr "filename" { + type = string + required = true + } + } +} +``` + +The contents of `block_set` are the same as for `block_list`. + +### `block_map` spec blocks + +The `block_map` spec type is similar to `block`, but it accepts zero or +more blocks of a specified type rather than requiring zero or one. The +result is a JSON object, or possibly multiple nested JSON objects, whose +properties are derived from the labels set on each matching block. + +```hcl +block_map { + block_type = "log_file" + labels = ["filename"] + + object { + attr "level" { + type = string + required = true + } + } +} +``` + +`block_map` spec blocks accept the following arguments: + +* `block_type` (required) - The block type name to expect within the HCL + input file. This may be omitted when a default name selector is created + by a parent `object` spec, if the input block type name should match the + output JSON object property name. + +* `labels` (required) - A list of user-oriented block label names. Each entry + in this list creates one level of object within the output value, and + requires one additional block header label on any child block of this type. + Block header labels are the quoted strings that appear after the block type + name but before the opening `{`. + +`block` creates a validation constraint on the number of labels that blocks +of the given type must have. + +`block` expects a single nested spec block, which is applied to the body of +each matching block to produce the resulting map items. + +## `literal` spec blocks + +The `literal` spec type returns a given literal value, and creates no +validation constraints. It is most commonly used with the `default` spec +type to create a fallback value, but can also be used e.g. to fill out +required properties in an `object` spec that do not correspond to any +construct in the input configuration. + +```hcl +literal { + value = "hello world" +} +``` + +`literal` spec blocks accept the following argument: + +* `value` (required) - The value to return. + +`literal` is a leaf spec type, so no nested spec blocks are permitted. + +## `default` spec blocks + +The `default` spec type evaluates a sequence of nested specs in turn and +returns the result of the first one that produces a non-null value. +It creates no validation constraints of its own, but passes on the validation +constraints from its first nested block. + +```hcl +default { + attr { + name = "private" + type = bool + } + literal { + value = false + } +} +``` + +A `default` spec block must have at least one nested spec block, and should +generally have at least two since otherwise the `default` wrapper is a no-op. + +The second and any subsequent spec blocks are _fallback_ specs. These exhibit +their usual behavior but are not able to impose validation constraints on the +current body since they are not evaluated unless all prior specs produce +`null` as their result. + +## Type Expressions + +Type expressions are used to describe the expected type of an attribute, as +an additional validation constraint. + +A type expression uses primitive type names and compound type constructors. +A type constructor builds a new type based on one or more type expression +arguments. + +The following type names and type constructors are supported: + +* `any` is a wildcard that accepts a value of any type. (In HCL terms, this + is the _dynamic pseudo-type_.) +* `string` is a Unicode string. +* `number` is an arbitrary-precision floating point number. +* `bool` is a boolean value (`true` or `false`) +* `list(element_type)` constructs a list type with the given element type +* `set(element_type)` constructs a set type with the given element type +* `map(element_type)` constructs a map type with the given element type +* `object({name1 = element_type, name2 = element_type, ...})` constructs + an object type with the given attribute types. +* `tuple([element_type, element_type, ...])` constructs a tuple type with + the given element types. This can be used, for example, to require an + array with a particular number of elements, or with elements of different + types. + +The above types are as defined by +[the HCL syntax-agnostic information model](../../hcl/spec.md). After +validation, values are lowered to JSON's type system, which is a subset +of the HCL type system. + +`null` is a valid value of any type, and not a type itself. diff --git a/cmd/hcldec/spec.go b/cmd/hcldec/spec.go new file mode 100644 index 0000000..b7053bb --- /dev/null +++ b/cmd/hcldec/spec.go @@ -0,0 +1,474 @@ +package main + +import ( + "fmt" + + "github.com/hashicorp/hcl2/gohcl" + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +func loadSpecFile(filename string) (hcldec.Spec, hcl.Diagnostics) { + file, diags := parser.ParseHCLFile(filename) + if diags.HasErrors() { + return errSpec, diags + } + + return decodeSpecRoot(file.Body) +} + +func decodeSpecRoot(body hcl.Body) (hcldec.Spec, hcl.Diagnostics) { + content, diags := body.Content(specSchemaUnlabelled) + + if len(content.Blocks) == 0 { + if diags.HasErrors() { + // If we already have errors then they probably explain + // why we have no blocks, so we'll skip our additional + // error message added below. + return errSpec, diags + } + + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing spec block", + Detail: "A spec file must have exactly one root block specifying how to map to a JSON value.", + Subject: body.MissingItemRange().Ptr(), + }) + return errSpec, diags + } + + if len(content.Blocks) > 1 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Extraneous spec block", + Detail: "A spec file must have exactly one root block specifying how to map to a JSON value.", + Subject: &content.Blocks[1].DefRange, + }) + return errSpec, diags + } + + spec, specDiags := decodeSpecBlock(content.Blocks[0]) + diags = append(diags, specDiags...) + return spec, diags +} + +func decodeSpecBlock(block *hcl.Block) (hcldec.Spec, hcl.Diagnostics) { + var impliedName string + if len(block.Labels) > 0 { + impliedName = block.Labels[0] + } + + switch block.Type { + + case "object": + return decodeObjectSpec(block.Body) + + case "attr": + return decodeAttrSpec(block.Body, impliedName) + + case "block": + return decodeBlockSpec(block.Body, impliedName) + + case "block_list": + return decodeBlockListSpec(block.Body, impliedName) + + case "block_set": + return decodeBlockSetSpec(block.Body, impliedName) + + case "block_map": + return decodeBlockMapSpec(block.Body, impliedName) + + case "default": + return decodeDefaultSpec(block.Body) + + case "literal": + return decodeLiteralSpec(block.Body) + + default: + // Should never happen, because the above cases should be exhaustive + // for our schema. + var diags hcl.Diagnostics + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid spec block", + Detail: fmt.Sprintf("Blocks of type %q are not expected here.", block.Type), + Subject: &block.TypeRange, + }) + return errSpec, diags + } +} + +func decodeObjectSpec(body hcl.Body) (hcldec.Spec, hcl.Diagnostics) { + content, diags := body.Content(specSchemaLabelled) + + spec := make(hcldec.ObjectSpec) + for _, block := range content.Blocks { + propSpec, propDiags := decodeSpecBlock(block) + diags = append(diags, propDiags...) + spec[block.Labels[0]] = propSpec + } + + return spec, diags +} + +func decodeAttrSpec(body hcl.Body, impliedName string) (hcldec.Spec, hcl.Diagnostics) { + type content struct { + Name *string `hcl:"name"` + Type hcl.Expression `hcl:"type"` + Required *bool `hcl:"required"` + } + + var args content + diags := gohcl.DecodeBody(body, nil, &args) + if diags.HasErrors() { + return errSpec, diags + } + + spec := &hcldec.AttrSpec{ + Name: impliedName, + } + + if args.Required != nil { + spec.Required = *args.Required + } + if args.Name != nil { + spec.Name = *args.Name + } + + var typeDiags hcl.Diagnostics + spec.Type, typeDiags = evalTypeExpr(args.Type) + diags = append(diags, typeDiags...) + + if spec.Name == "" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing name in attribute spec", + Detail: "The name attribute is required, to specify the attribute name that is expected in an input HCL file.", + Subject: body.MissingItemRange().Ptr(), + }) + return errSpec, diags + } + + return spec, diags +} + +func decodeBlockSpec(body hcl.Body, impliedName string) (hcldec.Spec, hcl.Diagnostics) { + type content struct { + TypeName *string `hcl:"block_type"` + Required *bool `hcl:"required"` + Nested hcl.Body `hcl:",remain"` + } + + var args content + diags := gohcl.DecodeBody(body, nil, &args) + if diags.HasErrors() { + return errSpec, diags + } + + spec := &hcldec.BlockSpec{ + TypeName: impliedName, + } + + if args.Required != nil { + spec.Required = *args.Required + } + if args.TypeName != nil { + spec.TypeName = *args.TypeName + } + + nested, nestedDiags := decodeBlockNestedSpec(args.Nested) + diags = append(diags, nestedDiags...) + spec.Nested = nested + + return spec, diags +} + +func decodeBlockListSpec(body hcl.Body, impliedName string) (hcldec.Spec, hcl.Diagnostics) { + type content struct { + TypeName *string `hcl:"block_type"` + MinItems *int `hcl:"min_items"` + MaxItems *int `hcl:"max_items"` + Nested hcl.Body `hcl:",remain"` + } + + var args content + diags := gohcl.DecodeBody(body, nil, &args) + if diags.HasErrors() { + return errSpec, diags + } + + spec := &hcldec.BlockListSpec{ + TypeName: impliedName, + } + + if args.MinItems != nil { + spec.MinItems = *args.MinItems + } + if args.MaxItems != nil { + spec.MaxItems = *args.MaxItems + } + if args.TypeName != nil { + spec.TypeName = *args.TypeName + } + + nested, nestedDiags := decodeBlockNestedSpec(args.Nested) + diags = append(diags, nestedDiags...) + spec.Nested = nested + + if spec.TypeName == "" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing block_type in block_list spec", + Detail: "The block_type attribute is required, to specify the block type name that is expected in an input HCL file.", + Subject: body.MissingItemRange().Ptr(), + }) + return errSpec, diags + } + + return spec, diags +} + +func decodeBlockSetSpec(body hcl.Body, impliedName string) (hcldec.Spec, hcl.Diagnostics) { + type content struct { + TypeName *string `hcl:"block_type"` + MinItems *int `hcl:"min_items"` + MaxItems *int `hcl:"max_items"` + Nested hcl.Body `hcl:",remain"` + } + + var args content + diags := gohcl.DecodeBody(body, nil, &args) + if diags.HasErrors() { + return errSpec, diags + } + + spec := &hcldec.BlockSetSpec{ + TypeName: impliedName, + } + + if args.MinItems != nil { + spec.MinItems = *args.MinItems + } + if args.MaxItems != nil { + spec.MaxItems = *args.MaxItems + } + if args.TypeName != nil { + spec.TypeName = *args.TypeName + } + + nested, nestedDiags := decodeBlockNestedSpec(args.Nested) + diags = append(diags, nestedDiags...) + spec.Nested = nested + + if spec.TypeName == "" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing block_type in block_set spec", + Detail: "The block_type attribute is required, to specify the block type name that is expected in an input HCL file.", + Subject: body.MissingItemRange().Ptr(), + }) + return errSpec, diags + } + + return spec, diags +} + +func decodeBlockMapSpec(body hcl.Body, impliedName string) (hcldec.Spec, hcl.Diagnostics) { + type content struct { + TypeName *string `hcl:"block_type"` + Labels []string `hcl:"labels"` + Nested hcl.Body `hcl:",remain"` + } + + var args content + diags := gohcl.DecodeBody(body, nil, &args) + if diags.HasErrors() { + return errSpec, diags + } + + spec := &hcldec.BlockMapSpec{ + TypeName: impliedName, + } + + if args.TypeName != nil { + spec.TypeName = *args.TypeName + } + spec.LabelNames = args.Labels + + nested, nestedDiags := decodeBlockNestedSpec(args.Nested) + diags = append(diags, nestedDiags...) + spec.Nested = nested + + if spec.TypeName == "" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing block_type in block_map spec", + Detail: "The block_type attribute is required, to specify the block type name that is expected in an input HCL file.", + Subject: body.MissingItemRange().Ptr(), + }) + return errSpec, diags + } + if len(spec.LabelNames) < 1 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid block label name list", + Detail: "A block_map must have at least one label specified.", + Subject: body.MissingItemRange().Ptr(), + }) + return errSpec, diags + } + + return spec, diags +} + +func decodeBlockNestedSpec(body hcl.Body) (hcldec.Spec, hcl.Diagnostics) { + content, diags := body.Content(specSchemaUnlabelled) + + if len(content.Blocks) == 0 { + if diags.HasErrors() { + // If we already have errors then they probably explain + // why we have no blocks, so we'll skip our additional + // error message added below. + return errSpec, diags + } + + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing spec block", + Detail: "A block spec must have exactly one child spec specifying how to decode block contents.", + Subject: body.MissingItemRange().Ptr(), + }) + return errSpec, diags + } + + if len(content.Blocks) > 1 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Extraneous spec block", + Detail: "A block spec must have exactly one child spec specifying how to decode block contents.", + Subject: &content.Blocks[1].DefRange, + }) + return errSpec, diags + } + + spec, specDiags := decodeSpecBlock(content.Blocks[0]) + diags = append(diags, specDiags...) + return spec, diags +} + +func decodeLiteralSpec(body hcl.Body) (hcldec.Spec, hcl.Diagnostics) { + type content struct { + Value cty.Value `hcl:"value"` + } + + var args content + diags := gohcl.DecodeBody(body, nil, &args) + if diags.HasErrors() { + return errSpec, diags + } + + return &hcldec.LiteralSpec{ + Value: args.Value, + }, diags +} + +func decodeDefaultSpec(body hcl.Body) (hcldec.Spec, hcl.Diagnostics) { + content, diags := body.Content(specSchemaUnlabelled) + + if len(content.Blocks) == 0 { + if diags.HasErrors() { + // If we already have errors then they probably explain + // why we have no blocks, so we'll skip our additional + // error message added below. + return errSpec, diags + } + + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing spec block", + Detail: "A default block must have at least one nested spec, each specifying a possible outcome.", + Subject: body.MissingItemRange().Ptr(), + }) + return errSpec, diags + } + + if len(content.Blocks) == 1 && !diags.HasErrors() { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "Useless default block", + Detail: "A default block with only one spec is equivalent to using that spec alone.", + Subject: &content.Blocks[1].DefRange, + }) + } + + var spec hcldec.Spec + for _, block := range content.Blocks { + candidateSpec, candidateDiags := decodeSpecBlock(block) + diags = append(diags, candidateDiags...) + if candidateDiags.HasErrors() { + continue + } + + if spec == nil { + spec = candidateSpec + } else { + spec = &hcldec.DefaultSpec{ + Primary: spec, + Default: candidateSpec, + } + } + } + + return spec, diags +} + +var errSpec = &hcldec.LiteralSpec{ + Value: cty.NullVal(cty.DynamicPseudoType), +} + +var specBlockTypes = []string{ + "object", + "tuple", + + "literal", + + "attr", + + "block", + "block_list", + "block_map", + "block_set", + + "default", +} + +var specSchemaUnlabelled *hcl.BodySchema +var specSchemaLabelled *hcl.BodySchema + +var specSchemaLabelledLabels = []string{"key"} + +func init() { + specSchemaLabelled = &hcl.BodySchema{ + Blocks: make([]hcl.BlockHeaderSchema, 0, len(specBlockTypes)), + } + specSchemaUnlabelled = &hcl.BodySchema{ + Blocks: make([]hcl.BlockHeaderSchema, 0, len(specBlockTypes)), + } + + for _, name := range specBlockTypes { + specSchemaLabelled.Blocks = append( + specSchemaLabelled.Blocks, + hcl.BlockHeaderSchema{ + Type: name, + LabelNames: specSchemaLabelledLabels, + }, + ) + specSchemaUnlabelled.Blocks = append( + specSchemaUnlabelled.Blocks, + hcl.BlockHeaderSchema{ + Type: name, + }, + ) + } +} diff --git a/cmd/hcldec/type_expr.go b/cmd/hcldec/type_expr.go new file mode 100644 index 0000000..559277f --- /dev/null +++ b/cmd/hcldec/type_expr.go @@ -0,0 +1,129 @@ +package main + +import ( + "fmt" + "reflect" + + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" +) + +var typeType = cty.Capsule("type", reflect.TypeOf(cty.NilType)) + +var typeEvalCtx = &hcl.EvalContext{ + Variables: map[string]cty.Value{ + "string": wrapTypeType(cty.String), + "bool": wrapTypeType(cty.Bool), + "number": wrapTypeType(cty.Number), + "any": wrapTypeType(cty.DynamicPseudoType), + }, + Functions: map[string]function.Function{ + "list": function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "element_type", + Type: typeType, + }, + }, + Type: function.StaticReturnType(typeType), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + ety := unwrapTypeType(args[0]) + ty := cty.List(ety) + return wrapTypeType(ty), nil + }, + }), + "set": function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "element_type", + Type: typeType, + }, + }, + Type: function.StaticReturnType(typeType), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + ety := unwrapTypeType(args[0]) + ty := cty.Set(ety) + return wrapTypeType(ty), nil + }, + }), + "map": function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "element_type", + Type: typeType, + }, + }, + Type: function.StaticReturnType(typeType), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + ety := unwrapTypeType(args[0]) + ty := cty.Map(ety) + return wrapTypeType(ty), nil + }, + }), + "tuple": function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "element_types", + Type: cty.List(typeType), + }, + }, + Type: function.StaticReturnType(typeType), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + etysVal := args[0] + etys := make([]cty.Type, 0, etysVal.LengthInt()) + for it := etysVal.ElementIterator(); it.Next(); { + _, wrapEty := it.Element() + etys = append(etys, unwrapTypeType(wrapEty)) + } + ty := cty.Tuple(etys) + return wrapTypeType(ty), nil + }, + }), + "object": function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "attribute_types", + Type: cty.Map(typeType), + }, + }, + Type: function.StaticReturnType(typeType), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + atysVal := args[0] + atys := make(map[string]cty.Type) + for it := atysVal.ElementIterator(); it.Next(); { + nameVal, wrapAty := it.Element() + name := nameVal.AsString() + atys[name] = unwrapTypeType(wrapAty) + } + ty := cty.Object(atys) + return wrapTypeType(ty), nil + }, + }), + }, +} + +func evalTypeExpr(expr hcl.Expression) (cty.Type, hcl.Diagnostics) { + result, diags := expr.Value(typeEvalCtx) + if result.IsNull() { + return cty.DynamicPseudoType, diags + } + if !result.Type().Equals(typeType) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid type expression", + Detail: fmt.Sprintf("A type is required, not %s.", result.Type().FriendlyName()), + }) + return cty.DynamicPseudoType, diags + } + + return unwrapTypeType(result), diags +} + +func wrapTypeType(ty cty.Type) cty.Value { + return cty.CapsuleVal(typeType, &ty) +} + +func unwrapTypeType(val cty.Value) cty.Type { + return *(val.EncapsulatedValue().(*cty.Type)) +} diff --git a/cmd/hcldec/vars.go b/cmd/hcldec/vars.go new file mode 100644 index 0000000..f2ec1b4 --- /dev/null +++ b/cmd/hcldec/vars.go @@ -0,0 +1,74 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" +) + +func parseVarsArg(src string, argIdx int) (map[string]cty.Value, hcl.Diagnostics) { + fakeFn := fmt.Sprintf("", argIdx) + f, diags := parser.ParseJSON([]byte(src), fakeFn) + if f == nil { + return nil, diags + } + vals, valsDiags := parseVarsBody(f.Body) + diags = append(diags, valsDiags...) + return vals, diags +} + +func parseVarsFile(filename string) (map[string]cty.Value, hcl.Diagnostics) { + var f *hcl.File + var diags hcl.Diagnostics + + if strings.HasSuffix(filename, ".json") { + f, diags = parser.ParseJSONFile(filename) + } else { + f, diags = parser.ParseHCLFile(filename) + } + + if f == nil { + return nil, diags + } + + vals, valsDiags := parseVarsBody(f.Body) + diags = append(diags, valsDiags...) + return vals, diags + +} + +func parseVarsBody(body hcl.Body) (map[string]cty.Value, hcl.Diagnostics) { + attrs, diags := body.JustAttributes() + if attrs == nil { + return nil, diags + } + + vals := make(map[string]cty.Value, len(attrs)) + for name, attr := range attrs { + val, valDiags := attr.Expr.Value(nil) + diags = append(diags, valDiags...) + vals[name] = val + } + return vals, diags +} + +// varSpecs is an implementation of pflag.Value that accumulates a list of +// raw values, ignoring any quoting. This is similar to pflag.StringSlice +// but does not complain if there are literal quotes inside the value, which +// is important for us to accept JSON literals here. +type varSpecs []string + +func (vs *varSpecs) String() string { + return strings.Join([]string(*vs), ", ") +} + +func (vs *varSpecs) Set(new string) error { + *vs = append(*vs, new) + return nil +} + +func (vs *varSpecs) Type() string { + return "json-or-file" +}