ext/typeexpr: HCL extension for "type expressions"

This uses the expression static analysis features to interpret
a combination of static calls and static traversals as the description
of a type.

This is intended for situations where applications need to accept type
information from their end-users, providing a concise syntax for doing
so.

Since this is implemented using static analysis, the type vocabulary is
constrained only to keywords representing primitive types and type
construction functions for complex types. No other expression elements
are allowed.

A separate function is provided for parsing type constraints, which allows
the additonal keyword "any" to represent the dynamic pseudo-type.

Finally, a helper function is provided to convert a type back into a
string representation resembling the original input, as an aid to
applications that need to produce error messages relating to user-entered
types.
This commit is contained in:
Martin Atkins 2018-03-04 14:45:25 -08:00
parent ab87bc9ded
commit be66a72aa8
6 changed files with 855 additions and 0 deletions

67
ext/typeexpr/README.md Normal file
View File

@ -0,0 +1,67 @@
# HCL Type Expressions Extension
This HCL extension defines a convention for describing HCL types using function
call and variable reference syntax, allowing configuration formats to include
type information provided by users.
The type syntax is processed statically from a hcl.Expression, so it cannot
use any of the usual language operators. This is similar to type expressions
in statically-typed programming languages.
```hcl
variable "example" {
type = list(string)
}
```
The extension is built using the `hcl.ExprAsKeyword` and `hcl.ExprCall`
functions, and so it relies on the underlying syntax to define how "keyword"
and "call" are interpreted. The above shows how they are interpreted in
the HCL native syntax, while the following shows the same information
expressed in JSON:
```json
{
"variable": {
"example": {
"type": "list(string)"
}
}
}
```
Notice that since we have additional contextual information that we intend
to allow only calls and keywords the JSON syntax is able to parse the given
string directly as an expression, rather than as a template as would be
the case for normal expression evaluation.
For more information, see [the godoc reference](http://godoc.org/github.com/hashicorp/hcl2/ext/typeexpr).
## Type Expression Syntax
When expressed in the native syntax, the following expressions are permitted
in a type expression:
* `string` - string
* `bool` - boolean
* `number` - number
* `any` - `cty.DynamicPseudoType` (in function `TypeConstraint` only)
* `list(<type_expr>)` - list of the type given as an argument
* `set(<type_expr>)` - set of the type given as an argument
* `map(<type_expr>)` - map of the type given as an argument
* `tuple([<type_exprs...>])` - tuple with the element types given in the single list argument
* `object({<attr_name>=<type_expr>, ...}` - object with the attributes and corresponding types given in the single map argument
For example:
* `list(string)`
* `object({"name":string,"age":number})`
* `map(object({"name":string,"age":number}))`
Note that the object constructor syntax is not fully-general for all possible
object types because it requires the attribute names to be valid identifiers.
In practice it is expected that any time an object type is being fixed for
type checking it will be one that has identifiers as its attributes; object
types with weird attributes generally show up only from arbitrary object
constructors in configuration files, which are usually treated either as maps
or as the dynamic pseudo-type.

11
ext/typeexpr/doc.go Normal file
View File

@ -0,0 +1,11 @@
// Package typeexpr extends HCL with a convention for describing HCL types
// within configuration files.
//
// The type syntax is processed statically from a hcl.Expression, so it cannot
// use any of the usual language operators. This is similar to type expressions
// in statically-typed programming languages.
//
// variable "example" {
// type = list(string)
// }
package typeexpr

196
ext/typeexpr/get_type.go Normal file
View File

@ -0,0 +1,196 @@
package typeexpr
import (
"fmt"
"github.com/hashicorp/hcl2/hcl"
"github.com/zclconf/go-cty/cty"
)
const invalidTypeSummary = "Invalid type specification"
// getType is the internal implementation of both Type and TypeConstraint,
// using the passed flag to distinguish. When constraint is false, the "any"
// keyword will produce an error.
func getType(expr hcl.Expression, constraint bool) (cty.Type, hcl.Diagnostics) {
// First we'll try for one of our keywords
kw := hcl.ExprAsKeyword(expr)
switch kw {
case "bool":
return cty.Bool, nil
case "string":
return cty.String, nil
case "number":
return cty.Number, nil
case "any":
if constraint {
return cty.DynamicPseudoType, nil
}
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("The keyword %q cannot be used in this type specification: an exact type is required.", kw),
Subject: expr.Range().Ptr(),
}}
case "list", "map", "set":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("The %s type constructor requires one argument specifying the element type.", kw),
Subject: expr.Range().Ptr(),
}}
case "object":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "The object type constructor requires one argument specifying the attribute types and values as a map.",
Subject: expr.Range().Ptr(),
}}
case "tuple":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "The tuple type constructor requires one argument specifying the element types as a list.",
Subject: expr.Range().Ptr(),
}}
case "":
// okay! we'll fall through and try processing as a call, then.
default:
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("The keyword %q is not a valid type specification.", kw),
Subject: expr.Range().Ptr(),
}}
}
// If we get down here then our expression isn't just a keyword, so we'll
// try to process it as a call instead.
call, diags := hcl.ExprCall(expr)
if diags.HasErrors() {
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "A type specification is either a primitive type keyword (bool, number, string) or a complex type constructor call, like list(string).",
Subject: expr.Range().Ptr(),
}}
}
switch call.Name {
case "bool", "string", "number", "any":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("Primitive type keyword %q does not expect arguments.", call.Name),
Subject: &call.ArgsRange,
}}
}
if len(call.Arguments) != 1 {
contextRange := call.ArgsRange
subjectRange := call.ArgsRange
if len(call.Arguments) > 1 {
// If we have too many arguments (as opposed to too _few_) then
// we'll highlight the extraneous arguments as the diagnostic
// subject.
subjectRange = hcl.RangeBetween(call.Arguments[1].Range(), call.Arguments[len(call.Arguments)-1].Range())
}
switch call.Name {
case "list", "set", "map":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("The %s type constructor requires one argument specifying the element type.", call.Name),
Subject: &subjectRange,
Context: &contextRange,
}}
case "object":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "The object type constructor requires one argument specifying the attribute types and values as a map.",
Subject: &subjectRange,
Context: &contextRange,
}}
case "tuple":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "The tuple type constructor requires one argument specifying the element types as a list.",
Subject: &subjectRange,
Context: &contextRange,
}}
}
}
switch call.Name {
case "list":
ety, diags := getType(call.Arguments[0], constraint)
return cty.List(ety), diags
case "set":
ety, diags := getType(call.Arguments[0], constraint)
return cty.Set(ety), diags
case "map":
ety, diags := getType(call.Arguments[0], constraint)
return cty.Map(ety), diags
case "object":
attrDefs, diags := hcl.ExprMap(call.Arguments[0])
if diags.HasErrors() {
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "Object type constructor requires a map whose keys are attribute names and whose values are the corresponding attribute types.",
Subject: call.Arguments[0].Range().Ptr(),
Context: expr.Range().Ptr(),
}}
}
atys := make(map[string]cty.Type)
for _, attrDef := range attrDefs {
attrName := hcl.ExprAsKeyword(attrDef.Key)
if attrName == "" {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "Object constructor map keys must be attribute names.",
Subject: attrDef.Key.Range().Ptr(),
Context: expr.Range().Ptr(),
})
continue
}
aty, attrDiags := getType(attrDef.Value, constraint)
diags = append(diags, attrDiags...)
atys[attrName] = aty
}
return cty.Object(atys), diags
case "tuple":
elemDefs, diags := hcl.ExprList(call.Arguments[0])
if diags.HasErrors() {
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "Tuple type constructor requires a list of element types.",
Subject: call.Arguments[0].Range().Ptr(),
Context: expr.Range().Ptr(),
}}
}
etys := make([]cty.Type, len(elemDefs))
for i, defExpr := range elemDefs {
ety, elemDiags := getType(defExpr, constraint)
diags = append(diags, elemDiags...)
etys[i] = ety
}
return cty.Tuple(etys), diags
default:
// Can't access call.Arguments in this path because we've not validated
// that it contains exactly one expression here.
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("Keyword %q is not a valid type constructor.", call.Name),
Subject: expr.Range().Ptr(),
}}
}
}

View File

@ -0,0 +1,352 @@
package typeexpr
import (
"testing"
"github.com/hashicorp/hcl2/gohcl"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/hcl2/hcl/hclsyntax"
"github.com/hashicorp/hcl2/hcl/json"
"github.com/zclconf/go-cty/cty"
)
func TestGetType(t *testing.T) {
tests := []struct {
Source string
Constraint bool
Want cty.Type
WantError string
}{
// keywords
{
`bool`,
false,
cty.Bool,
"",
},
{
`number`,
false,
cty.Number,
"",
},
{
`string`,
false,
cty.String,
"",
},
{
`any`,
false,
cty.DynamicPseudoType,
`The keyword "any" cannot be used in this type specification: an exact type is required.`,
},
{
`any`,
true,
cty.DynamicPseudoType,
"",
},
{
`list`,
false,
cty.DynamicPseudoType,
"The list type constructor requires one argument specifying the element type.",
},
{
`map`,
false,
cty.DynamicPseudoType,
"The map type constructor requires one argument specifying the element type.",
},
{
`set`,
false,
cty.DynamicPseudoType,
"The set type constructor requires one argument specifying the element type.",
},
{
`object`,
false,
cty.DynamicPseudoType,
"The object type constructor requires one argument specifying the attribute types and values as a map.",
},
{
`tuple`,
false,
cty.DynamicPseudoType,
"The tuple type constructor requires one argument specifying the element types as a list.",
},
// constructors
{
`bool()`,
false,
cty.DynamicPseudoType,
`Primitive type keyword "bool" does not expect arguments.`,
},
{
`number()`,
false,
cty.DynamicPseudoType,
`Primitive type keyword "number" does not expect arguments.`,
},
{
`string()`,
false,
cty.DynamicPseudoType,
`Primitive type keyword "string" does not expect arguments.`,
},
{
`any()`,
false,
cty.DynamicPseudoType,
`Primitive type keyword "any" does not expect arguments.`,
},
{
`any()`,
true,
cty.DynamicPseudoType,
`Primitive type keyword "any" does not expect arguments.`,
},
{
`list(string)`,
false,
cty.List(cty.String),
``,
},
{
`set(string)`,
false,
cty.Set(cty.String),
``,
},
{
`map(string)`,
false,
cty.Map(cty.String),
``,
},
{
`list()`,
false,
cty.DynamicPseudoType,
`The list type constructor requires one argument specifying the element type.`,
},
{
`list(string, string)`,
false,
cty.DynamicPseudoType,
`The list type constructor requires one argument specifying the element type.`,
},
{
`list(any)`,
false,
cty.List(cty.DynamicPseudoType),
`The keyword "any" cannot be used in this type specification: an exact type is required.`,
},
{
`list(any)`,
true,
cty.List(cty.DynamicPseudoType),
``,
},
{
`object({})`,
false,
cty.EmptyObject,
``,
},
{
`object({name=string})`,
false,
cty.Object(map[string]cty.Type{"name": cty.String}),
``,
},
{
`object({"name"=string})`,
false,
cty.EmptyObject,
`Object constructor map keys must be attribute names.`,
},
{
`object({name=nope})`,
false,
cty.Object(map[string]cty.Type{"name": cty.DynamicPseudoType}),
`The keyword "nope" is not a valid type specification.`,
},
{
`object()`,
false,
cty.DynamicPseudoType,
`The object type constructor requires one argument specifying the attribute types and values as a map.`,
},
{
`object(string)`,
false,
cty.DynamicPseudoType,
`Object type constructor requires a map whose keys are attribute names and whose values are the corresponding attribute types.`,
},
{
`tuple([])`,
false,
cty.EmptyTuple,
``,
},
{
`tuple([string, bool])`,
false,
cty.Tuple([]cty.Type{cty.String, cty.Bool}),
``,
},
{
`tuple([nope])`,
false,
cty.Tuple([]cty.Type{cty.DynamicPseudoType}),
`The keyword "nope" is not a valid type specification.`,
},
{
`tuple()`,
false,
cty.DynamicPseudoType,
`The tuple type constructor requires one argument specifying the element types as a list.`,
},
{
`tuple(string)`,
false,
cty.DynamicPseudoType,
`Tuple type constructor requires a list of element types.`,
},
{
`shwoop(string)`,
false,
cty.DynamicPseudoType,
`Keyword "shwoop" is not a valid type constructor.`,
},
{
`list("string")`,
false,
cty.List(cty.DynamicPseudoType),
`A type specification is either a primitive type keyword (bool, number, string) or a complex type constructor call, like list(string).`,
},
// More interesting combinations
{
`list(object({}))`,
false,
cty.List(cty.EmptyObject),
``,
},
{
`list(map(tuple([])))`,
false,
cty.List(cty.Map(cty.EmptyTuple)),
``,
},
}
for _, test := range tests {
t.Run(test.Source, func(t *testing.T) {
expr, diags := hclsyntax.ParseExpression([]byte(test.Source), "", hcl.Pos{Line: 1, Column: 1})
if diags.HasErrors() {
t.Fatalf("failed to parse: %s", diags)
}
got, diags := getType(expr, test.Constraint)
if test.WantError == "" {
for _, diag := range diags {
t.Error(diag)
}
} else {
found := false
for _, diag := range diags {
t.Log(diag)
if diag.Severity == hcl.DiagError && diag.Detail == test.WantError {
found = true
}
}
if !found {
t.Errorf("missing expected error detail message: %s", test.WantError)
}
}
if !got.Equals(test.Want) {
t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, test.Want)
}
})
}
}
func TestGetTypeJSON(t *testing.T) {
// We have fewer test cases here because we're mainly exercising the
// extra indirection in the JSON syntax package, which ultimately calls
// into the native syntax parser (which we tested extensively in
// TestGetType).
tests := []struct {
Source string
Constraint bool
Want cty.Type
WantError string
}{
{
`{"expr":"bool"}`,
false,
cty.Bool,
"",
},
{
`{"expr":"list(bool)"}`,
false,
cty.List(cty.Bool),
"",
},
{
`{"expr":"list"}`,
false,
cty.DynamicPseudoType,
"The list type constructor requires one argument specifying the element type.",
},
}
for _, test := range tests {
t.Run(test.Source, func(t *testing.T) {
file, diags := json.Parse([]byte(test.Source), "")
if diags.HasErrors() {
t.Fatalf("failed to parse: %s", diags)
}
type TestContent struct {
Expr hcl.Expression `hcl:"expr"`
}
var content TestContent
diags = gohcl.DecodeBody(file.Body, nil, &content)
if diags.HasErrors() {
t.Fatalf("failed to decode: %s", diags)
}
got, diags := getType(content.Expr, test.Constraint)
if test.WantError == "" {
for _, diag := range diags {
t.Error(diag)
}
} else {
found := false
for _, diag := range diags {
t.Log(diag)
if diag.Severity == hcl.DiagError && diag.Detail == test.WantError {
found = true
}
}
if !found {
t.Errorf("missing expected error detail message: %s", test.WantError)
}
}
if !got.Equals(test.Want) {
t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, test.Want)
}
})
}
}

129
ext/typeexpr/public.go Normal file
View File

@ -0,0 +1,129 @@
package typeexpr
import (
"bytes"
"fmt"
"sort"
"github.com/hashicorp/hcl2/hcl/hclsyntax"
"github.com/hashicorp/hcl2/hcl"
"github.com/zclconf/go-cty/cty"
)
// Type attempts to process the given expression as a type expression and, if
// successful, returns the resulting type. If unsuccessful, error diagnostics
// are returned.
func Type(expr hcl.Expression) (cty.Type, hcl.Diagnostics) {
return getType(expr, false)
}
// TypeConstraint attempts to parse the given expression as a type constraint
// and, if successful, returns the resulting type. If unsuccessful, error
// diagnostics are returned.
//
// A type constraint has the same structure as a type, but it additionally
// allows the keyword "any" to represent cty.DynamicPseudoType, which is often
// used as a wildcard in type checking and type conversion operations.
func TypeConstraint(expr hcl.Expression) (cty.Type, hcl.Diagnostics) {
return getType(expr, true)
}
// TypeString returns a string rendering of the given type as it would be
// expected to appear in the HCL native syntax.
//
// This is primarily intended for showing types to the user in an application
// that uses typexpr, where the user can be assumed to be familiar with the
// type expression syntax. In applications that do not use typeexpr these
// results may be confusing to the user and so type.FriendlyName may be
// preferable, even though it's less precise.
//
// TypeString produces reasonable results only for types like what would be
// produced by the Type and TypeConstraint functions. In particular, it cannot
// support capsule types.
func TypeString(ty cty.Type) string {
// Easy cases first
switch ty {
case cty.String:
return "string"
case cty.Bool:
return "bool"
case cty.Number:
return "number"
case cty.DynamicPseudoType:
return "any"
}
if ty.IsCapsuleType() {
panic("TypeString does not support capsule types")
}
if ty.IsCollectionType() {
ety := ty.ElementType()
etyString := TypeString(ety)
switch {
case ty.IsListType():
return fmt.Sprintf("list(%s)", etyString)
case ty.IsSetType():
return fmt.Sprintf("set(%s)", etyString)
case ty.IsMapType():
return fmt.Sprintf("map(%s)", etyString)
default:
// Should never happen because the above is exhaustive
panic("unsupported collection type")
}
}
if ty.IsObjectType() {
var buf bytes.Buffer
buf.WriteString("object({")
atys := ty.AttributeTypes()
names := make([]string, 0, len(atys))
for name := range atys {
names = append(names, name)
}
sort.Strings(names)
first := true
for _, name := range names {
aty := atys[name]
if !first {
buf.WriteByte(',')
}
if !hclsyntax.ValidIdentifier(name) {
// Should never happen for any type produced by this package,
// but we'll do something reasonable here just so we don't
// produce garbage if someone gives us a hand-assembled object
// type that has weird attribute names.
// Using Go-style quoting here isn't perfect, since it doesn't
// exactly match HCL syntax, but it's fine for an edge-case.
buf.WriteString(fmt.Sprintf("%q", name))
} else {
buf.WriteString(name)
}
buf.WriteByte('=')
buf.WriteString(TypeString(aty))
first = false
}
buf.WriteString("})")
return buf.String()
}
if ty.IsTupleType() {
var buf bytes.Buffer
buf.WriteString("tuple([")
etys := ty.TupleElementTypes()
first := true
for _, ety := range etys {
if !first {
buf.WriteByte(',')
}
buf.WriteString(TypeString(ety))
first = false
}
buf.WriteString("])")
return buf.String()
}
// Should never happen because we covered all cases above.
panic(fmt.Errorf("unsupported type %#v", ty))
}

View File

@ -0,0 +1,100 @@
package typeexpr
import (
"testing"
"github.com/zclconf/go-cty/cty"
)
func TestTypeString(t *testing.T) {
tests := []struct {
Type cty.Type
Want string
}{
{
cty.DynamicPseudoType,
"any",
},
{
cty.String,
"string",
},
{
cty.Number,
"number",
},
{
cty.Bool,
"bool",
},
{
cty.List(cty.Number),
"list(number)",
},
{
cty.Set(cty.Bool),
"set(bool)",
},
{
cty.Map(cty.String),
"map(string)",
},
{
cty.EmptyObject,
"object({})",
},
{
cty.Object(map[string]cty.Type{"foo": cty.Bool}),
"object({foo=bool})",
},
{
cty.Object(map[string]cty.Type{"foo": cty.Bool, "bar": cty.String}),
"object({bar=string,foo=bool})",
},
{
cty.EmptyTuple,
"tuple([])",
},
{
cty.Tuple([]cty.Type{cty.Bool}),
"tuple([bool])",
},
{
cty.Tuple([]cty.Type{cty.Bool, cty.String}),
"tuple([bool,string])",
},
{
cty.List(cty.DynamicPseudoType),
"list(any)",
},
{
cty.Tuple([]cty.Type{cty.DynamicPseudoType}),
"tuple([any])",
},
{
cty.Object(map[string]cty.Type{"foo": cty.DynamicPseudoType}),
"object({foo=any})",
},
{
// We don't expect to find attributes that aren't valid identifiers
// because we only promise to support types that this package
// would've created, but we allow this situation during rendering
// just because it's convenient for applications trying to produce
// error messages about mismatched types. Note that the quoted
// attribute name is not actually accepted by our Type and
// TypeConstraint functions, so this is one situation where the
// TypeString result cannot be re-parsed by those functions.
cty.Object(map[string]cty.Type{"foo bar baz": cty.String}),
`object({"foo bar baz"=string})`,
},
}
for _, test := range tests {
t.Run(test.Type.GoString(), func(t *testing.T) {
got := TypeString(test.Type)
if got != test.Want {
t.Errorf("wrong result\ntype: %#v\ngot: %s\nwant: %s", test.Type, got, test.Want)
}
})
}
}