zcldec: initial work on decoding bodies directly to cty.Value

This package is an alternative to gocty for situations where static Go
types are not desired and the application instead wishes to remain in the
cty dynamic type system.

zcldec/decode.go

@@ -0,0 +1,25 @@
+package zcldec
+
+import (
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+func decode(body zcl.Body, block *zcl.Block, ctx *zcl.EvalContext, spec Spec, partial bool) (cty.Value, zcl.Body, zcl.Diagnostics) {
+	schema := ImpliedSchema(spec)
+
+	var content *zcl.BodyContent
+	var diags zcl.Diagnostics
+	var leftovers zcl.Body
+
+	if partial {
+		content, leftovers, diags = body.PartialContent(schema)
+	} else {
+		content, diags = body.Content(schema)
+	}
+
+	val, valDiags := spec.decode(content, block, ctx)
+	diags = append(diags, valDiags...)
+
+	return val, leftovers, diags
+}

zcldec/doc.go

@@ -0,0 +1,12 @@
+// Package zcldec provides a higher-level API for unpacking the content of
+// zcl bodies, implemented in terms of the low-level "Content" API exposed
+// by the bodies themselves.
+//
+// It allows decoding an entire nested configuration in a single operation
+// by providing a description of the intended structure.
+//
+// For some applications it may be more convenient to use the "gozcl"
+// package, which has a similar purpose but decodes directly into native
+// Go data types. zcldec instead targets the cty type system, and thus allows
+// a cty-driven application to remain within that type system.
+package zcldec

zcldec/gob.go

@@ -0,0 +1,22 @@
+package zcldec
+
+import (
+	"encoding/gob"
+)
+
+func init() {
+	// Every Spec implementation should be registered with gob, so that
+	// specs can be sent over gob channels, such as using
+	// github.com/hashicorp/go-plugin with plugins that need to describe
+	// what shape of configuration they are expecting.
+	gob.Register(ObjectSpec(nil))
+	gob.Register(TupleSpec(nil))
+	gob.Register((*AttrSpec)(nil))
+	gob.Register((*LiteralSpec)(nil))
+	gob.Register((*ExprSpec)(nil))
+	gob.Register((*BlockSpec)(nil))
+	gob.Register((*BlockListSpec)(nil))
+	gob.Register((*BlockSetSpec)(nil))
+	gob.Register((*BlockMapSpec)(nil))
+	gob.Register((*BlockLabelSpec)(nil))
+}

zcldec/public.go

@@ -0,0 +1,27 @@
+package zcldec
+
+import (
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+// Decode interprets the given body using the given specification and returns
+// the resulting value. If the given body is not valid per the spec, error
+// diagnostics are returned and the returned value is likely to be incomplete.
+//
+// The ctx argument may be nil, in which case any references to variables or
+// functions will produce error diagnostics.
+func Decode(body zcl.Body, spec Spec, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	val, _, diags := decode(body, nil, ctx, spec, false)
+	return val, diags
+}
+
+// PartialDecode is like Decode except that it permits "leftover" items in
+// the top-level body, which are returned as a new body to allow for
+// further processing.
+//
+// Any descendent block bodies are _not_ decoded partially and thus must
+// be fully described by the given specification.
+func PartialDecode(body zcl.Body, spec Spec, ctx *zcl.EvalContext) (cty.Value, zcl.Body, zcl.Diagnostics) {
+	return decode(body, nil, ctx, spec, true)
+}

zcldec/public_test.go

@@ -0,0 +1,82 @@
+package zcldec
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-zcl/zcl"
+	"github.com/zclconf/go-zcl/zcl/zclsyntax"
+)
+
+func TestDecode(t *testing.T) {
+	tests := []struct {
+		config    string
+		spec      Spec
+		ctx       *zcl.EvalContext
+		want      cty.Value
+		diagCount int
+	}{
+		{
+			``,
+			&ObjectSpec{},
+			nil,
+			cty.EmptyObjectVal,
+			0,
+		},
+		{
+			`a = 1`,
+			&ObjectSpec{},
+			nil,
+			cty.EmptyObjectVal,
+			1, // attribute named "a" is not expected here
+		},
+		{
+			`a = 1`,
+			&ObjectSpec{
+				"a": &AttrSpec{
+					Name: "a",
+					Type: cty.Number,
+				},
+			},
+			nil,
+			cty.ObjectVal(map[string]cty.Value{
+				"a": cty.NumberIntVal(1),
+			}),
+			0,
+		},
+		{
+			`a = 1`,
+			&AttrSpec{
+				Name: "a",
+				Type: cty.Number,
+			},
+			nil,
+			cty.NumberIntVal(1),
+			0,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%02d-%s", i, test.config), func(t *testing.T) {
+			file, parseDiags := zclsyntax.ParseConfig([]byte(test.config), "", zcl.Pos{Line: 1, Column: 1, Byte: 0})
+			body := file.Body
+			got, valDiags := Decode(body, test.spec, test.ctx)
+
+			var diags zcl.Diagnostics
+			diags = append(diags, parseDiags...)
+			diags = append(diags, valDiags...)
+
+			if len(diags) != test.diagCount {
+				t.Errorf("wrong number of diagnostics %d; want %d", len(diags), test.diagCount)
+				for _, diag := range diags {
+					t.Logf(" - %s", diag.Error())
+				}
+			}
+
+			if !got.RawEquals(test.want) {
+				t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, test.want)
+			}
+		})
+	}
+}

zcldec/schema.go

@@ -0,0 +1,34 @@
+package zcldec
+
+import (
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+// ImpliedSchema returns the *zcl.BodySchema implied by the given specification.
+// This is the schema that the Decode function will use internally to
+// access the content of a given body.
+func ImpliedSchema(spec Spec) *zcl.BodySchema {
+	var attrs []zcl.AttributeSchema
+	var blocks []zcl.BlockHeaderSchema
+
+	// visitSameBodyChildren walks through the spec structure, calling
+	// the given callback for each descendent spec encountered. We are
+	// interested in the specs that reference attributes and blocks.
+	visit := func(s Spec) {
+		if as, ok := s.(attrSpec); ok {
+			attrs = append(attrs, as.attrSchemata()...)
+		}
+
+		if bs, ok := s.(blockSpec); ok {
+			blocks = append(blocks, bs.blockHeaderSchemata()...)
+		}
+	}
+
+	visit(spec)
+	spec.visitSameBodyChildren(visit)
+
+	return &zcl.BodySchema{
+		Attributes: attrs,
+		Blocks:     blocks,
+	}
+}

zcldec/spec.go

@@ -0,0 +1,306 @@
+package zcldec
+
+import (
+	"fmt"
+
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+// A Spec is a description of how to decode a zcl.Body to a cty.Value.
+//
+// The various other types in this package whose names end in "Spec" are
+// the spec implementations. The most common top-level spec is ObjectSpec,
+// which decodes body content into a cty.Value of an object type.
+type Spec interface {
+	// Perform the decode operation on the given body, in the context of
+	// the given block (which might be null), using the given eval context.
+	//
+	// "block" is provided only by the nested calls performed by the spec
+	// types that work on block bodies.
+	decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics)
+
+	// Call the given callback once for each of the nested specs that would
+	// get decoded with the same body and block as the receiver. This should
+	// not descend into the nested specs used when decoding blocks.
+	visitSameBodyChildren(cb visitFunc)
+}
+
+type visitFunc func(spec Spec)
+
+// An ObjectSpec is a Spec that produces a cty.Value of an object type whose
+// attributes correspond to the keys of the spec map.
+type ObjectSpec map[string]Spec
+
+// attrSpec is implemented by specs that require attributes from the body.
+type attrSpec interface {
+	attrSchemata() []zcl.AttributeSchema
+}
+
+// blockSpec is implemented by specs that require blocks from the body.
+type blockSpec interface {
+	blockHeaderSchemata() []zcl.BlockHeaderSchema
+}
+
+// specNeedingVariables is implemented by specs that can use variables
+// from the EvalContext, to declare which variables they need.
+type specNeedingVariables interface {
+	variablesNeeded(content *zcl.BodyContent) []zcl.Traversal
+}
+
+func (s ObjectSpec) visitSameBodyChildren(cb visitFunc) {
+	for _, c := range s {
+		cb(c)
+	}
+}
+
+func (s ObjectSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	vals := make(map[string]cty.Value, len(s))
+	var diags zcl.Diagnostics
+
+	for k, spec := range s {
+		var kd zcl.Diagnostics
+		vals[k], kd = spec.decode(content, block, ctx)
+		diags = append(diags, kd...)
+	}
+
+	return cty.ObjectVal(vals), diags
+}
+
+// A TupleSpec is a Spec that produces a cty.Value of a tuple type whose
+// elements correspond to the elements of the spec slice.
+type TupleSpec []Spec
+
+func (s TupleSpec) visitSameBodyChildren(cb visitFunc) {
+	for _, c := range s {
+		cb(c)
+	}
+}
+
+func (s TupleSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	vals := make([]cty.Value, len(s))
+	var diags zcl.Diagnostics
+
+	for i, spec := range s {
+		var ed zcl.Diagnostics
+		vals[i], ed = spec.decode(content, block, ctx)
+		diags = append(diags, ed...)
+	}
+
+	return cty.TupleVal(vals), diags
+}
+
+// An AttrSpec is a Spec that evaluates a particular attribute expression in
+// the body and returns its resulting value converted to the requested type,
+// or produces a diagnostic if the type is incorrect.
+type AttrSpec struct {
+	Name     string
+	Type     cty.Type
+	Required bool
+}
+
+func (s *AttrSpec) visitSameBodyChildren(cb visitFunc) {
+	// leaf node
+}
+
+// specNeedingVariables implementation
+func (s *AttrSpec) variablesNeeded(content *zcl.BodyContent) []zcl.Traversal {
+	attr, exists := content.Attributes[s.Name]
+	if !exists {
+		return nil
+	}
+
+	return attr.Expr.Variables()
+}
+
+// attrSpec implementation
+func (s *AttrSpec) attrSchemata() []zcl.AttributeSchema {
+	return []zcl.AttributeSchema{
+		{
+			Name:     s.Name,
+			Required: s.Required,
+		},
+	}
+}
+
+func (s *AttrSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	attr, exists := content.Attributes[s.Name]
+	if !exists {
+		// We don't need to check required and emit a diagnostic here, because
+		// that would already have happened when building "content".
+		return cty.DynamicVal, nil
+	}
+
+	// TODO: Also try to convert the result value to s.Type
+	return attr.Expr.Value(ctx)
+}
+
+// A LiteralSpec is a Spec that produces the given literal value, ignoring
+// the given body.
+type LiteralSpec struct {
+	Value cty.Value
+}
+
+func (s *LiteralSpec) visitSameBodyChildren(cb visitFunc) {
+	// leaf node
+}
+
+func (s *LiteralSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	return s.Value, nil
+}
+
+// An ExprSpec is a Spec that evaluates the given expression, ignoring the
+// given body.
+type ExprSpec struct {
+	Expr zcl.Expression
+}
+
+func (s *ExprSpec) visitSameBodyChildren(cb visitFunc) {
+	// leaf node
+}
+
+// specNeedingVariables implementation
+func (s *ExprSpec) variablesNeeded(content *zcl.BodyContent) []zcl.Traversal {
+	return s.Expr.Variables()
+}
+
+func (s *ExprSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	return s.Expr.Value(ctx)
+}
+
+// A BlockSpec is a Spec that produces a cty.Value by decoding the contents
+// of a single nested block of a given type, using a nested spec.
+//
+// If the Required flag is not set, the nested block may be omitted, in which
+// case a null value is produced. If it _is_ set, an error diagnostic is
+// produced if there are no nested blocks of the given type.
+type BlockSpec struct {
+	TypeName string
+	Nested   Spec
+	Required bool
+}
+
+func (s *BlockSpec) visitSameBodyChildren(cb visitFunc) {
+	// leaf node ("Nested" does not use the same body)
+}
+
+func (s *BlockSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	var diags zcl.Diagnostics
+
+	var childBlock *zcl.Block
+	for _, candidate := range content.Blocks {
+		if candidate.Type != s.TypeName {
+			continue
+		}
+
+		if childBlock != nil {
+			diags = append(diags, &zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  fmt.Sprintf("Duplicate %s block", s.TypeName),
+				Detail: fmt.Sprintf(
+					"Only one block of type %q is allowed. Previous definition was at %s.",
+					s.TypeName, childBlock.DefRange.String(),
+				),
+				Subject: &candidate.DefRange,
+			})
+			break
+		}
+
+		childBlock = candidate
+	}
+
+	if childBlock == nil {
+		if s.Required {
+			diags = append(diags, &zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  fmt.Sprintf("Missing %s block", s.TypeName),
+				Detail: fmt.Sprintf(
+					"A block of type %q is required here.", s.TypeName,
+				),
+				Subject: &content.MissingItemRange,
+			})
+		}
+		return cty.NullVal(cty.DynamicPseudoType), diags
+	}
+
+	val, _, childDiags := decode(childBlock.Body, childBlock, ctx, s.Nested, false)
+	diags = append(diags, childDiags...)
+	return val, diags
+}
+
+// A BlockListSpec is a Spec that produces a cty list of the results of
+// decoding all of the nested blocks of a given type, using a nested spec.
+type BlockListSpec struct {
+	TypeName string
+	Nested   Spec
+	MinItems int
+	MaxItems int
+}
+
+func (s *BlockListSpec) visitSameBodyChildren(cb visitFunc) {
+	// leaf node ("Nested" does not use the same body)
+}
+
+func (s *BlockListSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	panic("BlockListSpec.decode not yet implemented")
+}
+
+// A BlockSetSpec is a Spec that produces a cty set of the results of
+// decoding all of the nested blocks of a given type, using a nested spec.
+type BlockSetSpec struct {
+	TypeName string
+	Nested   Spec
+	MinItems int
+	MaxItems int
+}
+
+func (s *BlockSetSpec) visitSameBodyChildren(cb visitFunc) {
+	// leaf node ("Nested" does not use the same body)
+}
+
+func (s *BlockSetSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	panic("BlockSetSpec.decode not yet implemented")
+}
+
+// A BlockMapSpec is a Spec that produces a cty map of the results of
+// decoding all of the nested blocks of a given type, using a nested spec.
+//
+// One level of map structure is created for each of the given label names.
+// There must be at least one given label name.
+type BlockMapSpec struct {
+	TypeName   string
+	LabelNames []string
+	Nested     Spec
+}
+
+func (s *BlockMapSpec) visitSameBodyChildren(cb visitFunc) {
+	// leaf node ("Nested" does not use the same body)
+}
+
+func (s *BlockMapSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	panic("BlockMapSpec.decode not yet implemented")
+}
+
+// A BlockLabelSpec is a Spec that returns a cty.String representing the
+// label of the block its given body belongs to, if indeed its given body
+// belongs to a block. It is a programming error to use this in a non-block
+// context, so this spec will panic in that case.
+//
+// This spec only works in the nested spec within a BlockSpec, BlockListSpec,
+// BlockSetSpec or BlockMapSpec.
+//
+// The full set of label specs used against a particular block must have a
+// consecutive set of indices starting at zero. The maximum index found
+// defines how many labels the corresponding blocks must have in cty source.
+type BlockLabelSpec struct {
+	Index int
+	Name  string
+}
+
+func (s *BlockLabelSpec) visitSameBodyChildren(cb visitFunc) {
+	// leaf node
+}
+
+func (s *BlockLabelSpec) decode(content *zcl.BodyContent, block *zcl.Block, ctx *zcl.EvalContext) (cty.Value, zcl.Diagnostics) {
+	panic("BlockLabelSpec.decode not yet implemented")
+}

zcldec/variables.go

@@ -0,0 +1,37 @@
+package zcldec
+
+import (
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+// Variables processes the given body with the given spec and returns a
+// list of the variable traversals that would be required to decode
+// the same pairing of body and spec.
+//
+// This can be used to conditionally populate the variables in the EvalContext
+// passed to Decode, for applications where a static scope is insufficient.
+//
+// If the given body is not compliant with the given schema, diagnostics are
+// returned describing the problem, which could also serve as a pre-evaluation
+// partial validation step.
+func Variables(body zcl.Body, spec Spec) ([]zcl.Traversal, zcl.Diagnostics) {
+	schema := ImpliedSchema(spec)
+
+	content, _, diags := body.PartialContent(schema)
+
+	var vars []zcl.Traversal
+	if diags.HasErrors() {
+		return vars, diags
+	}
+
+	if vs, ok := spec.(specNeedingVariables); ok {
+		vars = append(vars, vs.variablesNeeded(content)...)
+	}
+	spec.visitSameBodyChildren(func(s Spec) {
+		if vs, ok := s.(specNeedingVariables); ok {
+			vars = append(vars, vs.variablesNeeded(content)...)
+		}
+	})
+
+	return vars, diags
+}