ext/include: extension for including bodies into other bodies

This package implements a language extension that allows configuration
authors to include the content of another file into a body, using syntax
like this:

  include {
    path = "./foo.zcl"
  }

This is implemented as a transform.Transformer so that it can be used
as part of a transform chain when decoding nested block structures to
allow includes at any arbitrary point.

This capability is not built into the language because certain
applications will offer higher-level constructs for connecting multiple
separate config files, which may e.g. have a separate evaluation scope
for each file, etc.

ext/include/doc.go

@@ -0,0 +1,12 @@
+// Package include implements a zcl extension that allows inclusion of
+// one zcl body into another using blocks of type "include", with the following
+// structure:
+//
+//     include {
+//       path = "./foo.zcl"
+//     }
+//
+// The processing of the given path is delegated to the calling application,
+// allowing it to decide how to interpret the path and which syntaxes to
+// support for referenced files.
+package include

ext/include/file_resolver.go

@@ -0,0 +1,52 @@
+package include
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/zclconf/go-zcl/zcl"
+	"github.com/zclconf/go-zcl/zclparse"
+)
+
+// FileResolver creates and returns a Resolver that interprets include paths
+// as filesystem paths relative to the calling configuration file.
+//
+// When an include is requested, the source filename of the calling config
+// file is first interpreted relative to the given basePath, and then the
+// path given in configuration is interpreted relative to the resulting
+// absolute caller configuration directory.
+//
+// This resolver assumes that all calling bodies are loaded from local files
+// and that the paths to these files were correctly provided to the parser,
+// either absolute or relative to the given basePath.
+//
+// If the path given in configuration ends with ".json" then the referenced
+// file is interpreted as JSON. Otherwise, it is interpreted as zcl native
+// syntax.
+func FileResolver(baseDir string, parser *zclparse.Parser) Resolver {
+	return &fileResolver{
+		BaseDir: baseDir,
+		Parser:  parser,
+	}
+}
+
+type fileResolver struct {
+	BaseDir string
+	Parser  *zclparse.Parser
+}
+
+func (r fileResolver) ResolveBodyPath(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics) {
+	callerFile := filepath.Join(r.BaseDir, refRange.Filename)
+	callerDir := filepath.Dir(callerFile)
+	targetFile := filepath.Join(callerDir, path)
+
+	var f *zcl.File
+	var diags zcl.Diagnostics
+	if strings.HasSuffix(targetFile, ".json") {
+		f, diags = r.Parser.ParseJSONFile(targetFile)
+	} else {
+		f, diags = r.Parser.ParseZCLFile(targetFile)
+	}
+
+	return f.Body, diags
+}

ext/include/map_resolver.go

@@ -0,0 +1,29 @@
+package include
+
+import (
+	"fmt"
+
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+// MapResolver returns a Resolver that consults the given map for preloaded
+// bodies (the values) associated with static include paths (the keys).
+//
+// An error diagnostic is returned if a path is requested that does not appear
+// as a key in the given map.
+func MapResolver(m map[string]zcl.Body) Resolver {
+	return ResolverFunc(func(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics) {
+		if body, ok := m[path]; ok {
+			return body, nil
+		}
+
+		return nil, zcl.Diagnostics{
+			{
+				Severity: zcl.DiagError,
+				Summary:  "Invalid include path",
+				Detail:   fmt.Sprintf("The include path %q is not recognized.", path),
+				Subject:  &refRange,
+			},
+		}
+	})
+}

ext/include/resolver.go

@@ -0,0 +1,28 @@
+package include
+
+import (
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+// A Resolver maps an include path (an arbitrary string, but usually something
+// filepath-like) to a zcl.Body.
+//
+// The parameter "refRange" is the source range of the expression in the calling
+// body that provided the given path, for use in generating "invalid path"-type
+// diagnostics.
+//
+// If the returned body is nil, it will be ignored.
+//
+// Any returned diagnostics will be emitted when content is requested from the
+// final composed body (after all includes have been dealt with).
+type Resolver interface {
+	ResolveBodyPath(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics)
+}
+
+// ResolverFunc is a function type that implements Resolver.
+type ResolverFunc func(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics)
+
+// ResolveBodyPath is an implementation of Resolver.ResolveBodyPath.
+func (f ResolverFunc) ResolveBodyPath(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics) {
+	return f(path, refRange)
+}

ext/include/transformer.go

@@ -0,0 +1,92 @@
+package include
+
+import (
+	"github.com/zclconf/go-zcl/ext/transform"
+	"github.com/zclconf/go-zcl/gozcl"
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+// Transformer builds a transformer that finds any "include" blocks in a body
+// and produces a merged body that contains the original content plus the
+// content of the other bodies referenced by the include blocks.
+//
+// blockType specifies the type of block to interpret. The conventional type name
+// is "include".
+//
+// ctx provides an evaluation context for the path expressions in include blocks.
+// If nil, path expressions may not reference variables nor functions.
+//
+// The given resolver is used to translate path strings (after expression
+// evaluation) into bodies. FileResolver returns a reasonable implementation for
+// applications that read configuration files from local disk.
+//
+// The returned Transformer can either be used directly to process includes
+// in a shallow fashion on a single body, or it can be used with
+// transform.Deep (from the sibling transform package) to allow includes
+// at all levels of a nested block structure:
+//
+//    transformer = include.Transformer("include", nil, include.FileResolver(".", parser))
+//    body = transform.Deep(body, transformer)
+//    // "body" will now have includes resolved in its own content and that
+//    // of any descendent blocks.
+//
+func Transformer(blockType string, ctx *zcl.EvalContext, resolver Resolver) transform.Transformer {
+	return &transformer{
+		Schema: &zcl.BodySchema{
+			Blocks: []zcl.BlockHeaderSchema{
+				{
+					Type: blockType,
+				},
+			},
+		},
+		Ctx:      ctx,
+		Resolver: resolver,
+	}
+}
+
+type transformer struct {
+	Schema   *zcl.BodySchema
+	Ctx      *zcl.EvalContext
+	Resolver Resolver
+}
+
+func (t *transformer) TransformBody(in zcl.Body) zcl.Body {
+	content, remain, diags := in.PartialContent(t.Schema)
+
+	if content == nil || len(content.Blocks) == 0 {
+		// Nothing to do!
+		return transform.BodyWithDiagnostics(remain, diags)
+	}
+
+	bodies := make([]zcl.Body, 1, len(content.Blocks)+1)
+	bodies[0] = remain // content in "remain" takes priority over includes
+	for _, block := range content.Blocks {
+		incContent, incDiags := block.Body.Content(includeBlockSchema)
+		diags = append(diags, incDiags...)
+		if incDiags.HasErrors() {
+			continue
+		}
+
+		pathExpr := incContent.Attributes["path"].Expr
+		var path string
+		incDiags = gozcl.DecodeExpression(pathExpr, t.Ctx, &path)
+		diags = append(diags, incDiags...)
+		if incDiags.HasErrors() {
+			continue
+		}
+
+		incBody, incDiags := t.Resolver.ResolveBodyPath(path, pathExpr.Range())
+		bodies = append(bodies, transform.BodyWithDiagnostics(incBody, incDiags))
+	}
+
+	return zcl.MergeBodies(bodies)
+}
+
+var includeBlockSchema = &zcl.BodySchema{
+	Attributes: []zcl.AttributeSchema{
+		{
+			Name:     "path",
+			Required: true,
+		},
+	},
+}

ext/include/transformer_test.go

@@ -0,0 +1,112 @@
+package include
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-zcl/gozcl"
+	"github.com/zclconf/go-zcl/zcl"
+	"github.com/zclconf/go-zcl/zcltest"
+)
+
+func TestTransformer(t *testing.T) {
+	caller := zcltest.MockBody(&zcl.BodyContent{
+		Blocks: zcl.Blocks{
+			{
+				Type: "include",
+				Body: zcltest.MockBody(&zcl.BodyContent{
+					Attributes: zcltest.MockAttrs(map[string]zcl.Expression{
+						"path": zcltest.MockExprVariable("var_path"),
+					}),
+				}),
+			},
+			{
+				Type: "include",
+				Body: zcltest.MockBody(&zcl.BodyContent{
+					Attributes: zcltest.MockAttrs(map[string]zcl.Expression{
+						"path": zcltest.MockExprLiteral(cty.StringVal("include2")),
+					}),
+				}),
+			},
+			{
+				Type: "foo",
+				Body: zcltest.MockBody(&zcl.BodyContent{
+					Attributes: zcltest.MockAttrs(map[string]zcl.Expression{
+						"from": zcltest.MockExprLiteral(cty.StringVal("caller")),
+					}),
+				}),
+			},
+		},
+	})
+
+	resolver := MapResolver(map[string]zcl.Body{
+		"include1": zcltest.MockBody(&zcl.BodyContent{
+			Blocks: zcl.Blocks{
+				{
+					Type: "foo",
+					Body: zcltest.MockBody(&zcl.BodyContent{
+						Attributes: zcltest.MockAttrs(map[string]zcl.Expression{
+							"from": zcltest.MockExprLiteral(cty.StringVal("include1")),
+						}),
+					}),
+				},
+			},
+		}),
+		"include2": zcltest.MockBody(&zcl.BodyContent{
+			Blocks: zcl.Blocks{
+				{
+					Type: "foo",
+					Body: zcltest.MockBody(&zcl.BodyContent{
+						Attributes: zcltest.MockAttrs(map[string]zcl.Expression{
+							"from": zcltest.MockExprLiteral(cty.StringVal("include2")),
+						}),
+					}),
+				},
+			},
+		}),
+	})
+
+	ctx := &zcl.EvalContext{
+		Variables: map[string]cty.Value{
+			"var_path": cty.StringVal("include1"),
+		},
+	}
+
+	transformer := Transformer("include", ctx, resolver)
+	merged := transformer.TransformBody(caller)
+
+	type foo struct {
+		From string `zcl:"from,attr"`
+	}
+	type result struct {
+		Foos []foo `zcl:"foo,block"`
+	}
+	var got result
+	diags := gozcl.DecodeBody(merged, nil, &got)
+	if len(diags) != 0 {
+		t.Errorf("unexpected diags")
+		for _, diag := range diags {
+			t.Logf("- %s", diag)
+		}
+	}
+
+	want := result{
+		Foos: []foo{
+			{
+				From: "caller",
+			},
+			{
+				From: "include1",
+			},
+			{
+				From: "include2",
+			},
+		},
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want))
+	}
+}