From 523939034f6b07dcd1871a3ccbe2466dcc9688fd Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 27 Jul 2017 18:15:56 -0700 Subject: [PATCH] ext/include: extension for including bodies into other bodies This package implements a language extension that allows configuration authors to include the content of another file into a body, using syntax like this: include { path = "./foo.zcl" } This is implemented as a transform.Transformer so that it can be used as part of a transform chain when decoding nested block structures to allow includes at any arbitrary point. This capability is not built into the language because certain applications will offer higher-level constructs for connecting multiple separate config files, which may e.g. have a separate evaluation scope for each file, etc. --- ext/include/doc.go | 12 ++++ ext/include/file_resolver.go | 52 +++++++++++++++ ext/include/map_resolver.go | 29 +++++++++ ext/include/resolver.go | 28 ++++++++ ext/include/transformer.go | 92 ++++++++++++++++++++++++++ ext/include/transformer_test.go | 112 ++++++++++++++++++++++++++++++++ 6 files changed, 325 insertions(+) create mode 100644 ext/include/doc.go create mode 100644 ext/include/file_resolver.go create mode 100644 ext/include/map_resolver.go create mode 100644 ext/include/resolver.go create mode 100644 ext/include/transformer.go create mode 100644 ext/include/transformer_test.go diff --git a/ext/include/doc.go b/ext/include/doc.go new file mode 100644 index 0000000..efcf0af --- /dev/null +++ b/ext/include/doc.go @@ -0,0 +1,12 @@ +// Package include implements a zcl extension that allows inclusion of +// one zcl body into another using blocks of type "include", with the following +// structure: +// +// include { +// path = "./foo.zcl" +// } +// +// The processing of the given path is delegated to the calling application, +// allowing it to decide how to interpret the path and which syntaxes to +// support for referenced files. +package include diff --git a/ext/include/file_resolver.go b/ext/include/file_resolver.go new file mode 100644 index 0000000..5e5578b --- /dev/null +++ b/ext/include/file_resolver.go @@ -0,0 +1,52 @@ +package include + +import ( + "path/filepath" + "strings" + + "github.com/zclconf/go-zcl/zcl" + "github.com/zclconf/go-zcl/zclparse" +) + +// FileResolver creates and returns a Resolver that interprets include paths +// as filesystem paths relative to the calling configuration file. +// +// When an include is requested, the source filename of the calling config +// file is first interpreted relative to the given basePath, and then the +// path given in configuration is interpreted relative to the resulting +// absolute caller configuration directory. +// +// This resolver assumes that all calling bodies are loaded from local files +// and that the paths to these files were correctly provided to the parser, +// either absolute or relative to the given basePath. +// +// If the path given in configuration ends with ".json" then the referenced +// file is interpreted as JSON. Otherwise, it is interpreted as zcl native +// syntax. +func FileResolver(baseDir string, parser *zclparse.Parser) Resolver { + return &fileResolver{ + BaseDir: baseDir, + Parser: parser, + } +} + +type fileResolver struct { + BaseDir string + Parser *zclparse.Parser +} + +func (r fileResolver) ResolveBodyPath(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics) { + callerFile := filepath.Join(r.BaseDir, refRange.Filename) + callerDir := filepath.Dir(callerFile) + targetFile := filepath.Join(callerDir, path) + + var f *zcl.File + var diags zcl.Diagnostics + if strings.HasSuffix(targetFile, ".json") { + f, diags = r.Parser.ParseJSONFile(targetFile) + } else { + f, diags = r.Parser.ParseZCLFile(targetFile) + } + + return f.Body, diags +} diff --git a/ext/include/map_resolver.go b/ext/include/map_resolver.go new file mode 100644 index 0000000..0711067 --- /dev/null +++ b/ext/include/map_resolver.go @@ -0,0 +1,29 @@ +package include + +import ( + "fmt" + + "github.com/zclconf/go-zcl/zcl" +) + +// MapResolver returns a Resolver that consults the given map for preloaded +// bodies (the values) associated with static include paths (the keys). +// +// An error diagnostic is returned if a path is requested that does not appear +// as a key in the given map. +func MapResolver(m map[string]zcl.Body) Resolver { + return ResolverFunc(func(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics) { + if body, ok := m[path]; ok { + return body, nil + } + + return nil, zcl.Diagnostics{ + { + Severity: zcl.DiagError, + Summary: "Invalid include path", + Detail: fmt.Sprintf("The include path %q is not recognized.", path), + Subject: &refRange, + }, + } + }) +} diff --git a/ext/include/resolver.go b/ext/include/resolver.go new file mode 100644 index 0000000..824407d --- /dev/null +++ b/ext/include/resolver.go @@ -0,0 +1,28 @@ +package include + +import ( + "github.com/zclconf/go-zcl/zcl" +) + +// A Resolver maps an include path (an arbitrary string, but usually something +// filepath-like) to a zcl.Body. +// +// The parameter "refRange" is the source range of the expression in the calling +// body that provided the given path, for use in generating "invalid path"-type +// diagnostics. +// +// If the returned body is nil, it will be ignored. +// +// Any returned diagnostics will be emitted when content is requested from the +// final composed body (after all includes have been dealt with). +type Resolver interface { + ResolveBodyPath(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics) +} + +// ResolverFunc is a function type that implements Resolver. +type ResolverFunc func(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics) + +// ResolveBodyPath is an implementation of Resolver.ResolveBodyPath. +func (f ResolverFunc) ResolveBodyPath(path string, refRange zcl.Range) (zcl.Body, zcl.Diagnostics) { + return f(path, refRange) +} diff --git a/ext/include/transformer.go b/ext/include/transformer.go new file mode 100644 index 0000000..a7447f0 --- /dev/null +++ b/ext/include/transformer.go @@ -0,0 +1,92 @@ +package include + +import ( + "github.com/zclconf/go-zcl/ext/transform" + "github.com/zclconf/go-zcl/gozcl" + "github.com/zclconf/go-zcl/zcl" +) + +// Transformer builds a transformer that finds any "include" blocks in a body +// and produces a merged body that contains the original content plus the +// content of the other bodies referenced by the include blocks. +// +// blockType specifies the type of block to interpret. The conventional type name +// is "include". +// +// ctx provides an evaluation context for the path expressions in include blocks. +// If nil, path expressions may not reference variables nor functions. +// +// The given resolver is used to translate path strings (after expression +// evaluation) into bodies. FileResolver returns a reasonable implementation for +// applications that read configuration files from local disk. +// +// The returned Transformer can either be used directly to process includes +// in a shallow fashion on a single body, or it can be used with +// transform.Deep (from the sibling transform package) to allow includes +// at all levels of a nested block structure: +// +// transformer = include.Transformer("include", nil, include.FileResolver(".", parser)) +// body = transform.Deep(body, transformer) +// // "body" will now have includes resolved in its own content and that +// // of any descendent blocks. +// +func Transformer(blockType string, ctx *zcl.EvalContext, resolver Resolver) transform.Transformer { + return &transformer{ + Schema: &zcl.BodySchema{ + Blocks: []zcl.BlockHeaderSchema{ + { + Type: blockType, + }, + }, + }, + Ctx: ctx, + Resolver: resolver, + } +} + +type transformer struct { + Schema *zcl.BodySchema + Ctx *zcl.EvalContext + Resolver Resolver +} + +func (t *transformer) TransformBody(in zcl.Body) zcl.Body { + content, remain, diags := in.PartialContent(t.Schema) + + if content == nil || len(content.Blocks) == 0 { + // Nothing to do! + return transform.BodyWithDiagnostics(remain, diags) + } + + bodies := make([]zcl.Body, 1, len(content.Blocks)+1) + bodies[0] = remain // content in "remain" takes priority over includes + for _, block := range content.Blocks { + incContent, incDiags := block.Body.Content(includeBlockSchema) + diags = append(diags, incDiags...) + if incDiags.HasErrors() { + continue + } + + pathExpr := incContent.Attributes["path"].Expr + var path string + incDiags = gozcl.DecodeExpression(pathExpr, t.Ctx, &path) + diags = append(diags, incDiags...) + if incDiags.HasErrors() { + continue + } + + incBody, incDiags := t.Resolver.ResolveBodyPath(path, pathExpr.Range()) + bodies = append(bodies, transform.BodyWithDiagnostics(incBody, incDiags)) + } + + return zcl.MergeBodies(bodies) +} + +var includeBlockSchema = &zcl.BodySchema{ + Attributes: []zcl.AttributeSchema{ + { + Name: "path", + Required: true, + }, + }, +} diff --git a/ext/include/transformer_test.go b/ext/include/transformer_test.go new file mode 100644 index 0000000..4babbc8 --- /dev/null +++ b/ext/include/transformer_test.go @@ -0,0 +1,112 @@ +package include + +import ( + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-zcl/gozcl" + "github.com/zclconf/go-zcl/zcl" + "github.com/zclconf/go-zcl/zcltest" +) + +func TestTransformer(t *testing.T) { + caller := zcltest.MockBody(&zcl.BodyContent{ + Blocks: zcl.Blocks{ + { + Type: "include", + Body: zcltest.MockBody(&zcl.BodyContent{ + Attributes: zcltest.MockAttrs(map[string]zcl.Expression{ + "path": zcltest.MockExprVariable("var_path"), + }), + }), + }, + { + Type: "include", + Body: zcltest.MockBody(&zcl.BodyContent{ + Attributes: zcltest.MockAttrs(map[string]zcl.Expression{ + "path": zcltest.MockExprLiteral(cty.StringVal("include2")), + }), + }), + }, + { + Type: "foo", + Body: zcltest.MockBody(&zcl.BodyContent{ + Attributes: zcltest.MockAttrs(map[string]zcl.Expression{ + "from": zcltest.MockExprLiteral(cty.StringVal("caller")), + }), + }), + }, + }, + }) + + resolver := MapResolver(map[string]zcl.Body{ + "include1": zcltest.MockBody(&zcl.BodyContent{ + Blocks: zcl.Blocks{ + { + Type: "foo", + Body: zcltest.MockBody(&zcl.BodyContent{ + Attributes: zcltest.MockAttrs(map[string]zcl.Expression{ + "from": zcltest.MockExprLiteral(cty.StringVal("include1")), + }), + }), + }, + }, + }), + "include2": zcltest.MockBody(&zcl.BodyContent{ + Blocks: zcl.Blocks{ + { + Type: "foo", + Body: zcltest.MockBody(&zcl.BodyContent{ + Attributes: zcltest.MockAttrs(map[string]zcl.Expression{ + "from": zcltest.MockExprLiteral(cty.StringVal("include2")), + }), + }), + }, + }, + }), + }) + + ctx := &zcl.EvalContext{ + Variables: map[string]cty.Value{ + "var_path": cty.StringVal("include1"), + }, + } + + transformer := Transformer("include", ctx, resolver) + merged := transformer.TransformBody(caller) + + type foo struct { + From string `zcl:"from,attr"` + } + type result struct { + Foos []foo `zcl:"foo,block"` + } + var got result + diags := gozcl.DecodeBody(merged, nil, &got) + if len(diags) != 0 { + t.Errorf("unexpected diags") + for _, diag := range diags { + t.Logf("- %s", diag) + } + } + + want := result{ + Foos: []foo{ + { + From: "caller", + }, + { + From: "include1", + }, + { + From: "include2", + }, + }, + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want)) + } +}