hclpack: New package for wire representations of hcl bodies
In most applications it's possible to fully evaluate configuration at the beginning and work only with resolved values after that, but in some unusual cases it's necessary to split parsing and decoding between two separate processes connected by a pipe or network connection. hclpack is intended to provide compact wire formats for sending bodies over the network such that they can be decoded and evaluated and get the same results. This is not something that can happen fully automatically because a hcl.Body is an abstract node rather than a physical construct, and so access to the original source code is required to construct such a representation, and to interpret any source ranges that emerged from the final evaluation.
This commit is contained in:
Martin Atkins
parent
0a1bd51123
commit
5e07d8e1f9
24
hclpack/didyoumean.go
Normal file
24
hclpack/didyoumean.go
Normal file
@ -0,0 +1,24 @@
|
||||
package hclpack
|
||||
|
||||
import (
|
||||
"github.com/agext/levenshtein"
|
||||
)
|
||||
|
||||
// nameSuggestion tries to find a name from the given slice of suggested names
|
||||
// that is close to the given name and returns it if found. If no suggestion
|
||||
// is close enough, returns the empty string.
|
||||
//
|
||||
// The suggestions are tried in order, so earlier suggestions take precedence
|
||||
// if the given string is similar to two or more suggestions.
|
||||
//
|
||||
// This function is intended to be used with a relatively-small number of
|
||||
// suggestions. It's not optimized for hundreds or thousands of them.
|
||||
func nameSuggestion(given string, suggestions []string) string {
|
||||
for _, suggestion := range suggestions {
|
||||
dist := levenshtein.Distance(given, suggestion, nil)
|
||||
if dist < 3 { // threshold determined experimentally
|
||||
return suggestion
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
14
hclpack/doc.go
Normal file
14
hclpack/doc.go
Normal file
@ -0,0 +1,14 @@
|
||||
// Package hclpack provides a straightforward representation of HCL block/body
|
||||
// structure that can be easily serialized and deserialized for compact
|
||||
// transmission (e.g. over a network) without transmitting the full source code.
|
||||
//
|
||||
// Expressions are retained in native syntax source form so that their
|
||||
// evaluation can be delayed until a package structure is decoded by some
|
||||
// other system that has enough information to populate the evaluation context.
|
||||
//
|
||||
// Packed structures retain source location information but do not retain
|
||||
// actual source code. To make sense of source locations returned in diagnostics
|
||||
// and via other APIs the caller must somehow gain access to the original source
|
||||
// code that the packed representation was built from, which is a problem that
|
||||
// must be solved somehow by the calling application.
|
||||
package hclpack
|
||||
104
hclpack/expression.go
Normal file
104
hclpack/expression.go
Normal file
@ -0,0 +1,104 @@
|
||||
package hclpack
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/hashicorp/hcl2/hcl"
|
||||
"github.com/hashicorp/hcl2/hcl/hclsyntax"
|
||||
"github.com/zclconf/go-cty/cty"
|
||||
)
|
||||
|
||||
// Expression is an implementation of hcl.Expression in terms of some raw
|
||||
// expression source code. The methods of this type will first parse the
|
||||
// source code and then pass the call through to the real expression that
|
||||
// is produced.
|
||||
type Expression struct {
|
||||
// Source is the raw source code of the expression, which should be parsed
|
||||
// as the syntax specified by SourceType.
|
||||
Source []byte
|
||||
SourceType ExprSourceType
|
||||
|
||||
// Range_ and StartRange_ describe the physical extents of the expression
|
||||
// in the original source code. SourceRange_ is its entire range while
|
||||
// StartRange is just the tokens that introduce the expression type. For
|
||||
// simple expression types, SourceRange and StartRange are identical.
|
||||
Range_, StartRange_ hcl.Range
|
||||
}
|
||||
|
||||
var _ hcl.Expression = (*Expression)(nil)
|
||||
|
||||
// Value implements the Value method of hcl.Expression but with the additional
|
||||
// step of first parsing the expression source code. This implementation is
|
||||
// unusual in that it can potentially return syntax errors, whereas other
|
||||
// Value implementations usually work with already-parsed expressions.
|
||||
func (e *Expression) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
|
||||
expr, diags := e.Parse()
|
||||
if diags.HasErrors() {
|
||||
return cty.DynamicVal, diags
|
||||
}
|
||||
|
||||
val, moreDiags := expr.Value(ctx)
|
||||
diags = append(diags, moreDiags...)
|
||||
return val, diags
|
||||
}
|
||||
|
||||
// Variables implements the Variables method of hcl.Expression but with the
|
||||
// additional step of first parsing the expression source code.
|
||||
//
|
||||
// Since this method cannot return errors, it will return a nil slice if
|
||||
// parsing fails, indicating that no variables are present. This is okay in
|
||||
// practice because a subsequent call to Value would fail with syntax errors
|
||||
// regardless of what variables are in the context.
|
||||
func (e *Expression) Variables() []hcl.Traversal {
|
||||
expr, diags := e.Parse()
|
||||
if diags.HasErrors() {
|
||||
return nil
|
||||
}
|
||||
return expr.Variables()
|
||||
}
|
||||
|
||||
func (e *Expression) Range() hcl.Range {
|
||||
return e.Range_
|
||||
}
|
||||
|
||||
func (e *Expression) StartRange() hcl.Range {
|
||||
return e.StartRange_
|
||||
}
|
||||
|
||||
// Parse attempts to parse the source code of the receiving expression using
|
||||
// its indicated source type, returning the expression if possible and any
|
||||
// diagnostics produced during parsing.
|
||||
func (e *Expression) Parse() (hcl.Expression, hcl.Diagnostics) {
|
||||
switch e.SourceType {
|
||||
case ExprNative:
|
||||
return hclsyntax.ParseExpression(e.Source, e.Range_.Filename, e.Range_.Start)
|
||||
case ExprTemplate:
|
||||
return hclsyntax.ParseTemplate(e.Source, e.Range_.Filename, e.Range_.Start)
|
||||
default:
|
||||
// This should never happen for a valid Expression.
|
||||
return nil, hcl.Diagnostics{
|
||||
{
|
||||
Severity: hcl.DiagError,
|
||||
Summary: "Invalid expression source type",
|
||||
Detail: fmt.Sprintf("Packed version of this expression has an invalid source type %s. This is always a bug.", e.SourceType),
|
||||
Subject: &e.Range_,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ExprSourceType defines the syntax type used for an expression's source code,
|
||||
// which is then used to select a suitable parser for it when evaluating.
|
||||
type ExprSourceType rune
|
||||
|
||||
//go:generate stringer -type ExprSourceType
|
||||
|
||||
const (
|
||||
// ExprNative indicates that an expression must be parsed as native
|
||||
// expression syntax, with hclsyntax.ParseExpression.
|
||||
ExprNative ExprSourceType = 'N'
|
||||
|
||||
// ExprTemplate indicates that an expression must be parsed as nave
|
||||
// template syntax, with hclsyntax.ParseTemplate.
|
||||
ExprTemplate ExprSourceType = 'T'
|
||||
)
|
||||
70
hclpack/expression_test.go
Normal file
70
hclpack/expression_test.go
Normal file
@ -0,0 +1,70 @@
|
||||
package hclpack
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/hcl2/hcl"
|
||||
"github.com/zclconf/go-cty/cty"
|
||||
)
|
||||
|
||||
func TestExpressionValue(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
Expr *Expression
|
||||
Ctx *hcl.EvalContext
|
||||
Want cty.Value
|
||||
}{
|
||||
"simple literal expr": {
|
||||
&Expression{
|
||||
Source: []byte(`"hello"`),
|
||||
SourceType: ExprNative,
|
||||
},
|
||||
nil,
|
||||
cty.StringVal("hello"),
|
||||
},
|
||||
"simple literal template": {
|
||||
&Expression{
|
||||
Source: []byte(`hello ${5}`),
|
||||
SourceType: ExprTemplate,
|
||||
},
|
||||
nil,
|
||||
cty.StringVal("hello 5"),
|
||||
},
|
||||
"expr with variable": {
|
||||
&Expression{
|
||||
Source: []byte(`foo`),
|
||||
SourceType: ExprNative,
|
||||
},
|
||||
&hcl.EvalContext{
|
||||
Variables: map[string]cty.Value{
|
||||
"foo": cty.StringVal("bar"),
|
||||
},
|
||||
},
|
||||
cty.StringVal("bar"),
|
||||
},
|
||||
"template with variable": {
|
||||
&Expression{
|
||||
Source: []byte(`foo ${foo}`),
|
||||
SourceType: ExprTemplate,
|
||||
},
|
||||
&hcl.EvalContext{
|
||||
Variables: map[string]cty.Value{
|
||||
"foo": cty.StringVal("bar"),
|
||||
},
|
||||
},
|
||||
cty.StringVal("foo bar"),
|
||||
},
|
||||
}
|
||||
|
||||
for name, test := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, diags := test.Expr.Value(test.Ctx)
|
||||
for _, diag := range diags {
|
||||
t.Errorf("unexpected diagnostic: %s", diag.Error())
|
||||
}
|
||||
|
||||
if !test.Want.RawEquals(got) {
|
||||
t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, test.Want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
21
hclpack/exprsourcetype_string.go
Normal file
21
hclpack/exprsourcetype_string.go
Normal file
@ -0,0 +1,21 @@
|
||||
// Code generated by "stringer -type ExprSourceType"; DO NOT EDIT.
|
||||
|
||||
package hclpack
|
||||
|
||||
import "strconv"
|
||||
|
||||
const (
|
||||
_ExprSourceType_name_0 = "ExprNative"
|
||||
_ExprSourceType_name_1 = "ExprTemplate"
|
||||
)
|
||||
|
||||
func (i ExprSourceType) String() string {
|
||||
switch {
|
||||
case i == 78:
|
||||
return _ExprSourceType_name_0
|
||||
case i == 84:
|
||||
return _ExprSourceType_name_1
|
||||
default:
|
||||
return "ExprSourceType(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
}
|
||||
256
hclpack/structure.go
Normal file
256
hclpack/structure.go
Normal file
@ -0,0 +1,256 @@
|
||||
package hclpack
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/hashicorp/hcl2/hcl"
|
||||
)
|
||||
|
||||
// Body is an implementation of hcl.Body.
|
||||
type Body struct {
|
||||
Attributes map[string]Attribute
|
||||
ChildBlocks []Block
|
||||
|
||||
MissingItemRange_ hcl.Range
|
||||
}
|
||||
|
||||
var _ hcl.Body = (*Body)(nil)
|
||||
|
||||
// Content is an implementation of the method of the same name on hcl.Body.
|
||||
func (b *Body) Content(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Diagnostics) {
|
||||
return b.content(schema, nil)
|
||||
}
|
||||
|
||||
// PartialContent is an implementation of the method of the same name on hcl.Body.
|
||||
//
|
||||
// The returned "remain" body may share some backing objects with the receiver,
|
||||
// so neither the receiver nor the returned remain body, or any descendent
|
||||
// objects within them, may be mutated after this method is used.
|
||||
func (b *Body) PartialContent(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Body, hcl.Diagnostics) {
|
||||
remain := &Body{}
|
||||
content, diags := b.content(schema, remain)
|
||||
return content, remain, diags
|
||||
}
|
||||
|
||||
func (b *Body) content(schema *hcl.BodySchema, remain *Body) (*hcl.BodyContent, hcl.Diagnostics) {
|
||||
if b == nil {
|
||||
b = &Body{} // We'll treat a nil body like an empty one, for convenience
|
||||
}
|
||||
var diags hcl.Diagnostics
|
||||
|
||||
var attrs map[string]*hcl.Attribute
|
||||
var attrUsed map[string]struct{}
|
||||
if len(b.Attributes) > 0 {
|
||||
attrs = make(map[string]*hcl.Attribute, len(b.Attributes))
|
||||
attrUsed = make(map[string]struct{}, len(b.Attributes))
|
||||
}
|
||||
for _, attrS := range schema.Attributes {
|
||||
name := attrS.Name
|
||||
attr, exists := b.Attributes[name]
|
||||
if !exists {
|
||||
if attrS.Required {
|
||||
diags = append(diags, &hcl.Diagnostic{
|
||||
Severity: hcl.DiagError,
|
||||
Summary: "Missing required argument",
|
||||
Detail: fmt.Sprintf("The argument %q is required, but no definition was found.", attrS.Name),
|
||||
Subject: &b.MissingItemRange_,
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
attrs[name] = attr.asHCLAttribute(name)
|
||||
attrUsed[name] = struct{}{}
|
||||
}
|
||||
|
||||
for name, attr := range b.Attributes {
|
||||
if _, used := attrUsed[name]; used {
|
||||
continue
|
||||
}
|
||||
if remain != nil {
|
||||
remain.setAttribute(name, attr)
|
||||
continue
|
||||
}
|
||||
var suggestions []string
|
||||
for _, attrS := range schema.Attributes {
|
||||
if _, defined := attrs[name]; defined {
|
||||
continue
|
||||
}
|
||||
suggestions = append(suggestions, attrS.Name)
|
||||
}
|
||||
suggestion := nameSuggestion(name, suggestions)
|
||||
if suggestion != "" {
|
||||
suggestion = fmt.Sprintf(" Did you mean %q?", suggestion)
|
||||
} else {
|
||||
// Is there a block of the same name?
|
||||
for _, blockS := range schema.Blocks {
|
||||
if blockS.Type == name {
|
||||
suggestion = fmt.Sprintf(" Did you mean to define a block of type %q?", name)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
diags = append(diags, &hcl.Diagnostic{
|
||||
Severity: hcl.DiagError,
|
||||
Summary: "Unsupported argument",
|
||||
Detail: fmt.Sprintf("An argument named %q is not expected here.%s", name, suggestion),
|
||||
Subject: &attr.NameRange,
|
||||
})
|
||||
}
|
||||
|
||||
blocksWanted := make(map[string]hcl.BlockHeaderSchema)
|
||||
for _, blockS := range schema.Blocks {
|
||||
blocksWanted[blockS.Type] = blockS
|
||||
}
|
||||
|
||||
var blocks []*hcl.Block
|
||||
for _, block := range b.ChildBlocks {
|
||||
blockTy := block.Type
|
||||
blockS, wanted := blocksWanted[blockTy]
|
||||
if !wanted {
|
||||
if remain != nil {
|
||||
remain.appendBlock(block)
|
||||
continue
|
||||
}
|
||||
var suggestions []string
|
||||
for _, blockS := range schema.Blocks {
|
||||
suggestions = append(suggestions, blockS.Type)
|
||||
}
|
||||
suggestion := nameSuggestion(blockTy, suggestions)
|
||||
if suggestion != "" {
|
||||
suggestion = fmt.Sprintf(" Did you mean %q?", suggestion)
|
||||
} else {
|
||||
// Is there an attribute of the same name?
|
||||
for _, attrS := range schema.Attributes {
|
||||
if attrS.Name == blockTy {
|
||||
suggestion = fmt.Sprintf(" Did you mean to define argument %q?", blockTy)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
diags = append(diags, &hcl.Diagnostic{
|
||||
Severity: hcl.DiagError,
|
||||
Summary: "Unsupported block type",
|
||||
Detail: fmt.Sprintf("Blocks of type %q are not expected here.%s", blockTy, suggestion),
|
||||
Subject: &block.TypeRange,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
if len(block.Labels) != len(blockS.LabelNames) {
|
||||
if len(blockS.LabelNames) == 0 {
|
||||
diags = append(diags, &hcl.Diagnostic{
|
||||
Severity: hcl.DiagError,
|
||||
Summary: fmt.Sprintf("Extraneous label for %s", blockTy),
|
||||
Detail: fmt.Sprintf(
|
||||
"No labels are expected for %s blocks.", blockTy,
|
||||
),
|
||||
Subject: &block.DefRange,
|
||||
Context: &block.DefRange,
|
||||
})
|
||||
} else {
|
||||
diags = append(diags, &hcl.Diagnostic{
|
||||
Severity: hcl.DiagError,
|
||||
Summary: fmt.Sprintf("Wrong label count for %s", blockTy),
|
||||
Detail: fmt.Sprintf(
|
||||
"%s blocks expect %d label(s), but got %d.",
|
||||
blockTy, len(blockS.LabelNames), len(block.Labels),
|
||||
),
|
||||
Subject: &block.DefRange,
|
||||
Context: &block.DefRange,
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
blocks = append(blocks, block.asHCLBlock())
|
||||
}
|
||||
|
||||
return &hcl.BodyContent{
|
||||
Attributes: attrs,
|
||||
Blocks: blocks,
|
||||
MissingItemRange: b.MissingItemRange_,
|
||||
}, diags
|
||||
}
|
||||
|
||||
// JustAttributes is an implementation of the method of the same name on hcl.Body.
|
||||
func (b *Body) JustAttributes() (hcl.Attributes, hcl.Diagnostics) {
|
||||
var diags hcl.Diagnostics
|
||||
if len(b.ChildBlocks) > 0 {
|
||||
for _, block := range b.ChildBlocks {
|
||||
diags = append(diags, &hcl.Diagnostic{
|
||||
Severity: hcl.DiagError,
|
||||
Summary: fmt.Sprintf("Unexpected %s block", block.Type),
|
||||
Detail: "Blocks are not allowed here.",
|
||||
Context: &block.TypeRange,
|
||||
})
|
||||
}
|
||||
// We'll continue processing anyway, and return any attributes we find
|
||||
// so that the caller can do careful partial analysis.
|
||||
}
|
||||
|
||||
if len(b.Attributes) == 0 {
|
||||
return nil, diags
|
||||
}
|
||||
|
||||
ret := make(hcl.Attributes, len(b.Attributes))
|
||||
for n, a := range b.Attributes {
|
||||
ret[n] = a.asHCLAttribute(n)
|
||||
}
|
||||
return ret, diags
|
||||
}
|
||||
|
||||
// MissingItemRange is an implementation of the method of the same name on hcl.Body.
|
||||
func (b *Body) MissingItemRange() hcl.Range {
|
||||
return b.MissingItemRange_
|
||||
}
|
||||
|
||||
func (b *Body) setAttribute(name string, attr Attribute) {
|
||||
if b.Attributes == nil {
|
||||
b.Attributes = make(map[string]Attribute)
|
||||
}
|
||||
b.Attributes[name] = attr
|
||||
}
|
||||
|
||||
func (b *Body) appendBlock(block Block) {
|
||||
b.ChildBlocks = append(b.ChildBlocks, block)
|
||||
}
|
||||
|
||||
// Block represents a nested block within a body.
|
||||
type Block struct {
|
||||
Type string
|
||||
Labels []string
|
||||
Body Body
|
||||
|
||||
DefRange, TypeRange hcl.Range
|
||||
LabelRanges []hcl.Range
|
||||
}
|
||||
|
||||
func (b *Block) asHCLBlock() *hcl.Block {
|
||||
return &hcl.Block{
|
||||
Type: b.Type,
|
||||
Labels: b.Labels,
|
||||
Body: &b.Body,
|
||||
|
||||
TypeRange: b.TypeRange,
|
||||
DefRange: b.DefRange,
|
||||
LabelRanges: b.LabelRanges,
|
||||
}
|
||||
}
|
||||
|
||||
// Attribute represents an attribute definition within a body.
|
||||
type Attribute struct {
|
||||
Expr Expression
|
||||
|
||||
Range, NameRange hcl.Range
|
||||
}
|
||||
|
||||
func (a *Attribute) asHCLAttribute(name string) *hcl.Attribute {
|
||||
return &hcl.Attribute{
|
||||
Name: name,
|
||||
Expr: &a.Expr,
|
||||
Range: a.Range,
|
||||
NameRange: a.NameRange,
|
||||
}
|
||||
}
|
||||
93
hclpack/structure_test.go
Normal file
93
hclpack/structure_test.go
Normal file
@ -0,0 +1,93 @@
|
||||
package hclpack
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
"github.com/hashicorp/hcl2/hcl"
|
||||
)
|
||||
|
||||
func TestBodyContent(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
Body *Body
|
||||
Schema *hcl.BodySchema
|
||||
Want *hcl.BodyContent
|
||||
}{
|
||||
"empty": {
|
||||
&Body{},
|
||||
&hcl.BodySchema{},
|
||||
&hcl.BodyContent{},
|
||||
},
|
||||
"nil": {
|
||||
nil,
|
||||
&hcl.BodySchema{},
|
||||
&hcl.BodyContent{},
|
||||
},
|
||||
"attribute": {
|
||||
&Body{
|
||||
Attributes: map[string]Attribute{
|
||||
"foo": {
|
||||
Expr: Expression{
|
||||
Source: []byte(`"hello"`),
|
||||
SourceType: ExprNative,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
&hcl.BodySchema{
|
||||
Attributes: []hcl.AttributeSchema{
|
||||
{Name: "foo", Required: true},
|
||||
{Name: "bar", Required: false},
|
||||
},
|
||||
},
|
||||
&hcl.BodyContent{
|
||||
Attributes: hcl.Attributes{
|
||||
"foo": {
|
||||
Name: "foo",
|
||||
Expr: &Expression{
|
||||
Source: []byte(`"hello"`),
|
||||
SourceType: ExprNative,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"block": {
|
||||
&Body{
|
||||
ChildBlocks: []Block{
|
||||
{
|
||||
Type: "foo",
|
||||
},
|
||||
},
|
||||
},
|
||||
&hcl.BodySchema{
|
||||
Blocks: []hcl.BlockHeaderSchema{
|
||||
{Type: "foo"},
|
||||
},
|
||||
},
|
||||
&hcl.BodyContent{
|
||||
Blocks: hcl.Blocks{
|
||||
{
|
||||
Type: "foo",
|
||||
Body: &Body{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for name, test := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, diags := test.Body.Content(test.Schema)
|
||||
for _, diag := range diags {
|
||||
t.Errorf("unexpected diagnostic: %s", diag.Error())
|
||||
}
|
||||
|
||||
if !cmp.Equal(test.Want, got) {
|
||||
t.Errorf("wrong result\n%s", cmp.Diff(test.Want, got))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user