From 77c0b55a597ce9ff855c699dba2a99c1632690e1 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 1 Aug 2018 08:45:22 -0700 Subject: [PATCH] hclwrite: Simplify internal data structures The original prototype of hclwrite tried to track both the tokens and the AST as two parallel data structures. This quickly exploded in complexity, leading to lots of messy code to manage keeping those two structures in sync. This new approach melds the two structures together, creating first a physical token tree (made of "node" objects, and hidden from the caller) and then attaching the AST nodes to that token tree as additional sidecar data. The result is much easier to work with, leading to less code in the parser and considerably less complex data structures in the parser's tests. This commit is enough to reach feature parity with the previous prototype, but it remains a prototype. With a more usable foundation, we'll evolve this into a more complete implementation in subsequent commits. --- hclwrite/ast.go | 211 ++-- hclwrite/ast_body.go | 85 ++ hclwrite/ast_body_test.go | 215 ++++ hclwrite/ast_expression.go | 68 ++ hclwrite/ast_test.go | 128 +-- hclwrite/node.go | 200 ++++ hclwrite/parser.go | 325 ++++--- hclwrite/parser_test.go | 1838 ++++++----------------------------- hclwrite/public.go | 16 +- hclwrite/round_trip_test.go | 11 +- hclwrite/tokens.go | 99 +- 11 files changed, 1182 insertions(+), 2014 deletions(-) create mode 100644 hclwrite/ast_body.go create mode 100644 hclwrite/ast_body_test.go create mode 100644 hclwrite/ast_expression.go create mode 100644 hclwrite/node.go diff --git a/hclwrite/ast.go b/hclwrite/ast.go index 38ec090..8bf7331 100644 --- a/hclwrite/ast.go +++ b/hclwrite/ast.go @@ -3,29 +3,25 @@ package hclwrite import ( "bytes" "io" - - "github.com/hashicorp/hcl2/hcl" - "github.com/zclconf/go-cty/cty" ) -type Node interface { - walkChildNodes(w internalWalkFunc) - Tokens() *TokenSeq +type File struct { + inTree + + srcBytes []byte + body *node } -type internalWalkFunc func(Node) - -type File struct { - Name string - SrcBytes []byte - - Body *Body - AllTokens *TokenSeq +// Body returns the root body of the file, which contains the top-level +// attributes and blocks. +func (f *File) Body() *Body { + return f.body.content.(*Body) } // WriteTo writes the tokens underlying the receiving file to the given writer. func (f *File) WriteTo(wr io.Writer) (int, error) { - return f.AllTokens.WriteTo(wr) + tokens := f.inTree.children.BuildTokens(nil) + return tokens.WriteTo(wr) } // Bytes returns a buffer containing the source code resulting from the @@ -37,169 +33,74 @@ func (f *File) Bytes() []byte { return buf.Bytes() } -// Format makes in-place modifications to the tokens underlying the receiving -// file in order to change the whitespace to be in canonical form. -func (f *File) Format() { - format(f.Body.AllTokens.Tokens()) +type comments struct { + leafNode + + parent *node + tokens Tokens } -type Body struct { - // Items may contain Attribute, Block and Unstructured instances. - // Items and AllTokens should be updated only by methods of this type, - // since they must be kept synchronized for correct operation. - Items []Node - AllTokens *TokenSeq - - // IndentLevel is the number of spaces that should appear at the start - // of lines added within this body. - IndentLevel int -} - -func (n *Body) walkChildNodes(w internalWalkFunc) { - for _, item := range n.Items { - w(item) +func newComments(tokens Tokens) *comments { + return &comments{ + tokens: tokens, } } -func (n *Body) Tokens() *TokenSeq { - return n.AllTokens +func (c *comments) BuildTokens(to Tokens) Tokens { + return c.tokens.BuildTokens(to) } -func (n *Body) AppendItem(node Node) { - n.Items = append(n.Items, node) - n.AppendUnstructuredTokens(node.Tokens()) +type identifier struct { + leafNode + + parent *node + token *Token } -func (n *Body) AppendUnstructuredTokens(seq *TokenSeq) { - if n.AllTokens == nil { - new := make(TokenSeq, 0, 1) - n.AllTokens = &new - } - *(n.AllTokens) = append(*(n.AllTokens), seq) -} - -// FindAttribute returns the first attribute item from the body that has the -// given name, or returns nil if there is currently no matching attribute. -// -// A valid AST has only one definition of each attribute, but that constraint -// is not enforced in the hclwrite AST, so a tree that has been mutated by -// other calls may contain additional matching attributes that cannot be seen -// by this method. -func (n *Body) FindAttribute(name string) *Attribute { - nameBytes := []byte(name) - for _, item := range n.Items { - if attr, ok := item.(*Attribute); ok { - if attr.NameTokens.IsIdent(nameBytes) { - return attr - } - } - } - return nil -} - -// SetAttributeValue either replaces the expression of an existing attribute -// of the given name or adds a new attribute definition to the end of the block. -// -// The value is given as a cty.Value, and must therefore be a literal. To set -// a variable reference or other traversal, use SetAttributeTraversal. -// -// The return value is the attribute that was either modified in-place or -// created. -func (n *Body) SetAttributeValue(name string, val cty.Value) *Attribute { - panic("Body.SetAttributeValue not yet implemented") -} - -// SetAttributeTraversal either replaces the expression of an existing attribute -// of the given name or adds a new attribute definition to the end of the block. -// -// The new expression is given as a hcl.Traversal, which must be an absolute -// traversal. To set a literal value, use SetAttributeValue. -// -// The return value is the attribute that was either modified in-place or -// created. -func (n *Body) SetAttributeTraversal(name string, traversal hcl.Traversal) *Attribute { - panic("Body.SetAttributeTraversal not yet implemented") -} - -type Attribute struct { - AllTokens *TokenSeq - - LeadCommentTokens *TokenSeq - NameTokens *TokenSeq - EqualsTokens *TokenSeq - Expr *Expression - LineCommentTokens *TokenSeq - EOLTokens *TokenSeq -} - -func (a *Attribute) walkChildNodes(w internalWalkFunc) { - w(a.Expr) -} - -func (n *Attribute) Tokens() *TokenSeq { - return n.AllTokens -} - -type Block struct { - AllTokens *TokenSeq - - LeadCommentTokens *TokenSeq - TypeTokens *TokenSeq - LabelTokens []*TokenSeq - LabelTokensFlat *TokenSeq - OBraceTokens *TokenSeq - Body *Body - CBraceTokens *TokenSeq - EOLTokens *TokenSeq -} - -func (n *Block) walkChildNodes(w internalWalkFunc) { - w(n.Body) -} - -func (n *Block) Tokens() *TokenSeq { - return n.AllTokens -} - -type Expression struct { - AllTokens *TokenSeq - AbsTraversals []*Traversal -} - -func (n *Expression) walkChildNodes(w internalWalkFunc) { - for _, name := range n.AbsTraversals { - w(name) +func newIdentifier(token *Token) *identifier { + return &identifier{ + token: token, } } -func (n *Expression) Tokens() *TokenSeq { - return n.AllTokens +func (i *identifier) BuildTokens(to Tokens) Tokens { + return append(to, i.token) } -type Traversal struct { - AllTokens *TokenSeq - Steps []*Traverser +func (i *identifier) hasName(name string) bool { + return name == string(i.token.Bytes) } -func (n *Traversal) walkChildNodes(w internalWalkFunc) { - for _, step := range n.Steps { - w(step) +type number struct { + leafNode + + parent *node + token *Token +} + +func newNumber(token *Token) *number { + return &number{ + token: token, } } -func (n *Traversal) Tokens() *TokenSeq { - return n.AllTokens +func (n *number) BuildTokens(to Tokens) Tokens { + return append(to, n.token) } -type Traverser struct { - AllTokens *TokenSeq - Logical hcl.Traverser +type quoted struct { + leafNode + + parent *node + tokens Tokens } -func (n *Traverser) Tokens() *TokenSeq { - return n.AllTokens +func newQuoted(tokens Tokens) *quoted { + return "ed{ + tokens: tokens, + } } -func (n *Traverser) walkChildNodes(w internalWalkFunc) { - // No child nodes for a traversal step +func (q *quoted) BuildTokens(to Tokens) Tokens { + return q.tokens.BuildTokens(to) } diff --git a/hclwrite/ast_body.go b/hclwrite/ast_body.go new file mode 100644 index 0000000..1ca239b --- /dev/null +++ b/hclwrite/ast_body.go @@ -0,0 +1,85 @@ +package hclwrite + +import ( + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" +) + +type Body struct { + inTree + + items nodeSet + + // indentLevel is the number of spaces that should appear at the start + // of lines added within this body. + indentLevel int +} + +func (b *Body) appendItem(n *node) { + b.inTree.children.AppendNode(n) + b.items.Add(n) +} + +func (b *Body) AppendUnstructuredTokens(ts Tokens) { + b.inTree.children.Append(ts) +} + +// GetAttribute returns the attribute from the body that has the given name, +// or returns nil if there is currently no matching attribute. +func (b *Body) GetAttribute(name string) *Attribute { + for n := range b.items { + if attr, isAttr := n.content.(*Attribute); isAttr { + nameObj := attr.name.content.(*identifier) + if nameObj.hasName(name) { + // We've found it! + return attr + } + } + } + + return nil +} + +// SetAttributeValue either replaces the expression of an existing attribute +// of the given name or adds a new attribute definition to the end of the block. +// +// The value is given as a cty.Value, and must therefore be a literal. To set +// a variable reference or other traversal, use SetAttributeTraversal. +// +// The return value is the attribute that was either modified in-place or +// created. +func (b *Body) SetAttributeValue(name string, val cty.Value) *Attribute { + panic("Body.SetAttributeValue not yet implemented") +} + +// SetAttributeTraversal either replaces the expression of an existing attribute +// of the given name or adds a new attribute definition to the end of the block. +// +// The new expression is given as a hcl.Traversal, which must be an absolute +// traversal. To set a literal value, use SetAttributeValue. +// +// The return value is the attribute that was either modified in-place or +// created. +func (b *Body) SetAttributeTraversal(name string, traversal hcl.Traversal) *Attribute { + panic("Body.SetAttributeTraversal not yet implemented") +} + +type Attribute struct { + inTree + + leadComments *node + name *node + expr *node + lineComments *node +} + +type Block struct { + inTree + + leadComments *node + typeName *node + labels nodeSet + open *node + body *node + close *node +} diff --git a/hclwrite/ast_body_test.go b/hclwrite/ast_body_test.go new file mode 100644 index 0000000..f2da4b0 --- /dev/null +++ b/hclwrite/ast_body_test.go @@ -0,0 +1,215 @@ +package hclwrite + +import ( + "fmt" + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" +) + +func TestBodyGetAttribute(t *testing.T) { + tests := []struct { + src string + name string + want Tokens + }{ + { + "", + "a", + nil, + }, + { + "a = 1\n", + "a", + Tokens{ + { + Type: hclsyntax.TokenIdent, + Bytes: []byte{'a'}, + SpacesBefore: 0, + }, + { + Type: hclsyntax.TokenEqual, + Bytes: []byte{'='}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNumberLit, + Bytes: []byte{'1'}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNewline, + Bytes: []byte{'\n'}, + SpacesBefore: 0, + }, + }, + }, + { + "a = 1\nb = 1\nc = 1\n", + "a", + Tokens{ + { + Type: hclsyntax.TokenIdent, + Bytes: []byte{'a'}, + SpacesBefore: 0, + }, + { + Type: hclsyntax.TokenEqual, + Bytes: []byte{'='}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNumberLit, + Bytes: []byte{'1'}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNewline, + Bytes: []byte{'\n'}, + SpacesBefore: 0, + }, + }, + }, + { + "a = 1\nb = 2\nc = 3\n", + "b", + Tokens{ + { + Type: hclsyntax.TokenIdent, + Bytes: []byte{'b'}, + SpacesBefore: 0, + }, + { + Type: hclsyntax.TokenEqual, + Bytes: []byte{'='}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNumberLit, + Bytes: []byte{'2'}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNewline, + Bytes: []byte{'\n'}, + SpacesBefore: 0, + }, + }, + }, + { + "a = 1\nb = 2\nc = 3\n", + "c", + Tokens{ + { + Type: hclsyntax.TokenIdent, + Bytes: []byte{'c'}, + SpacesBefore: 0, + }, + { + Type: hclsyntax.TokenEqual, + Bytes: []byte{'='}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNumberLit, + Bytes: []byte{'3'}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNewline, + Bytes: []byte{'\n'}, + SpacesBefore: 0, + }, + }, + }, + { + "a = 1\n# b is a b\nb = 2\nc = 3\n", + "b", + Tokens{ + { + // Recognized as a lead comment and so attached to the attribute + Type: hclsyntax.TokenComment, + Bytes: []byte("# b is a b\n"), + SpacesBefore: 0, + }, + { + Type: hclsyntax.TokenIdent, + Bytes: []byte{'b'}, + SpacesBefore: 0, + }, + { + Type: hclsyntax.TokenEqual, + Bytes: []byte{'='}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNumberLit, + Bytes: []byte{'2'}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNewline, + Bytes: []byte{'\n'}, + SpacesBefore: 0, + }, + }, + }, + { + "a = 1\n# not attached to a or b\n\nb = 2\nc = 3\n", + "b", + Tokens{ + { + Type: hclsyntax.TokenIdent, + Bytes: []byte{'b'}, + SpacesBefore: 0, + }, + { + Type: hclsyntax.TokenEqual, + Bytes: []byte{'='}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNumberLit, + Bytes: []byte{'2'}, + SpacesBefore: 1, + }, + { + Type: hclsyntax.TokenNewline, + Bytes: []byte{'\n'}, + SpacesBefore: 0, + }, + }, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%s in %s", test.name, test.src), func(t *testing.T) { + f, diags := ParseConfig([]byte(test.src), "", hcl.Pos{Line: 1, Column: 1}) + if len(diags) != 0 { + for _, diag := range diags { + t.Logf("- %s", diag.Error()) + } + t.Fatalf("unexpected diagnostics") + } + + attr := f.Body().GetAttribute(test.name) + if attr == nil { + if test.want != nil { + t.Fatal("attribute not found, but want it to exist") + } + } else { + if test.want == nil { + t.Fatal("attribute found, but expecting not found") + } + + got := attr.BuildTokens(nil) + if !reflect.DeepEqual(got, test.want) { + t.Errorf("wrong result\ngot: %s\nwant: %s", spew.Sdump(got), spew.Sdump(test.want)) + } + } + }) + } +} diff --git a/hclwrite/ast_expression.go b/hclwrite/ast_expression.go new file mode 100644 index 0000000..0abca17 --- /dev/null +++ b/hclwrite/ast_expression.go @@ -0,0 +1,68 @@ +package hclwrite + +import ( + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" +) + +type Expression struct { + inTree + + absTraversals nodeSet +} + +func newExpression() *Expression { + return &Expression{ + inTree: newInTree(), + absTraversals: newNodeSet(), + } +} + +// NewExpressionLiteral constructs an an expression that represents the given +// literal value. +func NewExpressionLiteral(val cty.Value) *Expression { + panic("NewExpressionLiteral not yet implemented") +} + +// NewExpressionAbsTraversal constructs an expression that represents the +// given traversal, which must be absolute or this function will panic. +func NewExpressionAbsTraversal(traversal hcl.Traversal) { + panic("NewExpressionAbsTraversal not yet implemented") +} + +type Traversal struct { + inTree + + steps nodeSet +} + +func newTraversal() *Traversal { + return &Traversal{ + inTree: newInTree(), + steps: newNodeSet(), + } +} + +type TraverseName struct { + inTree + + name *node +} + +func newTraverseName() *TraverseName { + return &TraverseName{ + inTree: newInTree(), + } +} + +type TraverseIndex struct { + inTree + + key *node +} + +func newTraverseIndex() *TraverseIndex { + return &TraverseIndex{ + inTree: newInTree(), + } +} diff --git a/hclwrite/ast_test.go b/hclwrite/ast_test.go index 0f95334..08f7801 100644 --- a/hclwrite/ast_test.go +++ b/hclwrite/ast_test.go @@ -2,96 +2,46 @@ package hclwrite import ( "fmt" - "reflect" - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/hashicorp/hcl2/hcl/hclsyntax" - "github.com/hashicorp/hcl2/hcl" + "strings" ) -func TestBodyFindAttribute(t *testing.T) { - tests := []struct { - src string - name string - want *TokenSeq - }{ - { - "", - "a", - nil, - }, - { - "a = 1\n", - "a", - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte{'a'}, - }, - }, - }, - }, - { - "a = 1\nb = 1\nc = 1\n", - "a", - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte{'a'}, - }, - }, - }, - }, - { - "a = 1\nb = 1\nc = 1\n", - "b", - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte{'b'}, - }, - }, - }, - }, - { - "a = 1\nb = 1\nc = 1\n", - "c", - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte{'c'}, - }, - }, - }, - }, - } +type TestTreeNode struct { + Type string + Val string - for _, test := range tests { - t.Run(fmt.Sprintf("%s in %s", test.name, test.src), func(t *testing.T) { - f, diags := ParseConfig([]byte(test.src), "", hcl.Pos{Line: 1, Column: 1}) - if len(diags) != 0 { - for _, diag := range diags { - t.Logf("- %s", diag.Error()) - } - t.Fatalf("unexpected diagnostics") - } - - attr := f.Body.FindAttribute(test.name) - if attr == nil { - if test.want != nil { - t.Errorf("attribute found, but expecting not found") - } - } else { - got := attr.NameTokens - if !reflect.DeepEqual(got, test.want) { - t.Errorf("wrong result\ngot: %s\nwant: %s", spew.Sdump(got), spew.Sdump(test.want)) - } - } - }) - } + Children []TestTreeNode +} + +func makeTestTree(n *node) (root TestTreeNode) { + const us = "hclwrite." + const usPtr = "*hclwrite." + root.Type = fmt.Sprintf("%T", n.content) + if strings.HasPrefix(root.Type, us) { + root.Type = root.Type[len(us):] + } else if strings.HasPrefix(root.Type, usPtr) { + root.Type = root.Type[len(usPtr):] + } + + type WithVal interface { + testValue() string + } + hasTestVal := false + if withVal, ok := n.content.(WithVal); ok { + root.Val = withVal.testValue() + hasTestVal = true + } + + n.content.walkChildNodes(func(n *node) { + root.Children = append(root.Children, makeTestTree(n)) + }) + + // If we didn't end up with any children then this is probably a leaf + // node, so we'll set its content value to it raw bytes if we didn't + // already set a test value. + if !hasTestVal && len(root.Children) == 0 { + toks := n.content.BuildTokens(nil) + root.Val = toks.testValue() + } + + return root } diff --git a/hclwrite/node.go b/hclwrite/node.go new file mode 100644 index 0000000..5623847 --- /dev/null +++ b/hclwrite/node.go @@ -0,0 +1,200 @@ +package hclwrite + +import ( + "fmt" + + "github.com/google/go-cmp/cmp" +) + +// node represents a node in the AST. +type node struct { + content nodeContent + + list *nodes + before, after *node +} + +func newNode(c nodeContent) *node { + return &node{ + content: c, + } +} + +func (n *node) Equal(other *node) bool { + return cmp.Equal(n.content, other.content) +} + +func (n *node) BuildTokens(to Tokens) Tokens { + return n.content.BuildTokens(to) +} + +// Detach removes the receiver from the list it currently belongs to. If the +// node is not currently in a list, this is a no-op. +func (n *node) Detach() { + if n.list == nil { + return + } + if n.before != nil { + n.before.after = n.after + } + if n.after != nil { + n.after.before = n.before + } + if n.list.first == n { + n.list.first = n.after + } + if n.list.last == n { + n.list.last = n.before + } + n.list = nil + n.before = nil + n.after = nil +} + +func (n *node) assertUnattached() { + if n.list != nil { + panic(fmt.Sprintf("attempt to attach already-attached node %#v", n)) + } +} + +// nodeContent is the interface type implemented by all AST content types. +type nodeContent interface { + walkChildNodes(w internalWalkFunc) + BuildTokens(to Tokens) Tokens +} + +// nodes is a list of nodes. +type nodes struct { + first, last *node +} + +func (ns *nodes) BuildTokens(to Tokens) Tokens { + for n := ns.first; n != nil; n = n.after { + to = n.BuildTokens(to) + } + return to +} + +func (ns *nodes) Append(c nodeContent) *node { + n := &node{ + content: c, + } + ns.AppendNode(n) + return n +} + +func (ns *nodes) AppendNode(n *node) { + if ns.last != nil { + n.before = ns.last + ns.last.after = n + } + n.list = ns + ns.last = n + if ns.first == nil { + ns.first = n + } +} + +func (ns *nodes) AppendUnstructuredTokens(tokens Tokens) *node { + if len(tokens) == 0 { + return nil + } + n := newNode(tokens) + ns.AppendNode(n) + return n +} + +// nodeSet is an unordered set of nodes. It is used to describe a set of nodes +// that all belong to the same list that have some role or characteristic +// in common. +type nodeSet map[*node]struct{} + +func newNodeSet() nodeSet { + return make(nodeSet) +} + +func (ns nodeSet) Has(n *node) bool { + if ns == nil { + return false + } + _, exists := ns[n] + return exists +} + +func (ns nodeSet) Add(n *node) { + ns[n] = struct{}{} +} + +func (ns nodeSet) Remove(n *node) { + delete(ns, n) +} + +func (ns nodeSet) List() []*node { + if len(ns) == 0 { + return nil + } + + ret := make([]*node, 0, len(ns)) + + // Determine which list we are working with. We assume here that all of + // the nodes belong to the same list, since that is part of the contract + // for nodeSet. + var list *nodes + for n := range ns { + list = n.list + break + } + + // We recover the order by iterating over the whole list. This is not + // the most efficient way to do it, but our node lists should always be + // small so not worth making things more complex. + for n := list.first; n != nil; n = n.after { + if ns.Has(n) { + ret = append(ret, n) + } + } + return ret +} + +type internalWalkFunc func(*node) + +// inTree can be embedded into a content struct that has child nodes to get +// a standard implementation of the NodeContent interface and a record of +// a potential parent node. +type inTree struct { + parent *node + children *nodes +} + +func newInTree() inTree { + return inTree{ + children: &nodes{}, + } +} + +func (it *inTree) assertUnattached() { + if it.parent != nil { + panic(fmt.Sprintf("node is already attached to %T", it.parent.content)) + } +} + +func (it *inTree) walkChildNodes(w internalWalkFunc) { + for n := it.children.first; n != nil; n = n.after { + w(n) + } +} + +func (it *inTree) BuildTokens(to Tokens) Tokens { + for n := it.children.first; n != nil; n = n.after { + to = n.BuildTokens(to) + } + return to +} + +// leafNode can be embedded into a content struct to give it a do-nothing +// implementation of walkChildNodes +type leafNode struct { +} + +func (n *leafNode) walkChildNodes(w internalWalkFunc) { +} diff --git a/hclwrite/parser.go b/hclwrite/parser.go index 52891c8..eca3fb5 100644 --- a/hclwrite/parser.go +++ b/hclwrite/parser.go @@ -1,10 +1,12 @@ package hclwrite import ( + "fmt" "sort" "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcl/hclsyntax" + "github.com/zclconf/go-cty/cty" ) // Our "parser" here is actually not doing any parsing of its own. Instead, @@ -49,18 +51,19 @@ func parse(src []byte, filename string, start hcl.Pos) (*File, hcl.Diagnostics) } before, root, after := parseBody(file.Body.(*hclsyntax.Body), from) + ret := &File{ + inTree: newInTree(), - return &File{ - Name: filename, - SrcBytes: src, + srcBytes: src, + body: root, + } - Body: root, - AllTokens: &TokenSeq{ - before.Seq(), - root.AllTokens, - after.Seq(), - }, - }, nil + nodes := ret.inTree.children + nodes.Append(before.Tokens()) + nodes.AppendNode(root) + nodes.Append(after.Tokens()) + + return ret, diags } type inputTokens struct { @@ -76,6 +79,23 @@ func (it inputTokens) Partition(rng hcl.Range) (before, within, after inputToken return } +func (it inputTokens) PartitionType(ty hclsyntax.TokenType) (before, within, after inputTokens) { + for i, t := range it.writerTokens { + if t.Type == ty { + return it.Slice(0, i), it.Slice(i, i+1), it.Slice(i+1, len(it.nativeTokens)) + } + } + panic(fmt.Sprintf("didn't find any token of type %s", ty)) +} + +func (it inputTokens) PartitionTypeSingle(ty hclsyntax.TokenType) (before inputTokens, found *Token, after inputTokens) { + before, within, after := it.PartitionType(ty) + if within.Len() != 1 { + panic("PartitionType found more than one token") + } + return before, within.Tokens()[0], after +} + // PartitionIncludeComments is like Partition except the returned "within" // range includes any lead and line comments associated with the range. func (it inputTokens) PartitionIncludingComments(rng hcl.Range) (before, within, after inputTokens) { @@ -133,8 +153,8 @@ func (it inputTokens) Len() int { return len(it.nativeTokens) } -func (it inputTokens) Seq() *TokenSeq { - return &TokenSeq{it.writerTokens} +func (it inputTokens) Tokens() Tokens { + return it.writerTokens } func (it inputTokens) Types() []hclsyntax.TokenType { @@ -148,7 +168,7 @@ func (it inputTokens) Types() []hclsyntax.TokenType { // parseBody locates the given body within the given input tokens and returns // the resulting *Body object as well as the tokens that appeared before and // after it. -func parseBody(nativeBody *hclsyntax.Body, from inputTokens) (inputTokens, *Body, inputTokens) { +func parseBody(nativeBody *hclsyntax.Body, from inputTokens) (inputTokens, *node, inputTokens) { before, within, after := from.PartitionIncludingComments(nativeBody.SrcRange) // The main AST doesn't retain the original source ordering of the @@ -164,7 +184,10 @@ func parseBody(nativeBody *hclsyntax.Body, from inputTokens) (inputTokens, *Body sort.Sort(nativeNodeSorter{nativeItems}) body := &Body{ - IndentLevel: 0, // TODO: deal with this + inTree: newInTree(), + + indentLevel: 0, // TODO: deal with this + items: newNodeSet(), } remain := within @@ -172,24 +195,24 @@ func parseBody(nativeBody *hclsyntax.Body, from inputTokens) (inputTokens, *Body beforeItem, item, afterItem := parseBodyItem(nativeItem, remain) if beforeItem.Len() > 0 { - body.AppendUnstructuredTokens(beforeItem.Seq()) + body.AppendUnstructuredTokens(beforeItem.Tokens()) } - body.AppendItem(item) + body.appendItem(item) remain = afterItem } if remain.Len() > 0 { - body.AppendUnstructuredTokens(remain.Seq()) + body.AppendUnstructuredTokens(remain.Tokens()) } - return before, body, after + return before, newNode(body), after } -func parseBodyItem(nativeItem hclsyntax.Node, from inputTokens) (inputTokens, Node, inputTokens) { +func parseBodyItem(nativeItem hclsyntax.Node, from inputTokens) (inputTokens, *node, inputTokens) { before, leadComments, within, lineComments, newline, after := from.PartitionBlockItem(nativeItem.Range()) - var item Node + var item *node switch tItem := nativeItem.(type) { case *hclsyntax.Attribute: @@ -204,90 +227,96 @@ func parseBodyItem(nativeItem hclsyntax.Node, from inputTokens) (inputTokens, No return before, item, after } -func parseAttribute(nativeAttr *hclsyntax.Attribute, from, leadComments, lineComments, newline inputTokens) *Attribute { - var allTokens TokenSeq - attr := &Attribute{} +func parseAttribute(nativeAttr *hclsyntax.Attribute, from, leadComments, lineComments, newline inputTokens) *node { + attr := &Attribute{ + inTree: newInTree(), + } + children := attr.inTree.children - if leadComments.Len() > 0 { - attr.LeadCommentTokens = leadComments.Seq() - allTokens = append(allTokens, attr.LeadCommentTokens) + { + cn := newNode(newComments(leadComments.Tokens())) + attr.leadComments = cn + children.AppendNode(cn) } before, nameTokens, from := from.Partition(nativeAttr.NameRange) - if before.Len() > 0 { - allTokens = append(allTokens, before.Seq()) + { + children.AppendUnstructuredTokens(before.Tokens()) + if nameTokens.Len() != 1 { + // Should never happen with valid input + panic("attribute name is not exactly one token") + } + token := nameTokens.Tokens()[0] + in := newNode(newIdentifier(token)) + attr.name = in + children.AppendNode(in) } - attr.NameTokens = nameTokens.Seq() - allTokens = append(allTokens, attr.NameTokens) before, equalsTokens, from := from.Partition(nativeAttr.EqualsRange) - if before.Len() > 0 { - allTokens = append(allTokens, before.Seq()) - } - attr.EqualsTokens = equalsTokens.Seq() - allTokens = append(allTokens, attr.EqualsTokens) + children.AppendUnstructuredTokens(before.Tokens()) + children.AppendUnstructuredTokens(equalsTokens.Tokens()) before, exprTokens, from := from.Partition(nativeAttr.Expr.Range()) - if before.Len() > 0 { - allTokens = append(allTokens, before.Seq()) - } - attr.Expr = parseExpression(nativeAttr.Expr, exprTokens) - allTokens = append(allTokens, attr.Expr.AllTokens) - - if lineComments.Len() > 0 { - attr.LineCommentTokens = lineComments.Seq() - allTokens = append(allTokens, attr.LineCommentTokens) + { + children.AppendUnstructuredTokens(before.Tokens()) + exprNode := parseExpression(nativeAttr.Expr, exprTokens) + attr.expr = exprNode + children.AppendNode(exprNode) } - if newline.Len() > 0 { - attr.EOLTokens = newline.Seq() - allTokens = append(allTokens, attr.EOLTokens) + { + cn := newNode(newComments(lineComments.Tokens())) + attr.lineComments = cn + children.AppendNode(cn) } + children.AppendUnstructuredTokens(newline.Tokens()) + // Collect any stragglers, though there shouldn't be any - if from.Len() > 0 { - allTokens = append(allTokens, from.Seq()) - } + children.AppendUnstructuredTokens(from.Tokens()) - attr.AllTokens = &allTokens - - return attr + return newNode(attr) } -func parseBlock(nativeBlock *hclsyntax.Block, from, leadComments, lineComments, newline inputTokens) *Block { - var allTokens TokenSeq - block := &Block{} +func parseBlock(nativeBlock *hclsyntax.Block, from, leadComments, lineComments, newline inputTokens) *node { + block := &Block{ + inTree: newInTree(), + labels: newNodeSet(), + } + children := block.inTree.children - if leadComments.Len() > 0 { - block.LeadCommentTokens = leadComments.Seq() - allTokens = append(allTokens, block.LeadCommentTokens) + { + cn := newNode(newComments(leadComments.Tokens())) + block.leadComments = cn + children.AppendNode(cn) } before, typeTokens, from := from.Partition(nativeBlock.TypeRange) - if before.Len() > 0 { - allTokens = append(allTokens, before.Seq()) + { + children.AppendUnstructuredTokens(before.Tokens()) + if typeTokens.Len() != 1 { + // Should never happen with valid input + panic("block type name is not exactly one token") + } + token := typeTokens.Tokens()[0] + in := newNode(newIdentifier(token)) + block.typeName = in + children.AppendNode(in) } - block.TypeTokens = typeTokens.Seq() - allTokens = append(allTokens, block.TypeTokens) for _, rng := range nativeBlock.LabelRanges { var labelTokens inputTokens before, labelTokens, from = from.Partition(rng) - if before.Len() > 0 { - allTokens = append(allTokens, before.Seq()) - } - seq := labelTokens.Seq() - block.LabelTokens = append(block.LabelTokens, seq) - *(block.LabelTokensFlat) = append(*(block.LabelTokensFlat), seq) - allTokens = append(allTokens, seq) + children.AppendUnstructuredTokens(before.Tokens()) + tokens := labelTokens.Tokens() + ln := newNode(newQuoted(tokens)) + block.labels.Add(ln) + children.AppendNode(ln) } before, oBrace, from := from.Partition(nativeBlock.OpenBraceRange) - if before.Len() > 0 { - allTokens = append(allTokens, before.Seq()) - } - block.OBraceTokens = oBrace.Seq() - allTokens = append(allTokens, block.OBraceTokens) + children.AppendUnstructuredTokens(before.Tokens()) + children.AppendUnstructuredTokens(oBrace.Tokens()) // We go a bit out of order here: we go hunting for the closing brace // so that we have a delimited body, but then we'll deal with the body @@ -295,87 +324,109 @@ func parseBlock(nativeBlock *hclsyntax.Block, from, leadComments, lineComments, // that appear after it. bodyTokens, cBrace, from := from.Partition(nativeBlock.CloseBraceRange) before, body, after := parseBody(nativeBlock.Body, bodyTokens) + children.AppendUnstructuredTokens(before.Tokens()) + block.body = body + children.AppendNode(body) + children.AppendUnstructuredTokens(after.Tokens()) - if before.Len() > 0 { - allTokens = append(allTokens, before.Seq()) - } - block.Body = body - allTokens = append(allTokens, body.AllTokens) - if after.Len() > 0 { - allTokens = append(allTokens, after.Seq()) - } - - block.CBraceTokens = cBrace.Seq() - allTokens = append(allTokens, block.CBraceTokens) + children.AppendUnstructuredTokens(cBrace.Tokens()) // stragglers - if after.Len() > 0 { - allTokens = append(allTokens, from.Seq()) - } + children.AppendUnstructuredTokens(from.Tokens()) if lineComments.Len() > 0 { // blocks don't actually have line comments, so we'll just treat // them as extra stragglers - allTokens = append(allTokens, lineComments.Seq()) - } - if newline.Len() > 0 { - block.EOLTokens = newline.Seq() - allTokens = append(allTokens, block.EOLTokens) + children.AppendUnstructuredTokens(lineComments.Tokens()) } + children.AppendUnstructuredTokens(newline.Tokens()) - block.AllTokens = &allTokens - return block + return newNode(block) } -func parseExpression(nativeExpr hclsyntax.Expression, from inputTokens) *Expression { - var allTokens TokenSeq +func parseExpression(nativeExpr hclsyntax.Expression, from inputTokens) *node { + expr := newExpression() + children := expr.inTree.children + nativeVars := nativeExpr.Variables() - var absTraversals []*Traversal + for _, nativeTraversal := range nativeVars { - var traversalTokens TokenSeq - var before, traversalFrom inputTokens - before, traversalFrom, from = from.Partition(nativeTraversal.SourceRange()) - if before.Len() > 0 { - allTokens = append(allTokens, before.Seq()) - } - - var steps []*Traverser - - for _, nativeStep := range nativeTraversal { - var stepFrom inputTokens - before, stepFrom, traversalFrom = traversalFrom.Partition(nativeStep.SourceRange()) - stepTokens := stepFrom.Seq() - if before.Len() > 0 { - traversalTokens = append(traversalTokens, before.Seq()) - } - traversalTokens = append(traversalTokens, stepTokens) - step := &Traverser{ - AllTokens: stepTokens, - Logical: nativeStep, - } - steps = append(steps, step) - } - // Attach any straggler that don't belong to a step to the traversal itself. - if traversalFrom.Len() > 0 { - traversalTokens = append(traversalTokens, traversalFrom.Seq()) - } - allTokens = append(allTokens, &traversalTokens) - - absTraversals = append(absTraversals, &Traversal{ - AllTokens: &traversalTokens, - Steps: steps, - }) + before, traversal, after := parseTraversal(nativeTraversal, from) + children.AppendUnstructuredTokens(before.Tokens()) + children.AppendNode(traversal) + expr.absTraversals.Add(traversal) + from = after } // Attach any stragglers that don't belong to a traversal to the expression // itself. In an expression with no traversals at all, this is just the // entirety of "from". - if from.Len() > 0 { - allTokens = append(allTokens, from.Seq()) + children.AppendUnstructuredTokens(from.Tokens()) + + return newNode(expr) +} + +func parseTraversal(nativeTraversal hcl.Traversal, from inputTokens) (before inputTokens, n *node, after inputTokens) { + traversal := newTraversal() + children := traversal.inTree.children + before, from, after = from.Partition(nativeTraversal.SourceRange()) + + stepAfter := from + for _, nativeStep := range nativeTraversal { + before, step, after := parseTraversalStep(nativeStep, stepAfter) + children.AppendUnstructuredTokens(before.Tokens()) + children.AppendNode(step) + stepAfter = after } - return &Expression{ - AllTokens: &allTokens, - AbsTraversals: absTraversals, + return before, newNode(traversal), after +} + +func parseTraversalStep(nativeStep hcl.Traverser, from inputTokens) (before inputTokens, n *node, after inputTokens) { + var children *nodes + switch tNativeStep := nativeStep.(type) { + + case hcl.TraverseRoot, hcl.TraverseAttr: + step := newTraverseName() + children = step.inTree.children + before, from, after = from.Partition(nativeStep.SourceRange()) + inBefore, token, inAfter := from.PartitionTypeSingle(hclsyntax.TokenIdent) + name := newIdentifier(token) + children.AppendUnstructuredTokens(inBefore.Tokens()) + step.name = children.Append(name) + children.AppendUnstructuredTokens(inAfter.Tokens()) + return before, newNode(step), after + + case hcl.TraverseIndex: + step := newTraverseIndex() + children = step.inTree.children + before, from, after = from.Partition(nativeStep.SourceRange()) + + var inBefore, oBrack, keyTokens, cBrack inputTokens + inBefore, oBrack, from = from.PartitionType(hclsyntax.TokenOBrack) + children.AppendUnstructuredTokens(inBefore.Tokens()) + children.AppendUnstructuredTokens(oBrack.Tokens()) + keyTokens, cBrack, from = from.PartitionType(hclsyntax.TokenCBrack) + + keyVal := tNativeStep.Key + switch keyVal.Type() { + case cty.String: + key := newQuoted(keyTokens.Tokens()) + step.key = children.Append(key) + case cty.Number: + valBefore, valToken, valAfter := keyTokens.PartitionTypeSingle(hclsyntax.TokenNumberLit) + children.AppendUnstructuredTokens(valBefore.Tokens()) + key := newNumber(valToken) + step.key = children.Append(key) + children.AppendUnstructuredTokens(valAfter.Tokens()) + } + + children.AppendUnstructuredTokens(cBrack.Tokens()) + children.AppendUnstructuredTokens(from.Tokens()) + + return before, newNode(step), after + default: + panic(fmt.Sprintf("unsupported traversal step type %T", nativeStep)) } + } // writerTokens takes a sequence of tokens as produced by the main hclsyntax diff --git a/hclwrite/parser_test.go b/hclwrite/parser_test.go index 5a01672..e8f8fc9 100644 --- a/hclwrite/parser_test.go +++ b/hclwrite/parser_test.go @@ -2,148 +2,64 @@ package hclwrite import ( "fmt" - "math/big" "reflect" "testing" "github.com/davecgh/go-spew/spew" + + "github.com/google/go-cmp/cmp" + + "github.com/kylelemons/godebug/pretty" + "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcl/hclsyntax" - "github.com/kylelemons/godebug/pretty" - "github.com/zclconf/go-cty/cty" ) func TestParse(t *testing.T) { tests := []struct { src string - want *Body + want TestTreeNode }{ { "", - &Body{ - Items: nil, - AllTokens: nil, + TestTreeNode{ + Type: "Body", }, }, { "a = 1\n", - &Body{ - Items: []Node{ - &Attribute{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - NameTokens: &TokenSeq{Tokens{ + TestTreeNode{ + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Attribute", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, + Type: "comments", }, - }}, - EqualsTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, + Type: "identifier", + Val: "a", }, - }}, - Expr: &Expression{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - }, - EOLTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, + Type: "Tokens", + Val: " =", }, - }}, - }, - }, - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ + { + Type: "Expression", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, + Type: "Tokens", + Val: " 1", }, }, }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, + { + Type: "comments", + }, + { + Type: "Tokens", + Val: "\n", }, }, }, @@ -152,148 +68,39 @@ func TestParse(t *testing.T) { }, { "# aye aye aye\na = 1\n", - &Body{ - Items: []Node{ - &Attribute{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenComment, - Bytes: []byte("# aye aye aye\n"), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - LeadCommentTokens: &TokenSeq{Tokens{ + TestTreeNode{ + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Attribute", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenComment, - Bytes: []byte("# aye aye aye\n"), - SpacesBefore: 0, + Type: "comments", + Val: "# aye aye aye\n", }, - }}, - NameTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, + Type: "identifier", + Val: "a", }, - }}, - EqualsTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, + Type: "Tokens", + Val: " =", }, - }}, - Expr: &Expression{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - }, - EOLTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }}, - }, - }, - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenComment, - Bytes: []byte("# aye aye aye\n"), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ + Type: "Expression", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, + Type: "Tokens", + Val: " 1", }, }, }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, + { + Type: "comments", + }, + { + Type: "Tokens", + Val: "\n", }, }, }, @@ -302,123 +109,35 @@ func TestParse(t *testing.T) { }, { "a = 1 # because it is\n", - &Body{ - Items: []Node{ - &Attribute{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenComment, - Bytes: []byte("# because it is\n"), - SpacesBefore: 1, - }, - }, - }, - }, - NameTokens: &TokenSeq{Tokens{ + TestTreeNode{ + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Attribute", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, + Type: "comments", }, - }}, - EqualsTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, + Type: "identifier", + Val: "a", }, - }}, - Expr: &Expression{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - }, - LineCommentTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenComment, - Bytes: []byte("# because it is\n"), - SpacesBefore: 1, + Type: "Tokens", + Val: " =", }, - }}, - }, - }, - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ + { + Type: "Expression", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, + Type: "Tokens", + Val: " 1", }, }, }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenComment, - Bytes: []byte("# because it is\n"), - SpacesBefore: 1, - }, + { + Type: "comments", + Val: " # because it is\n", }, }, }, @@ -427,137 +146,43 @@ func TestParse(t *testing.T) { }, { "# bee bee bee\n\nb = 1\n", // two newlines separate the comment from the attribute - &Body{ - Items: []Node{ - &Attribute{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`b`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - NameTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`b`), - SpacesBefore: 0, - }, - }}, - EqualsTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }}, - Expr: &Expression{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - }, - EOLTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }}, + TestTreeNode{ + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Tokens", // Only lead/line comments attached to an object have type "comments" + Val: "# bee bee bee\n\n", }, - }, - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ + { + Type: "Attribute", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenComment, - Bytes: []byte("# bee bee bee\n"), - SpacesBefore: 0, + Type: "comments", }, { - Type: hclsyntax.TokenNewline, - Bytes: []byte("\n"), - SpacesBefore: 0, + Type: "identifier", + Val: "b", }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`b`), - SpacesBefore: 0, - }, + { + Type: "Tokens", + Val: " =", }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ + { + Type: "Expression", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, + Type: "Tokens", + Val: " 1", }, }, }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, + { + Type: "comments", + Val: "", + }, + { + Type: "Tokens", + Val: "\n", }, }, }, @@ -566,116 +191,33 @@ func TestParse(t *testing.T) { }, { "b {}\n", - &Body{ - Items: []Node{ - &Block{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`b`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenOBrace, - Bytes: []byte(`{`), - SpacesBefore: 1, - }, - }, - }, - (*TokenSeq)(nil), // the empty body - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenCBrace, - Bytes: []byte(`}`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - TypeTokens: &TokenSeq{Tokens{ + TestTreeNode{ + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Block", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`b`), - SpacesBefore: 0, + Type: "comments", }, - }}, - OBraceTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenOBrace, - Bytes: []byte(`{`), - SpacesBefore: 1, + Type: "identifier", + Val: "b", }, - }}, - Body: &Body{}, - CBraceTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenCBrace, - Bytes: []byte(`}`), - SpacesBefore: 0, + Type: "Tokens", + Val: " {", }, - }}, - EOLTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, + Type: "Body", }, - }}, - }, - }, - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`b`), - SpacesBefore: 0, - }, + { + Type: "Tokens", + Val: "}", }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenOBrace, - Bytes: []byte(`{`), - SpacesBefore: 1, - }, - }, - }, - (*TokenSeq)(nil), // the empty body - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenCBrace, - Bytes: []byte(`}`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, + { + Type: "Tokens", + Val: "\n", }, }, }, @@ -684,352 +226,71 @@ func TestParse(t *testing.T) { }, { "b {\n a = 1\n}\n", - &Body{ - Items: []Node{ - &Block{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`b`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenOBrace, - Bytes: []byte(`{`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 2, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenCBrace, - Bytes: []byte(`}`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - TypeTokens: &TokenSeq{Tokens{ + TestTreeNode{ + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Block", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`b`), - SpacesBefore: 0, + Type: "comments", }, - }}, - OBraceTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenOBrace, - Bytes: []byte(`{`), - SpacesBefore: 1, + Type: "identifier", + Val: "b", }, - }}, - Body: &Body{ - Items: []Node{ - &Attribute{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 2, - }, + { + Type: "Tokens", + Val: " {", + }, + { + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Tokens", + Val: "\n", + }, + { + Type: "Attribute", + Children: []TestTreeNode{ + { + Type: "comments", }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, + { + Type: "identifier", + Val: " a", }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ + { + Type: "Tokens", + Val: " =", + }, + { + Type: "Expression", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, + Type: "Tokens", + Val: " 1", }, }, }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - NameTokens: &TokenSeq{ - Tokens{ { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 2, + Type: "comments", }, - }, - }, - EqualsTokens: &TokenSeq{ - Tokens{ { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - Expr: &Expression{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - }, - EOLTokens: &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, + Type: "Tokens", + Val: "\n", }, }, }, }, }, - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 2, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - }, - }, - CBraceTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenCBrace, - Bytes: []byte(`}`), - SpacesBefore: 0, + Type: "Tokens", + Val: "}", }, - }}, - EOLTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }}, - }, - }, - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`b`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenOBrace, - Bytes: []byte(`{`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 2, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`1`), - SpacesBefore: 1, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenCBrace, - Bytes: []byte(`}`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, + Type: "Tokens", + Val: "\n", }, }, }, @@ -1038,163 +299,48 @@ func TestParse(t *testing.T) { }, { "a = foo\n", - &Body{ - Items: []Node{ - &Attribute{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, + TestTreeNode{ + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Attribute", + Children: []TestTreeNode{ + { + Type: "comments", }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, + { + Type: "identifier", + Val: "a", }, - &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ + { + Type: "Tokens", + Val: " =", + }, + { + Type: "Expression", + Children: []TestTreeNode{ + { + Type: "Traversal", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - NameTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }}, - EqualsTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }}, - Expr: &Expression{ - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - }, - }, - AbsTraversals: []*Traversal{ - { - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - }, - Steps: []*Traverser{ - { - AllTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }}, - Logical: hcl.TraverseRoot{ - Name: "foo", - SrcRange: hcl.Range{ - Filename: "", - Start: hcl.Pos{Line: 1, Column: 5, Byte: 4}, - End: hcl.Pos{Line: 1, Column: 8, Byte: 7}, + Type: "TraverseName", + Children: []TestTreeNode{ + { + Type: "identifier", + Val: " foo", + }, }, }, }, }, }, }, - }, - EOLTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, + Type: "comments", }, - }}, - }, - }, - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, + { + Type: "Tokens", + Val: "\n", }, }, }, @@ -1203,241 +349,61 @@ func TestParse(t *testing.T) { }, { "a = foo.bar\n", - &Body{ - Items: []Node{ - &Attribute{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenDot, - Bytes: []byte(`.`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`bar`), - SpacesBefore: 0, - }, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - NameTokens: &TokenSeq{Tokens{ + TestTreeNode{ + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Attribute", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, + Type: "comments", }, - }}, - EqualsTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, + Type: "identifier", + Val: "a", }, - }}, - Expr: &Expression{ - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenDot, - Bytes: []byte(`.`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`bar`), - SpacesBefore: 0, - }, - }, - }, - }, - }, - AbsTraversals: []*Traversal{ - { - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenDot, - Bytes: []byte(`.`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`bar`), - SpacesBefore: 0, - }, - }, - }, - }, - Steps: []*Traverser{ - { - AllTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }}, - Logical: hcl.TraverseRoot{ - Name: "foo", - SrcRange: hcl.Range{ - Filename: "", - Start: hcl.Pos{Line: 1, Column: 5, Byte: 4}, - End: hcl.Pos{Line: 1, Column: 8, Byte: 7}, - }, - }, - }, - { - AllTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenDot, - Bytes: []byte(`.`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`bar`), - SpacesBefore: 0, - }, - }}, - Logical: hcl.TraverseAttr{ - Name: "bar", - SrcRange: hcl.Range{ - Filename: "", - Start: hcl.Pos{Line: 1, Column: 8, Byte: 7}, - End: hcl.Pos{Line: 1, Column: 12, Byte: 11}, - }, - }, - }, - }, - }, - }, - }, - EOLTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, + Type: "Tokens", + Val: " =", }, - }}, - }, - }, - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenDot, - Bytes: []byte(`.`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`bar`), - SpacesBefore: 0, + { + Type: "Expression", + Children: []TestTreeNode{ + { + Type: "Traversal", + Children: []TestTreeNode{ + { + Type: "TraverseName", + Children: []TestTreeNode{ + { + Type: "identifier", + Val: " foo", + }, + }, + }, + { + Type: "TraverseName", + Children: []TestTreeNode{ + { + Type: "Tokens", + Val: ".", + }, + { + Type: "identifier", + Val: "bar", + }, + }, + }, }, }, }, }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, + { + Type: "comments", + }, + { + Type: "Tokens", + Val: "\n", }, }, }, @@ -1446,266 +412,65 @@ func TestParse(t *testing.T) { }, { "a = foo[0]\n", - &Body{ - Items: []Node{ - &Attribute{ - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenOBrack, - Bytes: []byte(`[`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`0`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenCBrack, - Bytes: []byte(`]`), - SpacesBefore: 0, - }, - }, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, - }, - }, - }, - NameTokens: &TokenSeq{Tokens{ + TestTreeNode{ + Type: "Body", + Children: []TestTreeNode{ + { + Type: "Attribute", + Children: []TestTreeNode{ { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, + Type: "comments", }, - }}, - EqualsTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, + Type: "identifier", + Val: "a", }, - }}, - Expr: &Expression{ - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenOBrack, - Bytes: []byte(`[`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`0`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenCBrack, - Bytes: []byte(`]`), - SpacesBefore: 0, - }, - }, - }, - }, - }, - AbsTraversals: []*Traversal{ - { - AllTokens: &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenOBrack, - Bytes: []byte(`[`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`0`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenCBrack, - Bytes: []byte(`]`), - SpacesBefore: 0, - }, - }, - }, - }, - Steps: []*Traverser{ - { - AllTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }}, - Logical: hcl.TraverseRoot{ - Name: "foo", - SrcRange: hcl.Range{ - Filename: "", - Start: hcl.Pos{Line: 1, Column: 5, Byte: 4}, - End: hcl.Pos{Line: 1, Column: 8, Byte: 7}, - }, - }, - }, - { - AllTokens: &TokenSeq{Tokens{ - { - Type: hclsyntax.TokenOBrack, - Bytes: []byte(`[`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`0`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenCBrack, - Bytes: []byte(`]`), - SpacesBefore: 0, - }, - }}, - Logical: hcl.TraverseIndex{ - Key: cty.NumberVal(big.NewFloat(0).SetPrec(512)), - SrcRange: hcl.Range{ - Filename: "", - Start: hcl.Pos{Line: 1, Column: 8, Byte: 7}, - End: hcl.Pos{Line: 1, Column: 11, Byte: 10}, - }, - }, - }, - }, - }, - }, - }, - EOLTokens: &TokenSeq{Tokens{ { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, + Type: "Tokens", + Val: " =", }, - }}, - }, - }, - AllTokens: &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`a`), - SpacesBefore: 0, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenEqual, - Bytes: []byte(`=`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - &TokenSeq{ - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenIdent, - Bytes: []byte(`foo`), - SpacesBefore: 1, - }, - }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenOBrack, - Bytes: []byte(`[`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenNumberLit, - Bytes: []byte(`0`), - SpacesBefore: 0, - }, - { - Type: hclsyntax.TokenCBrack, - Bytes: []byte(`]`), - SpacesBefore: 0, + { + Type: "Expression", + Children: []TestTreeNode{ + { + Type: "Traversal", + Children: []TestTreeNode{ + { + Type: "TraverseName", + Children: []TestTreeNode{ + { + Type: "identifier", + Val: " foo", + }, + }, + }, + { + Type: "TraverseIndex", + Children: []TestTreeNode{ + { + Type: "Tokens", + Val: "[", + }, + { + Type: "number", + Val: "0", + }, + { + Type: "Tokens", + Val: "]", + }, + }, + }, }, }, }, }, - }, - &TokenSeq{ - Tokens{ - { - Type: hclsyntax.TokenNewline, - Bytes: []byte{'\n'}, - SpacesBefore: 0, - }, + { + Type: "comments", + }, + { + Type: "Tokens", + Val: "\n", }, }, }, @@ -1714,12 +479,6 @@ func TestParse(t *testing.T) { }, } - prettyConfig := &pretty.Config{ - Diffable: true, - IncludeUnexported: true, - PrintStringers: true, - } - for _, test := range tests { t.Run(test.src, func(t *testing.T) { file, diags := parse([]byte(test.src), "", hcl.Pos{Line: 1, Column: 1}) @@ -1730,24 +489,17 @@ func TestParse(t *testing.T) { t.Fatalf("unexpected diagnostics") } - got := file.Body + got := makeTestTree(file.body) - if !reflect.DeepEqual(got, test.want) { - diff := prettyConfig.Compare(got, test.want) - if diff != "" { - t.Errorf( - "wrong result\ninput: %s\ndiff: %s", - test.src, - diff, - ) - } else { - t.Errorf( - "wrong result\ninput: %s\ngot: %s\nwant: %s", - test.src, - spew.Sdump(got), - spew.Sdump(test.want), - ) - } + if !cmp.Equal(got, test.want) { + diff := cmp.Diff(got, test.want) + t.Errorf( + "wrong result\ninput:\n%s\n\ngot:\n%s\nwant:%s\n\ndiff:\n%s", + test.src, + spew.Sdump(got), + spew.Sdump(test.want), + diff, + ) } }) } diff --git a/hclwrite/public.go b/hclwrite/public.go index d9b0dd5..b4d7f15 100644 --- a/hclwrite/public.go +++ b/hclwrite/public.go @@ -6,6 +6,20 @@ import ( "github.com/hashicorp/hcl2/hcl" ) +// NewFile creates a new file object that is empty and ready to have constructs +// added t it. +func NewFile() *File { + body := &Body{ + inTree: newInTree(), + indentLevel: 0, + } + file := &File{ + inTree: newInTree(), + } + file.body = file.inTree.children.Append(body) + return file +} + // ParseConfig interprets the given source bytes into a *hclwrite.File. The // resulting AST can be used to perform surgical edits on the source code // before turning it back into bytes again. @@ -25,6 +39,6 @@ func Format(src []byte) []byte { tokens := lexConfig(src) format(tokens) buf := &bytes.Buffer{} - (&TokenSeq{tokens}).WriteTo(buf) + tokens.WriteTo(buf) return buf.Bytes() } diff --git a/hclwrite/round_trip_test.go b/hclwrite/round_trip_test.go index b998dbe..1a00731 100644 --- a/hclwrite/round_trip_test.go +++ b/hclwrite/round_trip_test.go @@ -4,11 +4,13 @@ import ( "bytes" "testing" - "github.com/hashicorp/hcl2/hcl/hclsyntax" - "github.com/hashicorp/hcl2/hcl" + "github.com/sergi/go-diff/diffmatchpatch" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/function" "github.com/zclconf/go-cty/cty/function/stdlib" + + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" ) func TestRoundTripVerbatim(t *testing.T) { @@ -68,7 +70,10 @@ block { result := wr.Bytes() if !bytes.Equal(result, src) { - t.Errorf("wrong result\nresult:\n%s\ninput:\n%s", result, src) + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(string(src), string(result), false) + //t.Errorf("wrong result\nresult:\n%s\ninput:\n%s", result, src) + t.Errorf("wrong result\ndiff: (red indicates missing lines, and green indicates unexpected lines)\n%s", dmp.DiffPrettyText(diffs)) } }) } diff --git a/hclwrite/tokens.go b/hclwrite/tokens.go index 8fb87f2..f7033ff 100644 --- a/hclwrite/tokens.go +++ b/hclwrite/tokens.go @@ -8,19 +8,6 @@ import ( "github.com/hashicorp/hcl2/hcl/hclsyntax" ) -// TokenGen is an abstract type that can append tokens to a list. It is the -// low-level foundation underlying the hclwrite AST; the AST provides a -// convenient abstraction over raw token sequences to facilitate common tasks, -// but it's also possible to directly manipulate the tree of token generators -// to make changes that the AST API doesn't directly allow. -type TokenGen interface { - EachToken(TokenCallback) -} - -// TokenCallback is used with TokenGen implementations to specify the action -// that is to be taken for each token in the flattened token sequence. -type TokenCallback func(*Token) - // Token is a single sequence of bytes annotated with a type. It is similar // in purpose to hclsyntax.Token, but discards the source position information // since that is not useful in code generation. @@ -38,17 +25,16 @@ type Token struct { // Tokens is a flat list of tokens. type Tokens []*Token -func (ts Tokens) WriteTo(wr io.Writer) (int, error) { - seq := &TokenSeq{ts} - return seq.WriteTo(wr) -} - func (ts Tokens) Bytes() []byte { buf := &bytes.Buffer{} ts.WriteTo(buf) return buf.Bytes() } +func (ts Tokens) testValue() string { + return string(ts.Bytes()) +} + // Columns returns the number of columns (grapheme clusters) the token sequence // occupies. The result is not meaningful if there are newline or single-line // comment tokens in the sequence. @@ -62,43 +48,10 @@ func (ts Tokens) Columns() int { return ret } -// TokenSeq combines zero or more TokenGens together to produce a flat sequence -// of tokens from a tree of TokenGens. -type TokenSeq []TokenGen - -func (t *Token) EachToken(cb TokenCallback) { - cb(t) -} - -func (ts Tokens) EachToken(cb TokenCallback) { - for _, t := range ts { - cb(t) - } -} - -func (ts *TokenSeq) EachToken(cb TokenCallback) { - if ts == nil { - return - } - for _, gen := range *ts { - gen.EachToken(cb) - } -} - -// Tokens returns the flat list of tokens represented by the receiving -// token sequence. -func (ts *TokenSeq) Tokens() Tokens { - var tokens Tokens - ts.EachToken(func(token *Token) { - tokens = append(tokens, token) - }) - return tokens -} - // WriteTo takes an io.Writer and writes the bytes for each token to it, // along with the spacing that separates each token. In other words, this // allows serializing the tokens to a file or other such byte stream. -func (ts *TokenSeq) WriteTo(wr io.Writer) (int, error) { +func (ts Tokens) WriteTo(wr io.Writer) (int, error) { // We know we're going to be writing a lot of small chunks of repeated // space characters, so we'll prepare a buffer of these that we can // easily pass to wr.Write without any further allocation. @@ -109,9 +62,9 @@ func (ts *TokenSeq) WriteTo(wr io.Writer) (int, error) { var n int var err error - ts.EachToken(func(token *Token) { + for _, token := range ts { if err != nil { - return + return n, err } for spacesBefore := token.SpacesBefore; spacesBefore > 0; spacesBefore -= len(spaces) { @@ -123,48 +76,22 @@ func (ts *TokenSeq) WriteTo(wr io.Writer) (int, error) { thisN, err = wr.Write(spaces[:thisChunk]) n += thisN if err != nil { - return + return n, err } } var thisN int thisN, err = wr.Write(token.Bytes) n += thisN - }) + } return n, err } -// SoloToken returns the single token represented by the receiving sequence, -// or nil if the sequence does not represent exactly one token. -func (ts *TokenSeq) SoloToken() *Token { - var ret *Token - found := false - ts.EachToken(func(tok *Token) { - if ret == nil && !found { - ret = tok - found = true - } else if ret != nil && found { - ret = nil - } - }) - return ret +func (ts Tokens) walkChildNodes(w internalWalkFunc) { + // Unstructured tokens have no child nodes } -// IsIdent returns true if and only if the token sequence represents a single -// ident token whose name matches the given string. -func (ts *TokenSeq) IsIdent(name []byte) bool { - tok := ts.SoloToken() - if tok == nil { - return false - } - if tok.Type != hclsyntax.TokenIdent { - return false - } - return bytes.Equal(tok.Bytes, name) +func (ts Tokens) BuildTokens(to Tokens) Tokens { + return append(to, ts...) } - -// TokenSeqEmpty is a TokenSeq that contains no tokens. It can be used anywhere, -// but its primary purpose is to be assigned as a replacement for a non-empty -// TokenSeq when eliminating a section of an input file. -var TokenSeqEmpty = TokenSeq([]TokenGen(nil))