zclwrite: stub of zcl code generation package

This uses a separate, lower-level AST than the parser so that it can
retain the raw tokens and make surgical changes. As a consequence it
has much less semantic detail than the parser AST, and in particular
doesn't represent the structure of expressions except to retain variable
references to enable global rename operations.

zclwrite/ast.go

@@ -0,0 +1,107 @@
+package zclwrite
+
+type Node interface {
+	walkChildNodes(w internalWalkFunc)
+	Tokens() *TokenSeq
+}
+
+type internalWalkFunc func(Node)
+
+type File struct {
+	Name  string
+	Bytes []byte
+
+	Body Body
+}
+
+type Body struct {
+	// Items may contain Attribute, Block and Unstructured instances.
+	Items     []Node
+	AllTokens *TokenSeq
+
+	// IndentLevel is the number of spaces that should appear at the start
+	// of lines added within this body.
+	IndentLevel int
+}
+
+func (n *Body) walkChildNodes(w internalWalkFunc) {
+	for _, item := range n.Items {
+		w(item)
+	}
+}
+
+func (n *Body) Tokens() *TokenSeq {
+	return n.AllTokens
+}
+
+type Attribute struct {
+	AllTokens *TokenSeq
+
+	LeadCommentTokens *TokenSeq
+	NameToken         *Token
+	EqualsToken       *Token
+	Value             *Expression
+	LineCommentTokens *TokenSeq
+	EOLToken          *Token
+}
+
+func (a *Attribute) walkChildNodes(w internalWalkFunc) {
+	w(a.Value)
+}
+
+type Block struct {
+	AllTokens *TokenSeq
+
+	LeadCommentTokens *TokenSeq
+	TypeToken         *Token
+	LabelTokens       []*TokenSeq
+	LabelTokensFlat   *TokenSeq
+	OBraceToken       *Token
+	Body              *Body
+	CBraceToken       *Token
+	EOLToken          *Token
+}
+
+func (n *Block) walkChildNodes(w internalWalkFunc) {
+	w(n.Body)
+}
+
+// Unstructured represents consecutive sets of tokens within a Body that
+// aren't part of any particular construct. This includes blank lines
+// and comments that aren't immediately before an attribute or nested block.
+type Unstructured struct {
+	AllTokens *TokenSeq
+}
+
+func (n *Unstructured) Tokens() *TokenSeq {
+	return n.AllTokens
+}
+
+type Expression struct {
+	AllTokens *TokenSeq
+	VarRefs   []*VarRef
+}
+
+func (n *Expression) walkChildNodes(w internalWalkFunc) {
+	for _, name := range n.VarRefs {
+		w(name)
+	}
+}
+
+func (n *Expression) Tokens() *TokenSeq {
+	return n.AllTokens
+}
+
+type VarRef struct {
+	// Tokens alternate between TokenIdent and TokenDot, with the first
+	// and last elements always being TokenIdent.
+	AllTokens *TokenSeq
+}
+
+func (n *VarRef) walkChildNodes(w internalWalkFunc) {
+	// no child nodes of a variable name
+}
+
+func (n *VarRef) Tokens() *TokenSeq {
+	return n.AllTokens
+}

zclwrite/doc.go

@@ -0,0 +1,7 @@
+// Package zclwrite deals with the problem of generating zcl configuration
+// and of making specific surgical changes to existing zcl configurations.
+//
+// It operates at a different level of abstraction that the main zcl parser
+// and AST, since details such as the placement of comments and newlines
+// are preserved when unchanged.
+package zclwrite

zclwrite/tokens.go

@@ -0,0 +1,66 @@
+package zclwrite
+
+import (
+	"github.com/zclconf/go-zcl/zcl/zclsyntax"
+)
+
+// TokenGen is an abstract type that can append tokens to a list. It is the
+// low-level foundation underlying the zclwrite AST; the AST provides a
+// convenient abstraction over raw token sequences to facilitate common tasks,
+// but it's also possible to directly manipulate the tree of token generators
+// to make changes that the AST API doesn't directly allow.
+type TokenGen interface {
+	AppendToTokens(Tokens) Tokens
+}
+
+// Token is a single sequence of bytes annotated with a type. It is similar
+// in purpose to zclsyntax.Token, but discards the source position information
+// since that is not useful in code generation.
+type Token struct {
+	Type  zclsyntax.TokenType
+	Bytes []byte
+
+	// We record the number of spaces before each token so that we can
+	// reproduce the exact layout of the original file when we're making
+	// surgical changes in-place. When _new_ code is created it will always
+	// be in the canonical style, but we preserve layout of existing code.
+	SpaceBefore int
+}
+
+// Tokens is a flat list of tokens.
+type Tokens []*Token
+
+// TokenSeq combines zero or more TokenGens together to produce a flat sequence
+// of tokens from a tree of TokenGens.
+type TokenSeq []TokenGen
+
+func (t *Token) AppendToTokens(src Tokens) Tokens {
+	return append(src, t)
+}
+
+func (ts *Tokens) AppendToTokens(src Tokens) Tokens {
+	return append(src, (*ts)...)
+}
+
+func (ts *TokenSeq) AppendToTokens(src Tokens) Tokens {
+	toks := src
+	for _, gen := range *ts {
+		toks = gen.AppendToTokens(toks)
+	}
+	return toks
+}
+
+// Tokens returns the flat list of tokens represented by the receiving
+// token sequence.
+func (ts *TokenSeq) Tokens() Tokens {
+	return ts.AppendToTokens(nil)
+}
+
+func (ts *TokenSeq) Append(other *TokenSeq) {
+	*ts = append(*ts, other)
+}
+
+// TokenSeqEmpty is a TokenSeq that contains no tokens. It can be used anywhere,
+// but its primary purpose is to be assigned as a replacement for a non-empty
+// TokenSeq when eliminating a section of an input file.
+var TokenSeqEmpty = TokenSeq([]TokenGen(nil))