zclwrite: method for writing tokens to a writer

This now allows for round-tripping some input from bytes to tokens and
then back to bytes again. With no other changes, we expect this to produce
an identical result.

zclwrite/ast.go

@@ -1,5 +1,7 @@
 package zclwrite
 
+import "io"
+
 type Node interface {
 	walkChildNodes(w internalWalkFunc)
 	Tokens() *TokenSeq
@@ -14,6 +16,10 @@ type File struct {
 	Body *Body
 }
 
+func (f *File) WriteTo(wr io.Writer) (int, error) {
+	return f.Body.AllTokens.WriteTo(wr)
+}
+
 type Body struct {
 	// Items may contain Attribute, Block and Unstructured instances.
 	Items     []Node

zclwrite/round_trip_test.go

@@ -0,0 +1,58 @@
+package zclwrite
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+func TestRoundTrip(t *testing.T) {
+	tests := []string{
+		``,
+		`foo = 1`,
+		`
+foobar = 1
+baz    = 1
+`,
+		`
+# this file is awesome
+
+# tossed salads and scrambled eggs
+block "thing" {
+  foobar = 1 # quite stylish
+  baz    = 1
+}
+
+# and they all lived happily ever after
+`,
+	}
+
+	for _, test := range tests {
+		t.Run(test, func(t *testing.T) {
+			src := []byte(test)
+			file, diags := parse(src, "", zcl.Pos{Line: 1, Column: 1})
+			if len(diags) != 0 {
+				for _, diag := range diags {
+					t.Logf(" - %s", diag.Error())
+				}
+				t.Fatalf("unexpected diagnostics")
+			}
+
+			wr := &bytes.Buffer{}
+			n, err := file.WriteTo(wr)
+			if n != len(test) {
+				t.Errorf("wrong number of bytes %d; want %d", n, len(test))
+			}
+			if err != nil {
+				t.Fatalf("error from WriteTo")
+			}
+
+			result := wr.Bytes()
+
+			if !bytes.Equal(result, src) {
+				t.Errorf("wrong result\nresult:\n%s\ninput\n:%s", result, src)
+			}
+		})
+	}
+}

zclwrite/tokens.go

@@ -1,6 +1,8 @@
 package zclwrite
 
 import (
+	"io"
+
 	"github.com/zclconf/go-zcl/zcl/zclsyntax"
 )
 
@@ -64,8 +66,44 @@ func (ts *TokenSeq) Tokens() Tokens {
 	return tokens
 }
 
-func (ts *TokenSeq) Append(other *TokenSeq) {
+// WriteTo takes an io.Writer and writes the bytes for each token to it,
-	*ts = append(*ts, other)
+// along with the spacing that separates each token. In other words, this
+// allows serializing the tokens to a file or other such byte stream.
+func (ts *TokenSeq) WriteTo(wr io.Writer) (int, error) {
+	// We know we're going to be writing a lot of small chunks of repeated
+	// space characters, so we'll prepare a buffer of these that we can
+	// easily pass to wr.Write without any further allocation.
+	spaces := make([]byte, 40)
+	for i := range spaces {
+		spaces[i] = ' '
+	}
+
+	var n int
+	var err error
+	ts.EachToken(func(token *Token) {
+		if err != nil {
+			return
+		}
+
+		for spacesBefore := token.SpacesBefore; spacesBefore > 0; spacesBefore -= len(spaces) {
+			thisChunk := spacesBefore
+			if thisChunk > len(spaces) {
+				thisChunk = len(spaces)
+			}
+			var thisN int
+			thisN, err = wr.Write(spaces[:thisChunk])
+			n += thisN
+			if err != nil {
+				return
+			}
+		}
+
+		var thisN int
+		thisN, err = wr.Write(token.Bytes)
+		n += thisN
+	})
+
+	return n, err
 }
 
 // TokenSeqEmpty is a TokenSeq that contains no tokens. It can be used anywhere,