zclwrite: convert zclsyntax tokens into zclwrite tokens

In zclwrite we throw away the absolute source position information and
instead just retain the number of spaces before each token. This different
model allows us to rewrite parts of the token sequence without needing
to re-adjust all of the positions, and it also allows us to do simple
indentation and spacing adjustments just by walking through the token
list and adjusting these numbers.

zcl/zclsyntax/scan_tokens.rl

@@ -62,6 +62,9 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
         # use spaces instead.
         Tabs = 0x09+;
 
+        # Note: zclwrite assumes that only ASCII spaces appear between tokens,
+        # and uses this assumption to recreate the spaces between tokens by
+        # looking at byte offset differences.
         Spaces = ' '+;
 
         action beginStringTemplate {

zclwrite/parser.go

@@ -0,0 +1,34 @@
+package zclwrite
+
+import (
+	"github.com/zclconf/go-zcl/zcl"
+	"github.com/zclconf/go-zcl/zcl/zclsyntax"
+)
+
+// lexConfig uses the zclsyntax scanner to get a token stream and then
+// rewrites it into this package's token model.
+func lexConfig(src []byte) Tokens {
+	mainTokens := zclsyntax.LexConfig(src, "", zcl.Pos{Byte: 0, Line: 1, Column: 1})
+	ret := make(Tokens, len(mainTokens))
+	var lastByteOffset int
+	for i, mainToken := range mainTokens {
+		// Create a copy of the bytes so that we can mutate without
+		// corrupting the original token stream.
+		bytes := make([]byte, len(mainToken.Bytes))
+		copy(bytes, mainToken.Bytes)
+
+		ret[i] = &Token{
+			Type:  mainToken.Type,
+			Bytes: bytes,
+
+			// We assume here that spaces are always ASCII spaces, since
+			// that's what the scanner also assumes, and thus the number
+			// of bytes skipped is also the number of space characters.
+			SpacesBefore: mainToken.Range.Start.Byte - lastByteOffset,
+		}
+
+		lastByteOffset = mainToken.Range.End.Byte
+	}
+
+	return ret
+}

zclwrite/parser_test.go

@@ -0,0 +1,160 @@
+package zclwrite
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/zclconf/go-zcl/zcl/zclsyntax"
+)
+
+func TestLexConfig(t *testing.T) {
+	tests := []struct {
+		input string
+		want  Tokens
+	}{
+		{
+			`a  b `,
+			Tokens{
+				{
+					Type:         zclsyntax.TokenIdent,
+					Bytes:        []byte(`a`),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenIdent,
+					Bytes:        []byte(`b`),
+					SpacesBefore: 2,
+				},
+				{
+					Type:         zclsyntax.TokenEOF,
+					Bytes:        []byte{},
+					SpacesBefore: 1,
+				},
+			},
+		},
+		{
+			`
+foo "bar" "baz" {
+    pizza = " cheese "
+}
+`,
+			Tokens{
+				{
+					Type:         zclsyntax.TokenNewline,
+					Bytes:        []byte{'\n'},
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenIdent,
+					Bytes:        []byte(`foo`),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenOQuote,
+					Bytes:        []byte(`"`),
+					SpacesBefore: 1,
+				},
+				{
+					Type:         zclsyntax.TokenStringLit,
+					Bytes:        []byte(`bar`),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenCQuote,
+					Bytes:        []byte(`"`),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenOQuote,
+					Bytes:        []byte(`"`),
+					SpacesBefore: 1,
+				},
+				{
+					Type:         zclsyntax.TokenStringLit,
+					Bytes:        []byte(`baz`),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenCQuote,
+					Bytes:        []byte(`"`),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenOBrace,
+					Bytes:        []byte(`{`),
+					SpacesBefore: 1,
+				},
+				{
+					Type:         zclsyntax.TokenNewline,
+					Bytes:        []byte("\n"),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenIdent,
+					Bytes:        []byte(`pizza`),
+					SpacesBefore: 4,
+				},
+				{
+					Type:         zclsyntax.TokenEqual,
+					Bytes:        []byte(`=`),
+					SpacesBefore: 1,
+				},
+				{
+					Type:         zclsyntax.TokenOQuote,
+					Bytes:        []byte(`"`),
+					SpacesBefore: 1,
+				},
+				{
+					Type:         zclsyntax.TokenStringLit,
+					Bytes:        []byte(` cheese `),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenCQuote,
+					Bytes:        []byte(`"`),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenNewline,
+					Bytes:        []byte("\n"),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenCBrace,
+					Bytes:        []byte(`}`),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenNewline,
+					Bytes:        []byte("\n"),
+					SpacesBefore: 0,
+				},
+				{
+					Type:         zclsyntax.TokenEOF,
+					Bytes:        []byte{},
+					SpacesBefore: 0,
+				},
+			},
+		},
+	}
+
+	prettyConfig := &pretty.Config{
+		Diffable:          true,
+		IncludeUnexported: true,
+		PrintStringers:    true,
+	}
+
+	for _, test := range tests {
+		t.Run(test.input, func(t *testing.T) {
+			got := lexConfig([]byte(test.input))
+
+			if !reflect.DeepEqual(got, test.want) {
+				diff := prettyConfig.Compare(test.want, got)
+				t.Errorf(
+					"wrong result\ninput: %s\ndiff:  %s", test.input, diff,
+				)
+			}
+		})
+	}
+}

zclwrite/tokens.go

@@ -24,7 +24,7 @@ type Token struct {
 	// reproduce the exact layout of the original file when we're making
 	// surgical changes in-place. When _new_ code is created it will always
 	// be in the canonical style, but we preserve layout of existing code.
-	SpaceBefore int
+	SpacesBefore int
 }
 
 // Tokens is a flat list of tokens.