zclsyntax: scanning of bare (top-level) templates

This alternative scanning mode makes the scanner start in template
context rather than normal context. This will be later used by the parser
to allow parsing of standalone templates that aren't embedded inside a
zcl configuration file.

zcl/zclsyntax/scan_tokens.rl

@@ -123,6 +123,10 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
             token(TokenStringLit);
         }
 
+        action bareTemplateLiteral {
+            token(TokenStringLit);
+        }
+
         action beginTemplateInterp {
             token(TokenTemplateInterp);
             braces++;
@@ -189,6 +193,11 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
             ('!' ^'{') |
             (StringLiteralChars - ("$" | "!"))
         )*;
+        BareStringLiteral = (
+            ('$' ^'{') |
+            ('!' ^'{') |
+            (StringLiteralChars - ("$" | "!"))
+        )* Newline?;
 
         stringTemplate := |*
             TemplateInterp        => beginTemplateInterp;
@@ -207,6 +216,13 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
             BrokenUTF8            => { token(TokenBadUTF8); };
         *|;
 
+        bareTemplate := |*
+            TemplateInterp        => beginTemplateInterp;
+            TemplateControl       => beginTemplateControl;
+            BareStringLiteral     => bareTemplateLiteral;
+            BrokenUTF8            => { token(TokenBadUTF8); };
+        *|;
+
         main := |*
             Spaces           => {};
             NumberLit        => { token(TokenNumberLit) };
@@ -252,9 +268,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
     case scanNormal:
         cs = zcltok_en_main
     case scanTemplate:
-        // scanTemplate is a variant of heredoc scanning, so will
-        // be implemented once that is implemented.
-        panic("scanTemplate not yet implemented")
+        cs = zcltok_en_bareTemplate
     default:
         panic("invalid scanMode")
     }

zcl/zclsyntax/scan_tokens_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/zclconf/go-zcl/zcl"
 )
 
-func TestScanTokens(t *testing.T) {
+func TestScanTokens_normal(t *testing.T) {
 	tests := []struct {
 		input string
 		want  []Token
@@ -1304,3 +1304,99 @@ EOF
 		})
 	}
 }
+
+func TestScanTokens_template(t *testing.T) {
+	tests := []struct {
+		input string
+		want  []Token
+	}{
+		// Empty input
+		{
+			``,
+			[]Token{
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   zcl.Pos{Byte: 0, Line: 1, Column: 1},
+					},
+				},
+			},
+		},
+
+		// Simple literals
+		{
+			` hello `,
+			[]Token{
+				{
+					Type:  TokenStringLit,
+					Bytes: []byte(` hello `),
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   zcl.Pos{Byte: 7, Line: 1, Column: 8},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 7, Line: 1, Column: 8},
+						End:   zcl.Pos{Byte: 7, Line: 1, Column: 8},
+					},
+				},
+			},
+		},
+		{
+			"\nhello\n",
+			[]Token{
+				{
+					Type:  TokenStringLit,
+					Bytes: []byte("\n"),
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   zcl.Pos{Byte: 1, Line: 2, Column: 1},
+					},
+				},
+				{
+					Type:  TokenStringLit,
+					Bytes: []byte("hello\n"),
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 1, Line: 2, Column: 1},
+						End:   zcl.Pos{Byte: 7, Line: 3, Column: 1},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 7, Line: 3, Column: 1},
+						End:   zcl.Pos{Byte: 7, Line: 3, Column: 1},
+					},
+				},
+			},
+		},
+
+		// TODO: also test interpolation sequences
+	}
+
+	prettyConfig := &pretty.Config{
+		Diffable:          true,
+		IncludeUnexported: true,
+		PrintStringers:    true,
+	}
+
+	for _, test := range tests {
+		t.Run(test.input, func(t *testing.T) {
+			got := scanTokens([]byte(test.input), "", zcl.Pos{Byte: 0, Line: 1, Column: 1}, scanTemplate)
+
+			if !reflect.DeepEqual(got, test.want) {
+				diff := prettyConfig.Compare(test.want, got)
+				t.Errorf(
+					"wrong result\ninput: %s\ndiff:  %s",
+					test.input, diff,
+				)
+			}
+		})
+	}
+}