hcl/hclsyntax: Correct scanning of literal $ and % before quotes

The TemplateStringLiteral production was not quite right, causing a
literal $ or % immediately followed by " to consume the quotes and any
following characters on the line if there were any more characters on the
line.

Now we match things more precisely, but at the expense of generating some
redundant extra tokens when escapes and literal dollar/percent signs are
present. Those extra tokens don't matter in practice because the resulting
strings get concatenated together anyway, which is proven by the fact
that this changeset includes changes only to the scanner and parser tests,
and not to any of the expression result tests.

While here, I also improved the error message for when the user attempts
to split a quoted string over multiple lines. Previously it was just using
the generic "invalid character" message, which isn't particularly
actionable. Now we'll give the user a couple options of what to do
instead.

go.mod

@@ -22,9 +22,11 @@ require (
 	github.com/spf13/pflag v1.0.2
 	github.com/stretchr/testify v1.2.2 // indirect
 	github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2
-	golang.org/x/crypto v0.0.0-20180816225734-aabede6cba87
+	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734
-	golang.org/x/net v0.0.0-20181129055619-fae4c4e3ad76 // indirect
+	golang.org/x/net v0.0.0-20190502183928-7f726cade0ab // indirect
-	golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
+	golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 // indirect
+	golang.org/x/text v0.3.2 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 	howett.net/plist v0.0.0-20181124034731-591f970eefbb
 )

go.sum

@@ -47,23 +47,30 @@ github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
-github.com/zclconf/go-cty v0.0.0-20190124225737-a385d646c1e9 h1:hHCAGde+QfwbqXSAqOmBd4NlOrJ6nmjWp+Nu408ezD4=
-github.com/zclconf/go-cty v0.0.0-20190124225737-a385d646c1e9/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
 github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2 h1:Ai1LhlYNEqE39zGU07qHDNJ41iZVPZfZr1dSCoXrp1w=
 github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
-golang.org/x/crypto v0.0.0-20180816225734-aabede6cba87 h1:gCHhzI+1R9peHIMyiWVxoVaWlk1cYK7VThX5ptLtbXY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20180816225734-aabede6cba87/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181129055619-fae4c4e3ad76 h1:xx5MUFyRQRbPk6VjWjIE1epE/K5AoDD8QUN116NCy8k=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20181129055619-fae4c4e3ad76/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190502183928-7f726cade0ab h1:9RfW3ktsOZxgo9YNbBAjq1FWzc/igwEcUzZz8IXgSbk=
+golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w=
+golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

hcl/hclsyntax/expression_test.go

@@ -236,6 +236,18 @@ func TestExpressionParseAndValue(t *testing.T) {
 			cty.StringVal("hello $$nonescape"),
 			0,
 		},
+		{
+			`"$"`,
+			nil,
+			cty.StringVal("$"),
+			0,
+		},
+		{
+			`"%"`,
+			nil,
+			cty.StringVal("%"),
+			0,
+		},
 		{
 			`upper("foo")`,
 			&hcl.EvalContext{

hcl/hclsyntax/parser_template.go

@@ -2,10 +2,10 @@ package hclsyntax
 
 import (
 	"fmt"
-	"github.com/apparentlymart/go-textseg/textseg"
 	"strings"
 	"unicode"
 
+	"github.com/apparentlymart/go-textseg/textseg"
 	"github.com/hashicorp/hcl2/hcl"
 	"github.com/zclconf/go-cty/cty"
 )

hcl/hclsyntax/parser_test.go

@@ -693,10 +693,26 @@ block "valid" {}
 						Expr: &TemplateExpr{
 							Parts: []Expression{
 								&LiteralValueExpr{
-									Val: cty.StringVal("hello ${true}"),
+									Val: cty.StringVal("hello "),
 
 									SrcRange: hcl.Range{
 										Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
+										End:   hcl.Pos{Line: 1, Column: 12, Byte: 11},
+									},
+								},
+								&LiteralValueExpr{
+									Val: cty.StringVal("${"),
+
+									SrcRange: hcl.Range{
+										Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
+										End:   hcl.Pos{Line: 1, Column: 15, Byte: 14},
+									},
+								},
+								&LiteralValueExpr{
+									Val: cty.StringVal("true}"),
+
+									SrcRange: hcl.Range{
+										Start: hcl.Pos{Line: 1, Column: 15, Byte: 14},
 										End:   hcl.Pos{Line: 1, Column: 20, Byte: 19},
 									},
 								},
@@ -743,10 +759,26 @@ block "valid" {}
 						Expr: &TemplateExpr{
 							Parts: []Expression{
 								&LiteralValueExpr{
-									Val: cty.StringVal("hello %{true}"),
+									Val: cty.StringVal("hello "),
 
 									SrcRange: hcl.Range{
 										Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
+										End:   hcl.Pos{Line: 1, Column: 12, Byte: 11},
+									},
+								},
+								&LiteralValueExpr{
+									Val: cty.StringVal("%{"),
+
+									SrcRange: hcl.Range{
+										Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
+										End:   hcl.Pos{Line: 1, Column: 15, Byte: 14},
+									},
+								},
+								&LiteralValueExpr{
+									Val: cty.StringVal("true}"),
+
+									SrcRange: hcl.Range{
+										Start: hcl.Pos{Line: 1, Column: 15, Byte: 14},
 										End:   hcl.Pos{Line: 1, Column: 20, Byte: 19},
 									},
 								},
@@ -793,11 +825,11 @@ block "valid" {}
 						Expr: &TemplateExpr{
 							Parts: []Expression{
 								&LiteralValueExpr{
-									Val: cty.StringVal("hello $"),
+									Val: cty.StringVal("hello "),
 
 									SrcRange: hcl.Range{
 										Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
-										End:   hcl.Pos{Line: 1, Column: 13, Byte: 12},
+										End:   hcl.Pos{Line: 1, Column: 12, Byte: 11},
 									},
 								},
 								// This parses oddly due to how the scanner
@@ -806,6 +838,14 @@ block "valid" {}
 								&LiteralValueExpr{
 									Val: cty.StringVal("$"),
 
+									SrcRange: hcl.Range{
+										Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
+										End:   hcl.Pos{Line: 1, Column: 13, Byte: 12},
+									},
+								},
+								&LiteralValueExpr{
+									Val: cty.StringVal("$"),
+
 									SrcRange: hcl.Range{
 										Start: hcl.Pos{Line: 1, Column: 13, Byte: 12},
 										End:   hcl.Pos{Line: 1, Column: 14, Byte: 13},
@@ -854,10 +894,18 @@ block "valid" {}
 						Expr: &TemplateExpr{
 							Parts: []Expression{
 								&LiteralValueExpr{
-									Val: cty.StringVal("hello $"),
+									Val: cty.StringVal("hello "),
 
 									SrcRange: hcl.Range{
 										Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
+										End:   hcl.Pos{Line: 1, Column: 12, Byte: 11},
+									},
+								},
+								&LiteralValueExpr{
+									Val: cty.StringVal("$"),
+
+									SrcRange: hcl.Range{
+										Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
 										End:   hcl.Pos{Line: 1, Column: 13, Byte: 12},
 									},
 								},
@@ -904,19 +952,27 @@ block "valid" {}
 						Expr: &TemplateExpr{
 							Parts: []Expression{
 								&LiteralValueExpr{
-									Val: cty.StringVal("hello %"),
+									Val: cty.StringVal("hello "),
 
 									SrcRange: hcl.Range{
 										Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
-										End:   hcl.Pos{Line: 1, Column: 13, Byte: 12},
+										End:   hcl.Pos{Line: 1, Column: 12, Byte: 11},
 									},
 								},
 								// This parses oddly due to how the scanner
-								// handles escaping of the $ sequence, but it's
+								// handles escaping of the % sequence, but it's
 								// functionally equivalent to a single literal.
 								&LiteralValueExpr{
 									Val: cty.StringVal("%"),
 
+									SrcRange: hcl.Range{
+										Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
+										End:   hcl.Pos{Line: 1, Column: 13, Byte: 12},
+									},
+								},
+								&LiteralValueExpr{
+									Val: cty.StringVal("%"),
+
 									SrcRange: hcl.Range{
 										Start: hcl.Pos{Line: 1, Column: 13, Byte: 12},
 										End:   hcl.Pos{Line: 1, Column: 14, Byte: 13},
@@ -965,10 +1021,18 @@ block "valid" {}
 						Expr: &TemplateExpr{
 							Parts: []Expression{
 								&LiteralValueExpr{
-									Val: cty.StringVal("hello %"),
+									Val: cty.StringVal("hello "),
 
 									SrcRange: hcl.Range{
 										Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
+										End:   hcl.Pos{Line: 1, Column: 12, Byte: 11},
+									},
+								},
+								&LiteralValueExpr{
+									Val: cty.StringVal("%"),
+
+									SrcRange: hcl.Range{
+										Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
 										End:   hcl.Pos{Line: 1, Column: 13, Byte: 12},
 									},
 								},
@@ -2285,6 +2349,7 @@ block "valid" {}
 
 	for _, test := range tests {
 		t.Run(test.input, func(t *testing.T) {
+			t.Logf("\n%s", test.input)
 			file, diags := ParseConfig([]byte(test.input), "", hcl.Pos{Byte: 0, Line: 1, Column: 1})
 			if len(diags) != test.diagCount {
 				t.Errorf("wrong number of diagnostics %d; want %d", len(diags), test.diagCount)

hcl/hclsyntax/scan_tokens.rl

@@ -218,29 +218,35 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To
         TemplateInterp = "${" ("~")?;
         TemplateControl = "%{" ("~")?;
         EndStringTmpl = '"';
-        StringLiteralChars = (AnyUTF8 - ("\r"|"\n"));
+        NewlineChars = ("\r"|"\n");
+        NewlineCharsSeq = NewlineChars+;
+        StringLiteralChars = (AnyUTF8 - NewlineChars);
+        TemplateIgnoredNonBrace = (^'{' %{ fhold; });
+        TemplateNotInterp = '$' (TemplateIgnoredNonBrace | TemplateInterp);
+        TemplateNotControl = '%' (TemplateIgnoredNonBrace | TemplateControl);
+        QuotedStringLiteralWithEsc = ('\\' StringLiteralChars) | (StringLiteralChars - ("$" | '%' | '"' | "\\"));
         TemplateStringLiteral = (
-            ('$' ^'{' %{ fhold; }) |
+            (TemplateNotInterp) |
-            ('%' ^'{' %{ fhold; }) |
+            (TemplateNotControl) |
-            ('\\' StringLiteralChars) |
+            (QuotedStringLiteralWithEsc)+
-            (StringLiteralChars - ("$" | '%' | '"'))
+        );
-        )+;
         HeredocStringLiteral = (
-            ('$' ^'{' %{ fhold; }) |
+            (TemplateNotInterp) |
-            ('%' ^'{' %{ fhold; }) |
+            (TemplateNotControl) |
-            (StringLiteralChars - ("$" | '%'))
+            (StringLiteralChars - ("$" | '%'))*
-        )*;
+        );
         BareStringLiteral = (
-            ('$' ^'{') |
+            (TemplateNotInterp) |
-            ('%' ^'{') |
+            (TemplateNotControl) |
-            (StringLiteralChars - ("$" | '%'))
+            (StringLiteralChars - ("$" | '%'))*
-        )* Newline?;
+        ) Newline?;
 
         stringTemplate := |*
             TemplateInterp        => beginTemplateInterp;
             TemplateControl       => beginTemplateControl;
             EndStringTmpl         => endStringTemplate;
             TemplateStringLiteral => { token(TokenQuotedLit); };
+            NewlineCharsSeq       => { token(TokenQuotedNewline); };
             AnyUTF8               => { token(TokenInvalid); };
             BrokenUTF8            => { token(TokenBadUTF8); };
         *|;

hcl/hclsyntax/scan_tokens_test.go

@@ -439,6 +439,43 @@ func TestScanTokens_normal(t *testing.T) {
 				},
 			},
 		},
+		{
+			`"hello, \"world\"!"`,
+			[]Token{
+				{
+					Type:  TokenOQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`hello, \"world\"!`), // The escapes are handled by the parser, not the scanner
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
+						End:   hcl.Pos{Byte: 18, Line: 1, Column: 19},
+					},
+				},
+				{
+					Type:  TokenCQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 18, Line: 1, Column: 19},
+						End:   hcl.Pos{Byte: 19, Line: 1, Column: 20},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 19, Line: 1, Column: 20},
+						End:   hcl.Pos{Byte: 19, Line: 1, Column: 20},
+					},
+				},
+			},
+		},
 		{
 			`"hello $$"`,
 			[]Token{
@@ -452,16 +489,24 @@ func TestScanTokens_normal(t *testing.T) {
 				},
 				{
 					Type:  TokenQuotedLit,
-					Bytes: []byte(`hello $`),
+					Bytes: []byte(`hello `),
 					Range: hcl.Range{
 						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
-						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
+						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
 					},
 				},
 				// This one scans a little oddly because of how the scanner
 				// handles the escaping of the dollar sign, but it's still
 				// good enough for the parser since it'll just concatenate
 				// these two string literals together anyway.
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`$`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
+						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
+					},
+				},
 				{
 					Type:  TokenQuotedLit,
 					Bytes: []byte(`$`),
@@ -501,16 +546,24 @@ func TestScanTokens_normal(t *testing.T) {
 				},
 				{
 					Type:  TokenQuotedLit,
-					Bytes: []byte(`hello %`),
+					Bytes: []byte(`hello `),
 					Range: hcl.Range{
 						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
-						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
+						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
 					},
 				},
 				// This one scans a little oddly because of how the scanner
-				// handles the escaping of the dollar sign, but it's still
+				// handles the escaping of the percent sign, but it's still
 				// good enough for the parser since it'll just concatenate
 				// these two string literals together anyway.
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`%`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
+						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
+					},
+				},
 				{
 					Type:  TokenQuotedLit,
 					Bytes: []byte(`%`),
@@ -550,9 +603,17 @@ func TestScanTokens_normal(t *testing.T) {
 				},
 				{
 					Type:  TokenQuotedLit,
-					Bytes: []byte(`hello $`),
+					Bytes: []byte(`hello `),
 					Range: hcl.Range{
 						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
+						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`$`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
 						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
 					},
 				},
@@ -587,9 +648,17 @@ func TestScanTokens_normal(t *testing.T) {
 				},
 				{
 					Type:  TokenQuotedLit,
-					Bytes: []byte(`hello %`),
+					Bytes: []byte(`hello `),
 					Range: hcl.Range{
 						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
+						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`%`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
 						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
 					},
 				},
@@ -611,6 +680,181 @@ func TestScanTokens_normal(t *testing.T) {
 				},
 			},
 		},
+		{
+			`"hello $${world}"`,
+			[]Token{
+				{
+					Type:  TokenOQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`hello `),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
+						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`$${`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
+						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`world}`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 10, Line: 1, Column: 11},
+						End:   hcl.Pos{Byte: 16, Line: 1, Column: 17},
+					},
+				},
+				{
+					Type:  TokenCQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 16, Line: 1, Column: 17},
+						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 17, Line: 1, Column: 18},
+						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
+					},
+				},
+			},
+		},
+		{
+			`"hello %%{world}"`,
+			[]Token{
+				{
+					Type:  TokenOQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`hello `),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
+						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`%%{`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
+						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`world}`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 10, Line: 1, Column: 11},
+						End:   hcl.Pos{Byte: 16, Line: 1, Column: 17},
+					},
+				},
+				{
+					Type:  TokenCQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 16, Line: 1, Column: 17},
+						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 17, Line: 1, Column: 18},
+						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
+					},
+				},
+			},
+		},
+		{
+			`"hello %${world}"`,
+			[]Token{
+				{
+					Type:  TokenOQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`hello `),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
+						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`%`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
+						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
+					},
+				},
+				{
+					Type:  TokenTemplateInterp,
+					Bytes: []byte(`${`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
+						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte(`world`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 10, Line: 1, Column: 11},
+						End:   hcl.Pos{Byte: 15, Line: 1, Column: 16},
+					},
+				},
+				{
+					Type:  TokenTemplateSeqEnd,
+					Bytes: []byte(`}`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 15, Line: 1, Column: 16},
+						End:   hcl.Pos{Byte: 16, Line: 1, Column: 17},
+					},
+				},
+				{
+					Type:  TokenCQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 16, Line: 1, Column: 17},
+						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 17, Line: 1, Column: 18},
+						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
+					},
+				},
+			},
+		},
 
 		// Templates with interpolations and control sequences
 		{
@@ -1735,6 +1979,157 @@ EOF
 				},
 			},
 		},
+
+		// Misc combinations that have come up in bug reports, etc.
+		{
+			"locals {\n  is_percent = percent_sign == \"%\" ? true : false\n}\n",
+			[]Token{
+				{
+					Type:  TokenIdent,
+					Bytes: []byte(`locals`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
+					},
+				},
+				{
+					Type:  TokenOBrace,
+					Bytes: []byte{'{'},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
+						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte{'\n'},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
+						End:   hcl.Pos{Byte: 9, Line: 2, Column: 1},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte(`is_percent`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 11, Line: 2, Column: 3},
+						End:   hcl.Pos{Byte: 21, Line: 2, Column: 13},
+					},
+				},
+				{
+					Type:  TokenEqual,
+					Bytes: []byte(`=`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 22, Line: 2, Column: 14},
+						End:   hcl.Pos{Byte: 23, Line: 2, Column: 15},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte(`percent_sign`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 24, Line: 2, Column: 16},
+						End:   hcl.Pos{Byte: 36, Line: 2, Column: 28},
+					},
+				},
+				{
+					Type:  TokenEqualOp,
+					Bytes: []byte(`==`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 37, Line: 2, Column: 29},
+						End:   hcl.Pos{Byte: 39, Line: 2, Column: 31},
+					},
+				},
+				{
+					Type:  TokenOQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 40, Line: 2, Column: 32},
+						End:   hcl.Pos{Byte: 41, Line: 2, Column: 33},
+					},
+				},
+				{
+					Type:  TokenQuotedLit,
+					Bytes: []byte(`%`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 41, Line: 2, Column: 33},
+						End:   hcl.Pos{Byte: 42, Line: 2, Column: 34},
+					},
+				},
+				{
+					Type:  TokenCQuote,
+					Bytes: []byte(`"`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 42, Line: 2, Column: 34},
+						End:   hcl.Pos{Byte: 43, Line: 2, Column: 35},
+					},
+				},
+				{
+					Type:  TokenQuestion,
+					Bytes: []byte(`?`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 44, Line: 2, Column: 36},
+						End:   hcl.Pos{Byte: 45, Line: 2, Column: 37},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte(`true`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 46, Line: 2, Column: 38},
+						End:   hcl.Pos{Byte: 50, Line: 2, Column: 42},
+					},
+				},
+				{
+					Type:  TokenColon,
+					Bytes: []byte(`:`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 51, Line: 2, Column: 43},
+						End:   hcl.Pos{Byte: 52, Line: 2, Column: 44},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte(`false`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 53, Line: 2, Column: 45},
+						End:   hcl.Pos{Byte: 58, Line: 2, Column: 50},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte{'\n'},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 58, Line: 2, Column: 50},
+						End:   hcl.Pos{Byte: 59, Line: 3, Column: 1},
+					},
+				},
+				{
+					Type:  TokenCBrace,
+					Bytes: []byte{'}'},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 59, Line: 3, Column: 1},
+						End:   hcl.Pos{Byte: 60, Line: 3, Column: 2},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte{'\n'},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 60, Line: 3, Column: 2},
+						End:   hcl.Pos{Byte: 61, Line: 4, Column: 1},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 61, Line: 4, Column: 1},
+						End:   hcl.Pos{Byte: 61, Line: 4, Column: 1},
+					},
+				},
+			},
+		},
 	}
 
 	prettyConfig := &pretty.Config{

hcl/hclsyntax/token.go

@@ -85,17 +85,18 @@ const (
 	// things that might work in other languages they are familiar with, or
 	// simply make incorrect assumptions about the HCL language.
 
-	TokenBitwiseAnd TokenType = '&'
+	TokenBitwiseAnd    TokenType = '&'
-	TokenBitwiseOr  TokenType = '|'
+	TokenBitwiseOr     TokenType = '|'
-	TokenBitwiseNot TokenType = '~'
+	TokenBitwiseNot    TokenType = '~'
-	TokenBitwiseXor TokenType = '^'
+	TokenBitwiseXor    TokenType = '^'
-	TokenStarStar   TokenType = '➚'
+	TokenStarStar      TokenType = '➚'
-	TokenApostrophe TokenType = '\''
+	TokenApostrophe    TokenType = '\''
-	TokenBacktick   TokenType = '`'
+	TokenBacktick      TokenType = '`'
-	TokenSemicolon  TokenType = ';'
+	TokenSemicolon     TokenType = ';'
-	TokenTabs       TokenType = '␉'
+	TokenTabs          TokenType = '␉'
-	TokenInvalid    TokenType = '<27>'
+	TokenInvalid       TokenType = '<27>'
-	TokenBadUTF8    TokenType = '💩'
+	TokenBadUTF8       TokenType = '💩'
+	TokenQuotedNewline TokenType = '␤'
 
 	// TokenNil is a placeholder for when a token is required but none is
 	// available, e.g. when reporting errors. The scanner will never produce
@@ -285,6 +286,13 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics {
 
 				toldBadUTF8++
 			}
+		case TokenQuotedNewline:
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid multi-line string",
+				Detail:   "Quoted strings may not be split over multiple lines. To produce a multi-line string, either use the \\n escape to represent a newline character or use the \"heredoc\" multi-line template syntax.",
+				Subject:  &tok.Range,
+			})
 		case TokenInvalid:
 			diags = append(diags, &hcl.Diagnostic{
 				Severity: hcl.DiagError,

hcl/hclsyntax/token_type_string.go

@@ -4,7 +4,67 @@ package hclsyntax
 
 import "strconv"
 
-const _TokenType_name = "TokenNilTokenNewlineTokenBangTokenPercentTokenBitwiseAndTokenApostropheTokenOParenTokenCParenTokenStarTokenPlusTokenCommaTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenCommentTokenOHeredocTokenIdentTokenNumberLitTokenQuotedLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenEllipsisTokenFatArrowTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenEqualOpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenStarStarTokenInvalidTokenBadUTF8"
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[TokenOBrace-123]
+	_ = x[TokenCBrace-125]
+	_ = x[TokenOBrack-91]
+	_ = x[TokenCBrack-93]
+	_ = x[TokenOParen-40]
+	_ = x[TokenCParen-41]
+	_ = x[TokenOQuote-171]
+	_ = x[TokenCQuote-187]
+	_ = x[TokenOHeredoc-72]
+	_ = x[TokenCHeredoc-104]
+	_ = x[TokenStar-42]
+	_ = x[TokenSlash-47]
+	_ = x[TokenPlus-43]
+	_ = x[TokenMinus-45]
+	_ = x[TokenPercent-37]
+	_ = x[TokenEqual-61]
+	_ = x[TokenEqualOp-8788]
+	_ = x[TokenNotEqual-8800]
+	_ = x[TokenLessThan-60]
+	_ = x[TokenLessThanEq-8804]
+	_ = x[TokenGreaterThan-62]
+	_ = x[TokenGreaterThanEq-8805]
+	_ = x[TokenAnd-8743]
+	_ = x[TokenOr-8744]
+	_ = x[TokenBang-33]
+	_ = x[TokenDot-46]
+	_ = x[TokenComma-44]
+	_ = x[TokenEllipsis-8230]
+	_ = x[TokenFatArrow-8658]
+	_ = x[TokenQuestion-63]
+	_ = x[TokenColon-58]
+	_ = x[TokenTemplateInterp-8747]
+	_ = x[TokenTemplateControl-955]
+	_ = x[TokenTemplateSeqEnd-8718]
+	_ = x[TokenQuotedLit-81]
+	_ = x[TokenStringLit-83]
+	_ = x[TokenNumberLit-78]
+	_ = x[TokenIdent-73]
+	_ = x[TokenComment-67]
+	_ = x[TokenNewline-10]
+	_ = x[TokenEOF-9220]
+	_ = x[TokenBitwiseAnd-38]
+	_ = x[TokenBitwiseOr-124]
+	_ = x[TokenBitwiseNot-126]
+	_ = x[TokenBitwiseXor-94]
+	_ = x[TokenStarStar-10138]
+	_ = x[TokenApostrophe-39]
+	_ = x[TokenBacktick-96]
+	_ = x[TokenSemicolon-59]
+	_ = x[TokenTabs-9225]
+	_ = x[TokenInvalid-65533]
+	_ = x[TokenBadUTF8-128169]
+	_ = x[TokenQuotedNewline-9252]
+	_ = x[TokenNil-0]
+}
+
+const _TokenType_name = "TokenNilTokenNewlineTokenBangTokenPercentTokenBitwiseAndTokenApostropheTokenOParenTokenCParenTokenStarTokenPlusTokenCommaTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenCommentTokenOHeredocTokenIdentTokenNumberLitTokenQuotedLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenEllipsisTokenFatArrowTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenEqualOpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenQuotedNewlineTokenStarStarTokenInvalidTokenBadUTF8"
 
 var _TokenType_map = map[TokenType]string{
 	0:      _TokenType_name[0:8],
@@ -57,9 +117,10 @@ var _TokenType_map = map[TokenType]string{
 	8805:   _TokenType_name[577:595],
 	9220:   _TokenType_name[595:603],
 	9225:   _TokenType_name[603:612],
-	10138:  _TokenType_name[612:625],
+	9252:   _TokenType_name[612:630],
-	65533:  _TokenType_name[625:637],
+	10138:  _TokenType_name[630:643],
-	128169: _TokenType_name[637:649],
+	65533:  _TokenType_name[643:655],
+	128169: _TokenType_name[655:667],
 }
 
 func (i TokenType) String() string {