hclsyntax: Tailored error for "curly quotes"

It seems to be somewhat common for someone to share HCL code via a forum
or a document and have the well-meaning word processor or CMS replace the
straight quotes with curly quotes, which then lead to confusing errors
when someone copies the result and tries to use it as valid HCL
configuration.

Here we add a special hint for that, giving a tailored error message
instead of the generic "This character is not used within the language"
error message.

HCL has always had some of these special hints implemented here, and they
were originally implemented with special token types to allow the parser
handle them. However, we later refactored to do the check all at once
inside the Lex* family of functions, prior to parsing, so it's now
relatively straightforward to handle it as a special case of TokenInvalid
rather than an entirely new token type. Perhaps later we'll rework the
existing ones to also just use TokenInvalid, but that's a decision for
another day.

hclsyntax/token.go

@@ -294,12 +294,23 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics {
 				Subject:  &tok.Range,
 			})
 		case TokenInvalid:
-			diags = append(diags, &hcl.Diagnostic{
-				Severity: hcl.DiagError,
-				Summary:  "Invalid character",
-				Detail:   "This character is not used within the language.",
-				Subject:  &tok.Range,
-			})
+			chars := string(tok.Bytes)
+			switch chars {
+			case "“", "”":
+				diags = append(diags, &hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid character",
+					Detail:   "\"Curly quotes\" are not valid here. These can sometimes be inadvertently introduced when sharing code via documents or discussion forums. It might help to replace the character with a \"straight quote\".",
+					Subject:  &tok.Range,
+				})
+			default:
+				diags = append(diags, &hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid character",
+					Detail:   "This character is not used within the language.",
+					Subject:  &tok.Range,
+				})
+			}
 		}
 	}
 	return diags

hclsyntax/token_test.go

@@ -0,0 +1,60 @@
+package hclsyntax
+
+import (
+	"testing"
+
+	"github.com/hashicorp/hcl/v2"
+)
+
+func TestCheckInvalidTokensTest(t *testing.T) {
+	tests := []struct {
+		Input       string
+		WantSummary string
+		WantDetail  string
+	}{
+		{
+			`block “invalid” {}`,
+			`Invalid character`,
+			`"Curly quotes" are not valid here. These can sometimes be inadvertently introduced when sharing code via documents or discussion forums. It might help to replace the character with a "straight quote".`,
+		},
+		{
+			`block 'invalid' {}`,
+			`Invalid character`,
+			`Single quotes are not valid. Use double quotes (") to enclose strings.`,
+		},
+		{
+			"block `invalid` {}",
+			`Invalid character`,
+			"The \"`\" character is not valid. To create a multi-line string, use the \"heredoc\" syntax, like \"<<EOT\".",
+		},
+		{
+			`foo = a & b`,
+			`Unsupported operator`,
+			`Bitwise operators are not supported. Did you mean boolean AND ("&&")?`,
+		},
+		{
+			`foo = a | b`,
+			`Unsupported operator`,
+			`Bitwise operators are not supported. Did you mean boolean OR ("||")?`,
+		},
+		{
+			`foo = ~a`,
+			`Unsupported operator`,
+			`Bitwise operators are not supported. Did you mean boolean NOT ("!")?`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Input, func(t *testing.T) {
+			_, diags := LexConfig([]byte(test.Input), "", hcl.Pos{Line: 1, Column: 1})
+			for _, diag := range diags {
+				if diag.Severity == hcl.DiagError && diag.Summary == test.WantSummary && diag.Detail == test.WantDetail {
+					return // success!
+				}
+			}
+			// If we fall out here then we didn't find the diagnostic we were
+			// looking for.
+			t.Errorf("wrong errors\ngot:  %s\nwant: %s; %s", diags.Error(), test.WantSummary, test.WantDetail)
+		})
+	}
+}