hclsyntax: allow underscore at the start of identifiers

We are leaning on the unicode identifier definitions here, but the
specified ID_Start does not include the underscore character and users
seem to expect this to be allowed due to experience with other languages.

Since allowing a leading underscore introduces no ambiguity, we'll allow
it. Calling applications may choose to reject it if they'd rather not have
such weird names.

hcl/hclsyntax/public_test.go

@@ -17,11 +17,12 @@ func TestValidIdentifier(t *testing.T) {
 		{"hello\n", false},
 		{"hello world", false},
 		{"aws_instance", true},
+		{"aws.instance", false},
 		{"foo-bar", true},
 		{"foo--bar", true},
 		{"foo_", true},
 		{"foo-", true},
-		{"_foobar", false},
+		{"_foobar", true},
 		{"-foobar", false},
 		{"blah1", true},
 		{"blah1blah", true},

hcl/hclsyntax/scan_tokens.rl

@@ -36,7 +36,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To
 
         NumberLitContinue = (digit|'.'|('e'|'E') ('+'|'-')? digit);
         NumberLit = digit ("" | (NumberLitContinue - '.') | (NumberLitContinue* (NumberLitContinue - '.')));
-        Ident = ID_Start (ID_Continue | '-')*;
+        Ident = (ID_Start | '_') (ID_Continue | '-')*;
 
         # Symbols that just represent themselves are handled as a single rule.
         SelfToken = "[" | "]" | "(" | ")" | "." | "," | "*" | "/" | "%" | "+" | "-" | "=" | "<" | ">" | "!" | "?" | ":" | "\n" | "&" | "|" | "~" | "^" | ";" | "`";

hcl/hclsyntax/scan_tokens_test.go

@@ -199,6 +199,69 @@ func TestScanTokens_normal(t *testing.T) {
 				},
 			},
 		},
+		{
+			`_ello`,
+			[]Token{
+				{
+					Type:  TokenIdent,
+					Bytes: []byte(`_ello`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
+						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
+					},
+				},
+			},
+		},
+		{
+			`hel_o`,
+			[]Token{
+				{
+					Type:  TokenIdent,
+					Bytes: []byte(`hel_o`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
+						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
+					},
+				},
+			},
+		},
+		{
+			`hel-o`,
+			[]Token{
+				{
+					Type:  TokenIdent,
+					Bytes: []byte(`hel-o`),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
+						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
+					},
+				},
+			},
+		},
 		{
 			`h3ll0`,
 			[]Token{