hclsyntax: ValidIdentifier function

Calling applications often need to validate strings provided by the user
that will eventually be variable or attribute names in the evaluation
scope, to ensure that they will be evaluable.

Rather than having each application specify its own different subset of
the full set we support (which is derived from Unicode specifications),
we provide a simple function to let callers easily check the validity
of a potential identifier using exactly the same scanning rules we use
within the expression scanner.

To achieve this we actually invoke the scanner and then assert on its
result, which is a pretty expensive way to just check one string but it's
easy to do with code we already have in place and we don't expect this
sort of validation to be going on in a tight loop.

hcl/hclsyntax/public.go

@@ -128,3 +128,17 @@ func LexTemplate(src []byte, filename string, start hcl.Pos) (Tokens, hcl.Diagno
 	diags := checkInvalidTokens(tokens)
 	return tokens, diags
 }
+
+// ValidIdentifier tests if the given string could be a valid identifier in
+// a native syntax expression.
+//
+// This is useful when accepting names from the user that will be used as
+// variable or attribute names in the scope, to ensure that any name chosen
+// will be traversable using the variable or attribute traversal syntax.
+func ValidIdentifier(s string) bool {
+	// This is a kinda-expensive way to do something pretty simple, but it
+	// is easiest to do with our existing scanner-related infrastructure here
+	// and nobody should be validating identifiers in a tight loop.
+	tokens := scanTokens([]byte(s), "", hcl.Pos{}, scanIdentOnly)
+	return len(tokens) == 2 && tokens[0].Type == TokenIdent && tokens[1].Type == TokenEOF
+}

hcl/hclsyntax/public_test.go

@@ -0,0 +1,45 @@
+package hclsyntax
+
+import (
+	"testing"
+)
+
+func TestValidIdentifier(t *testing.T) {
+	tests := []struct {
+		Input string
+		Want  bool
+	}{
+		{"", false},
+		{"hello", true},
+		{"hello.world", false},
+		{"hello ", false},
+		{" hello", false},
+		{"hello\n", false},
+		{"hello world", false},
+		{"aws_instance", true},
+		{"foo-bar", true},
+		{"foo--bar", true},
+		{"foo_", true},
+		{"foo-", true},
+		{"_foobar", false},
+		{"-foobar", false},
+		{"blah1", true},
+		{"blah1blah", true},
+		{"1blah1blah", false},
+		{"héllo", true}, // combining acute accent
+		{"Χαίρετε", true},
+		{"звать", true},
+		{"今日は", true},
+		{"\x80", false},  // UTF-8 continuation without an introducer
+		{"a\x80", false}, // UTF-8 continuation after a non-introducer
+	}
+
+	for _, test := range tests {
+		t.Run(test.Input, func(t *testing.T) {
+			got := ValidIdentifier(test.Input)
+			if got != test.Want {
+				t.Errorf("wrong result %#v; want %#v", got, test.Want)
+			}
+		})
+	}
+}

hcl/hclsyntax/scan_tokens.rl

@@ -236,6 +236,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To
             BrokenUTF8            => { token(TokenBadUTF8); };
         *|;
 
+        identOnly := |*
+            Ident            => { token(TokenIdent) };
+            BrokenUTF8       => { token(TokenBadUTF8) };
+            AnyUTF8          => { token(TokenInvalid) };
+        *|;
+
         main := |*
             Spaces           => {};
             NumberLit        => { token(TokenNumberLit) };
@@ -284,6 +290,8 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To
         cs = hcltok_en_main
     case scanTemplate:
         cs = hcltok_en_bareTemplate
+    case scanIdentOnly:
+        cs = hcltok_en_identOnly
     default:
         panic("invalid scanMode")
     }

hcl/hclsyntax/token.go

@@ -110,6 +110,7 @@ type scanMode int
 const (
 	scanNormal scanMode = iota
 	scanTemplate
+	scanIdentOnly
 )
 
 type tokenAccum struct {