zclsyntax: public interface to the scanner

This LexConfig, LexExpression and LexTemplate set of functions allow
outside callers to use the scanner in isolation, skipping the parser.
This may be useful for use-cases such as syntax highlighting, separate
parsers (such as the one in zclwrite), and so forth. Most callers should
use the parser (once implemented) though, to get a semantic AST.

zcl/zclsyntax/public.go

@@ -0,0 +1,25 @@
+package zclsyntax
+
+import (
+	"github.com/zclconf/go-zcl/zcl"
+)
+
+// LexConfig performs lexical analysis on the given buffer, treating it as a
+// whole zcl config file, and returns the resulting tokens.
+func LexConfig(src []byte, filename string, start zcl.Pos) Tokens {
+	return scanTokens(src, filename, start, scanNormal)
+}
+
+// LexExpression performs lexical analysis on the given buffer, treating it as
+// a standalone zcl expression, and returns the resulting tokens.
+func LexExpression(src []byte, filename string, start zcl.Pos) Tokens {
+	// This is actually just the same thing as LexConfig, since configs
+	// and expressions lex in the same way.
+	return scanTokens(src, filename, start, scanNormal)
+}
+
+// LexTemplate performs lexical analysis on the given buffer, treating it as a
+// standalone zcl template, and returns the resulting tokens.
+func LexTemplate(src []byte, filename string, start zcl.Pos) Tokens {
+	return scanTokens(src, filename, start, scanTemplate)
+}

zcl/zclsyntax/token.go

@@ -15,6 +15,9 @@ type Token struct {
 	Range zcl.Range
 }
 
+// Tokens is a slice of Token.
+type Tokens []Token
+
 // TokenType is an enumeration used for the Type field on Token.
 type TokenType rune