scanner: add string test for token type

scanner/token.go

@@ -67,10 +67,6 @@ var tokens = [...]string{
 }
 
 // String returns the string corresponding to the token tok.
-// For operators, delimiters, and keywords the string is the actual
-// token character sequence (e.g., for the token ADD, the string is
-// "+"). For all other tokens the string corresponds to the token
-// constant name (e.g. for the token IDENT, the string is "IDENT").
 func (t TokenType) String() string {
 	s := ""
 	if 0 <= t && t < TokenType(len(tokens)) {

scanner/token_test.go

@@ -0,0 +1,36 @@
+package scanner
+
+import "testing"
+
+func TestTokenTypeString(t *testing.T) {
+	var tokens = []struct {
+		tt  TokenType
+		str string
+	}{
+		{ILLEGAL, "ILLEGAL"},
+		{EOF, "EOF"},
+		{COMMENT, "COMMENT"},
+		{IDENT, "IDENT"},
+		{NUMBER, "NUMBER"},
+		{FLOAT, "FLOAT"},
+		{BOOL, "BOOL"},
+		{STRING, "STRING"},
+		{LBRACK, "LBRACK"},
+		{LBRACE, "LBRACE"},
+		{COMMA, "COMMA"},
+		{PERIOD, "PERIOD"},
+		{RBRACK, "RBRACK"},
+		{RBRACE, "RBRACE"},
+		{ASSIGN, "ASSIGN"},
+		{ADD, "ADD"},
+		{SUB, "SUB"},
+	}
+
+	for _, token := range tokens {
+		if token.tt.String() != token.str {
+			t.Errorf("want: %q got:%q\n", token.str, token.tt)
+
+		}
+	}
+
+}