parser: add scanning ident test

parser/lexer.go

@@ -72,10 +72,12 @@ func (l *Lexer) scanIdentifier() (Token, string) {
 	// Create a buffer and read the current character into it.
 	var buf bytes.Buffer
 
-	// write current character before we move to the next
-	buf.WriteRune(l.ch)
+	for isLetter(l.ch) || isDigit(l.ch) {
+		buf.WriteRune(l.ch)
+		l.next()
+	}
 
-	return 0, ""
+	return IDENT, buf.String()
 }
 
 // Pos returns the position of the character immediately after the character or

parser/lexer_test.go

@@ -0,0 +1,53 @@
+package parser
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+)
+
+type token struct {
+	tok  Token
+	text string
+}
+
+func TestIdent(t *testing.T) {
+	var identList = []token{
+		{IDENT, "a"},
+		{IDENT, "a0"},
+		{IDENT, "foobar"},
+		{IDENT, "abc123"},
+		{IDENT, "LGTM"},
+		{IDENT, "_"},
+		{IDENT, "_abc123"},
+		{IDENT, "abc123_"},
+		{IDENT, "_abc_123_"},
+		{IDENT, "_äöü"},
+		{IDENT, "_本"},
+		{IDENT, "äöü"},
+		{IDENT, "本"},
+		{IDENT, "a۰۱۸"},
+		{IDENT, "foo६४"},
+		{IDENT, "bar９８７６"},
+	}
+
+	// create artifical source code
+	buf := new(bytes.Buffer)
+	for _, ident := range identList {
+		fmt.Fprintf(buf, " \t%s\n", ident.text)
+	}
+
+	l := NewLexer(buf)
+
+	for _, ident := range identList {
+		tok, lit := l.Scan()
+		if tok != ident.tok {
+			t.Errorf("tok = %s want %s for %s\n", tok, ident.tok, ident.text)
+		}
+
+		if lit != ident.text {
+			t.Errorf("text = %s want %s", lit, ident.text)
+		}
+
+	}
+}