diff --git a/parser/lexer.go b/parser/lexer.go index 98e7165..be542f9 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -72,10 +72,12 @@ func (l *Lexer) scanIdentifier() (Token, string) { // Create a buffer and read the current character into it. var buf bytes.Buffer - // write current character before we move to the next - buf.WriteRune(l.ch) + for isLetter(l.ch) || isDigit(l.ch) { + buf.WriteRune(l.ch) + l.next() + } - return 0, "" + return IDENT, buf.String() } // Pos returns the position of the character immediately after the character or diff --git a/parser/lexer_test.go b/parser/lexer_test.go new file mode 100644 index 0000000..687dffa --- /dev/null +++ b/parser/lexer_test.go @@ -0,0 +1,53 @@ +package parser + +import ( + "bytes" + "fmt" + "testing" +) + +type token struct { + tok Token + text string +} + +func TestIdent(t *testing.T) { + var identList = []token{ + {IDENT, "a"}, + {IDENT, "a0"}, + {IDENT, "foobar"}, + {IDENT, "abc123"}, + {IDENT, "LGTM"}, + {IDENT, "_"}, + {IDENT, "_abc123"}, + {IDENT, "abc123_"}, + {IDENT, "_abc_123_"}, + {IDENT, "_äöü"}, + {IDENT, "_本"}, + {IDENT, "äöü"}, + {IDENT, "本"}, + {IDENT, "a۰۱۸"}, + {IDENT, "foo६४"}, + {IDENT, "bar9876"}, + } + + // create artifical source code + buf := new(bytes.Buffer) + for _, ident := range identList { + fmt.Fprintf(buf, " \t%s\n", ident.text) + } + + l := NewLexer(buf) + + for _, ident := range identList { + tok, lit := l.Scan() + if tok != ident.tok { + t.Errorf("tok = %s want %s for %s\n", tok, ident.tok, ident.text) + } + + if lit != ident.text { + t.Errorf("text = %s want %s", lit, ident.text) + } + + } +}