From 1b6ead19c22066ce650a25c4d2581e530b3ccb83 Mon Sep 17 00:00:00 2001
From: Fatih Arslan <ftharsln@gmail.com>
Date: Sat, 3 Oct 2015 21:06:30 +0300
Subject: [PATCH] parser: add scanning ident test
---
parser/lexer.go | 8 ++++---
parser/lexer_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 58 insertions(+), 3 deletions(-)
create mode 100644 parser/lexer_test.go
diff --git a/parser/lexer.go b/parser/lexer.go
index 98e7165..be542f9 100644
--- a/parser/lexer.go
+++ b/parser/lexer.go
@@ -72,10 +72,12 @@ func (l *Lexer) scanIdentifier() (Token, string) {
// Create a buffer and read the current character into it.
var buf bytes.Buffer
- // write current character before we move to the next
- buf.WriteRune(l.ch)
+ for isLetter(l.ch) || isDigit(l.ch) {
+ buf.WriteRune(l.ch)
+ l.next()
+ }
- return 0, ""
+ return IDENT, buf.String()
}
// Pos returns the position of the character immediately after the character or
diff --git a/parser/lexer_test.go b/parser/lexer_test.go
new file mode 100644
index 0000000..687dffa
--- /dev/null
+++ b/parser/lexer_test.go
@@ -0,0 +1,53 @@
+package parser
+
+import (
+ "bytes"
+ "fmt"
+ "testing"
+)
+
+type token struct {
+ tok Token
+ text string
+}
+
+func TestIdent(t *testing.T) {
+ var identList = []token{
+ {IDENT, "a"},
+ {IDENT, "a0"},
+ {IDENT, "foobar"},
+ {IDENT, "abc123"},
+ {IDENT, "LGTM"},
+ {IDENT, "_"},
+ {IDENT, "_abc123"},
+ {IDENT, "abc123_"},
+ {IDENT, "_abc_123_"},
+ {IDENT, "_äöü"},
+ {IDENT, "_本"},
+ {IDENT, "äöü"},
+ {IDENT, "本"},
+ {IDENT, "a۰۱۸"},
+ {IDENT, "foo६४"},
+ {IDENT, "bar９８７６"},
+ }
+
+ // create artifical source code
+ buf := new(bytes.Buffer)
+ for _, ident := range identList {
+ fmt.Fprintf(buf, " \t%s\n", ident.text)
+ }
+
+ l := NewLexer(buf)
+
+ for _, ident := range identList {
+ tok, lit := l.Scan()
+ if tok != ident.tok {
+ t.Errorf("tok = %s want %s for %s\n", tok, ident.tok, ident.text)
+ }
+
+ if lit != ident.text {
+ t.Errorf("text = %s want %s", lit, ident.text)
+ }
+
+ }
+}