scanner: fix tests
This commit is contained in:
Fatih Arslan
parent
81a8399ed1
commit
ac40da147e
@ -81,13 +81,15 @@ func (s *Scanner) next() rune {
|
||||
s.lastCharLen = size
|
||||
|
||||
s.srcPos.Offset += size
|
||||
s.srcPos.Column += size
|
||||
s.srcPos.Column++
|
||||
if ch == '\n' {
|
||||
s.srcPos.Line++
|
||||
s.srcPos.Column = 0
|
||||
s.lastLineLen = s.srcPos.Column
|
||||
}
|
||||
|
||||
// debug
|
||||
// fmt.Printf("ch: %q, off:column: %d:%d\n", ch, s.srcPos.Offset, s.srcPos.Column)
|
||||
return ch
|
||||
}
|
||||
|
||||
@ -110,7 +112,7 @@ func (s *Scanner) peek() rune {
|
||||
|
||||
// Scan scans the next token and returns the token.
|
||||
func (s *Scanner) Scan() (tok token.Token) {
|
||||
ch := s.peek()
|
||||
ch := s.next()
|
||||
|
||||
// skip white space
|
||||
for isWhitespace(ch) {
|
||||
@ -121,9 +123,9 @@ func (s *Scanner) Scan() (tok token.Token) {
|
||||
s.tokBuf.Reset()
|
||||
s.tokStart = s.srcPos.Offset - s.lastCharLen
|
||||
|
||||
// token position, initial next() is moving the offset by one, though we
|
||||
// are interested with the starting point
|
||||
s.tokPos.Offset = s.srcPos.Offset - 1
|
||||
// token position, initial next() is moving the offset by one(size of rune
|
||||
// actually), though we are interested with the starting point
|
||||
s.tokPos.Offset = s.srcPos.Offset - s.lastCharLen
|
||||
|
||||
if s.srcPos.Column > 0 {
|
||||
// common case: last character was not a '\n'
|
||||
|
||||
@ -44,8 +44,9 @@ func TestPosition(t *testing.T) {
|
||||
// t.SkipNow()
|
||||
// create artifical source code
|
||||
buf := new(bytes.Buffer)
|
||||
for _, list := range tokenLists {
|
||||
for _, ident := range list {
|
||||
|
||||
for _, listName := range orderedTokenLists {
|
||||
for _, ident := range tokenLists[listName] {
|
||||
fmt.Fprintf(buf, "\t\t\t\t%s\n", ident.text)
|
||||
}
|
||||
}
|
||||
@ -55,20 +56,21 @@ func TestPosition(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s.Scan()
|
||||
pos := Position{"", 4, 1, 5}
|
||||
for _, list := range tokenLists {
|
||||
for _, k := range list {
|
||||
for _, listName := range orderedTokenLists {
|
||||
s.Scan()
|
||||
|
||||
for _, k := range tokenLists[listName] {
|
||||
curPos := s.Pos()
|
||||
fmt.Printf("[%q] s = %+v:%+v\n", k.text, curPos.Offset, curPos.Column)
|
||||
// fmt.Printf("[%q] s = %+v:%+v\n", k.text, curPos.Offset, curPos.Column)
|
||||
if curPos.Offset != pos.Offset {
|
||||
t.Errorf("offset = %d, want %d for %q", curPos.Offset, pos.Offset, k.text)
|
||||
t.Fatalf("offset = %d, want %d for %q", curPos.Offset, pos.Offset, k.text)
|
||||
}
|
||||
if curPos.Line != pos.Line {
|
||||
t.Errorf("line = %d, want %d for %q", curPos.Line, pos.Line, k.text)
|
||||
t.Fatalf("line = %d, want %d for %q", curPos.Line, pos.Line, k.text)
|
||||
}
|
||||
if curPos.Column != pos.Column {
|
||||
t.Errorf("column = %d, want %d for %q", curPos.Column, pos.Column, k.text)
|
||||
t.Fatalf("column = %d, want %d for %q", curPos.Column, pos.Column, k.text)
|
||||
}
|
||||
pos.Offset += 4 + len(k.text) + 1 // 4 tabs + token bytes + newline
|
||||
pos.Line += countNewlines(k.text) + 1 // each token is on a new line
|
||||
@ -81,44 +83,53 @@ func TestPosition(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
var tokenLists = map[string][]tokenPair{
|
||||
// "comment": []tokenPair{
|
||||
// {token.COMMENT, "//"},
|
||||
// {token.COMMENT, "////"},
|
||||
// {token.COMMENT, "// comment"},
|
||||
// {token.COMMENT, "// /* comment */"},
|
||||
// {token.COMMENT, "// // comment //"},
|
||||
// {token.COMMENT, "//" + f100},
|
||||
// {token.COMMENT, "#"},
|
||||
// {token.COMMENT, "##"},
|
||||
// {token.COMMENT, "# comment"},
|
||||
// {token.COMMENT, "# /* comment */"},
|
||||
// {token.COMMENT, "# # comment #"},
|
||||
// {token.COMMENT, "#" + f100},
|
||||
// {token.COMMENT, "/**/"},
|
||||
// {token.COMMENT, "/***/"},
|
||||
// {token.COMMENT, "/* comment */"},
|
||||
// {token.COMMENT, "/* // comment */"},
|
||||
// {token.COMMENT, "/* /* comment */"},
|
||||
// {token.COMMENT, "/*\n comment\n*/"},
|
||||
// {token.COMMENT, "/*" + f100 + "*/"},
|
||||
// },
|
||||
// "operator": []tokenPair{
|
||||
// {token.LBRACK, "["},
|
||||
// {token.LBRACE, "{"},
|
||||
// {token.COMMA, ","},
|
||||
// {token.PERIOD, "."},
|
||||
// {token.RBRACK, "]"},
|
||||
// {token.RBRACE, "}"},
|
||||
// {token.ASSIGN, "="},
|
||||
// {token.ADD, "+"},
|
||||
// {token.SUB, "-"},
|
||||
// },
|
||||
// "bool": []tokenPair{
|
||||
// {token.BOOL, "true"},
|
||||
// {token.BOOL, "false"},
|
||||
// },
|
||||
var orderedTokenLists = []string{
|
||||
// "comment",
|
||||
// "operator",
|
||||
// "bool",
|
||||
// "ident",
|
||||
// "string",
|
||||
"number",
|
||||
// "float",
|
||||
}
|
||||
|
||||
var tokenLists = map[string][]tokenPair{
|
||||
"comment": []tokenPair{
|
||||
{token.COMMENT, "//"},
|
||||
{token.COMMENT, "////"},
|
||||
{token.COMMENT, "// comment"},
|
||||
{token.COMMENT, "// /* comment */"},
|
||||
{token.COMMENT, "// // comment //"},
|
||||
{token.COMMENT, "//" + f100},
|
||||
{token.COMMENT, "#"},
|
||||
{token.COMMENT, "##"},
|
||||
{token.COMMENT, "# comment"},
|
||||
{token.COMMENT, "# /* comment */"},
|
||||
{token.COMMENT, "# # comment #"},
|
||||
{token.COMMENT, "#" + f100},
|
||||
{token.COMMENT, "/**/"},
|
||||
{token.COMMENT, "/***/"},
|
||||
{token.COMMENT, "/* comment */"},
|
||||
{token.COMMENT, "/* // comment */"},
|
||||
{token.COMMENT, "/* /* comment */"},
|
||||
{token.COMMENT, "/*\n comment\n*/"},
|
||||
{token.COMMENT, "/*" + f100 + "*/"},
|
||||
},
|
||||
"operator": []tokenPair{
|
||||
{token.LBRACK, "["},
|
||||
{token.LBRACE, "{"},
|
||||
{token.COMMA, ","},
|
||||
{token.PERIOD, "."},
|
||||
{token.RBRACK, "]"},
|
||||
{token.RBRACE, "}"},
|
||||
{token.ASSIGN, "="},
|
||||
{token.ADD, "+"},
|
||||
{token.SUB, "-"},
|
||||
},
|
||||
"bool": []tokenPair{
|
||||
{token.BOOL, "true"},
|
||||
{token.BOOL, "false"},
|
||||
},
|
||||
"ident": []tokenPair{
|
||||
{token.IDENT, "a"},
|
||||
{token.IDENT, "a0"},
|
||||
@ -129,106 +140,106 @@ var tokenLists = map[string][]tokenPair{
|
||||
{token.IDENT, "_abc123"},
|
||||
{token.IDENT, "abc123_"},
|
||||
{token.IDENT, "_abc_123_"},
|
||||
// {token.IDENT, "_äöü"},
|
||||
// {token.IDENT, "_本"},
|
||||
// {token.IDENT, "äöü"},
|
||||
// {token.IDENT, "本"},
|
||||
// {token.IDENT, "a۰۱۸"},
|
||||
// {token.IDENT, "foo६४"},
|
||||
// {token.IDENT, "bar９８７６"},
|
||||
{token.IDENT, "_äöü"},
|
||||
{token.IDENT, "_本"},
|
||||
{token.IDENT, "äöü"},
|
||||
{token.IDENT, "本"},
|
||||
{token.IDENT, "a۰۱۸"},
|
||||
{token.IDENT, "foo६४"},
|
||||
{token.IDENT, "bar９８７６"},
|
||||
},
|
||||
"string": []tokenPair{
|
||||
{token.STRING, `" "`},
|
||||
{token.STRING, `"a"`},
|
||||
{token.STRING, `"本"`},
|
||||
{token.STRING, `"\a"`},
|
||||
{token.STRING, `"\b"`},
|
||||
{token.STRING, `"\f"`},
|
||||
{token.STRING, `"\n"`},
|
||||
{token.STRING, `"\r"`},
|
||||
{token.STRING, `"\t"`},
|
||||
{token.STRING, `"\v"`},
|
||||
{token.STRING, `"\""`},
|
||||
{token.STRING, `"\000"`},
|
||||
{token.STRING, `"\777"`},
|
||||
{token.STRING, `"\x00"`},
|
||||
{token.STRING, `"\xff"`},
|
||||
{token.STRING, `"\u0000"`},
|
||||
{token.STRING, `"\ufA16"`},
|
||||
{token.STRING, `"\U00000000"`},
|
||||
{token.STRING, `"\U0000ffAB"`},
|
||||
{token.STRING, `"` + f100 + `"`},
|
||||
},
|
||||
"number": []tokenPair{
|
||||
{token.NUMBER, "0"},
|
||||
{token.NUMBER, "1"},
|
||||
{token.NUMBER, "9"},
|
||||
{token.NUMBER, "42"},
|
||||
{token.NUMBER, "1234567890"},
|
||||
{token.NUMBER, "00"},
|
||||
{token.NUMBER, "01"},
|
||||
{token.NUMBER, "07"},
|
||||
{token.NUMBER, "042"},
|
||||
{token.NUMBER, "01234567"},
|
||||
{token.NUMBER, "0x0"},
|
||||
{token.NUMBER, "0x1"},
|
||||
{token.NUMBER, "0xf"},
|
||||
{token.NUMBER, "0x42"},
|
||||
{token.NUMBER, "0x123456789abcDEF"},
|
||||
{token.NUMBER, "0x" + f100},
|
||||
{token.NUMBER, "0X0"},
|
||||
{token.NUMBER, "0X1"},
|
||||
{token.NUMBER, "0XF"},
|
||||
{token.NUMBER, "0X42"},
|
||||
{token.NUMBER, "0X123456789abcDEF"},
|
||||
{token.NUMBER, "0X" + f100},
|
||||
{token.NUMBER, "0e0"},
|
||||
{token.NUMBER, "1e0"},
|
||||
{token.NUMBER, "42e0"},
|
||||
{token.NUMBER, "01234567890e0"},
|
||||
{token.NUMBER, "0E0"},
|
||||
{token.NUMBER, "1E0"},
|
||||
{token.NUMBER, "42E0"},
|
||||
{token.NUMBER, "01234567890E0"},
|
||||
{token.NUMBER, "0e+10"},
|
||||
{token.NUMBER, "1e-10"},
|
||||
{token.NUMBER, "42e+10"},
|
||||
{token.NUMBER, "01234567890e-10"},
|
||||
{token.NUMBER, "0E+10"},
|
||||
{token.NUMBER, "1E-10"},
|
||||
{token.NUMBER, "42E+10"},
|
||||
{token.NUMBER, "01234567890E-10"},
|
||||
},
|
||||
"float": []tokenPair{
|
||||
{token.FLOAT, "0."},
|
||||
{token.FLOAT, "1."},
|
||||
{token.FLOAT, "42."},
|
||||
{token.FLOAT, "01234567890."},
|
||||
{token.FLOAT, ".0"},
|
||||
{token.FLOAT, ".1"},
|
||||
{token.FLOAT, ".42"},
|
||||
{token.FLOAT, ".0123456789"},
|
||||
{token.FLOAT, "0.0"},
|
||||
{token.FLOAT, "1.0"},
|
||||
{token.FLOAT, "42.0"},
|
||||
{token.FLOAT, "01234567890.0"},
|
||||
{token.FLOAT, "01.8e0"},
|
||||
{token.FLOAT, "1.4e0"},
|
||||
{token.FLOAT, "42.2e0"},
|
||||
{token.FLOAT, "01234567890.12e0"},
|
||||
{token.FLOAT, "0.E0"},
|
||||
{token.FLOAT, "1.12E0"},
|
||||
{token.FLOAT, "42.123E0"},
|
||||
{token.FLOAT, "01234567890.213E0"},
|
||||
{token.FLOAT, "0.2e+10"},
|
||||
{token.FLOAT, "1.2e-10"},
|
||||
{token.FLOAT, "42.54e+10"},
|
||||
{token.FLOAT, "01234567890.98e-10"},
|
||||
{token.FLOAT, "0.1E+10"},
|
||||
{token.FLOAT, "1.1E-10"},
|
||||
{token.FLOAT, "42.1E+10"},
|
||||
{token.FLOAT, "01234567890.1E-10"},
|
||||
},
|
||||
// "string": []tokenPair{
|
||||
// {token.STRING, `" "`},
|
||||
// {token.STRING, `"a"`},
|
||||
// {token.STRING, `"本"`},
|
||||
// {token.STRING, `"\a"`},
|
||||
// {token.STRING, `"\b"`},
|
||||
// {token.STRING, `"\f"`},
|
||||
// {token.STRING, `"\n"`},
|
||||
// {token.STRING, `"\r"`},
|
||||
// {token.STRING, `"\t"`},
|
||||
// {token.STRING, `"\v"`},
|
||||
// {token.STRING, `"\""`},
|
||||
// {token.STRING, `"\000"`},
|
||||
// {token.STRING, `"\777"`},
|
||||
// {token.STRING, `"\x00"`},
|
||||
// {token.STRING, `"\xff"`},
|
||||
// {token.STRING, `"\u0000"`},
|
||||
// {token.STRING, `"\ufA16"`},
|
||||
// {token.STRING, `"\U00000000"`},
|
||||
// {token.STRING, `"\U0000ffAB"`},
|
||||
// {token.STRING, `"` + f100 + `"`},
|
||||
// },
|
||||
// "number": []tokenPair{
|
||||
// {token.NUMBER, "0"},
|
||||
// {token.NUMBER, "1"},
|
||||
// {token.NUMBER, "9"},
|
||||
// {token.NUMBER, "42"},
|
||||
// {token.NUMBER, "1234567890"},
|
||||
// {token.NUMBER, "00"},
|
||||
// {token.NUMBER, "01"},
|
||||
// {token.NUMBER, "07"},
|
||||
// {token.NUMBER, "042"},
|
||||
// {token.NUMBER, "01234567"},
|
||||
// {token.NUMBER, "0x0"},
|
||||
// {token.NUMBER, "0x1"},
|
||||
// {token.NUMBER, "0xf"},
|
||||
// {token.NUMBER, "0x42"},
|
||||
// {token.NUMBER, "0x123456789abcDEF"},
|
||||
// {token.NUMBER, "0x" + f100},
|
||||
// {token.NUMBER, "0X0"},
|
||||
// {token.NUMBER, "0X1"},
|
||||
// {token.NUMBER, "0XF"},
|
||||
// {token.NUMBER, "0X42"},
|
||||
// {token.NUMBER, "0X123456789abcDEF"},
|
||||
// {token.NUMBER, "0X" + f100},
|
||||
// {token.NUMBER, "0e0"},
|
||||
// {token.NUMBER, "1e0"},
|
||||
// {token.NUMBER, "42e0"},
|
||||
// {token.NUMBER, "01234567890e0"},
|
||||
// {token.NUMBER, "0E0"},
|
||||
// {token.NUMBER, "1E0"},
|
||||
// {token.NUMBER, "42E0"},
|
||||
// {token.NUMBER, "01234567890E0"},
|
||||
// {token.NUMBER, "0e+10"},
|
||||
// {token.NUMBER, "1e-10"},
|
||||
// {token.NUMBER, "42e+10"},
|
||||
// {token.NUMBER, "01234567890e-10"},
|
||||
// {token.NUMBER, "0E+10"},
|
||||
// {token.NUMBER, "1E-10"},
|
||||
// {token.NUMBER, "42E+10"},
|
||||
// {token.NUMBER, "01234567890E-10"},
|
||||
// },
|
||||
// "float": []tokenPair{
|
||||
// {token.FLOAT, "0."},
|
||||
// {token.FLOAT, "1."},
|
||||
// {token.FLOAT, "42."},
|
||||
// {token.FLOAT, "01234567890."},
|
||||
// {token.FLOAT, ".0"},
|
||||
// {token.FLOAT, ".1"},
|
||||
// {token.FLOAT, ".42"},
|
||||
// {token.FLOAT, ".0123456789"},
|
||||
// {token.FLOAT, "0.0"},
|
||||
// {token.FLOAT, "1.0"},
|
||||
// {token.FLOAT, "42.0"},
|
||||
// {token.FLOAT, "01234567890.0"},
|
||||
// {token.FLOAT, "01.8e0"},
|
||||
// {token.FLOAT, "1.4e0"},
|
||||
// {token.FLOAT, "42.2e0"},
|
||||
// {token.FLOAT, "01234567890.12e0"},
|
||||
// {token.FLOAT, "0.E0"},
|
||||
// {token.FLOAT, "1.12E0"},
|
||||
// {token.FLOAT, "42.123E0"},
|
||||
// {token.FLOAT, "01234567890.213E0"},
|
||||
// {token.FLOAT, "0.2e+10"},
|
||||
// {token.FLOAT, "1.2e-10"},
|
||||
// {token.FLOAT, "42.54e+10"},
|
||||
// {token.FLOAT, "01234567890.98e-10"},
|
||||
// {token.FLOAT, "0.1E+10"},
|
||||
// {token.FLOAT, "1.1E-10"},
|
||||
// {token.FLOAT, "42.1E+10"},
|
||||
// {token.FLOAT, "01234567890.1E-10"},
|
||||
// },
|
||||
}
|
||||
|
||||
func TestComment(t *testing.T) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user