go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

scanner: implement fully workable positions

Browse Source
This commit is contained in:
Fatih Arslan 2015-10-05 18:23:56 +03:00
parent ac40da147e
commit bc777d79f3
1 changed files with 78 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
scanner/scanner_test.go
View File

@ -15,84 +15,6 @@ type tokenPair struct {
text string
}
func testTokenList(t *testing.T, tokenList []tokenPair) {
// create artifical source code
buf := new(bytes.Buffer)
for _, ident := range tokenList {
fmt.Fprintf(buf, "%s\n", ident.text)
}
s, err := NewScanner(buf)
if err != nil {
t.Fatal(err)
}
for _, ident := range tokenList {
tok := s.Scan()
if tok != ident.tok {
t.Errorf("tok = %q want %q for %q\n", tok, ident.tok, ident.text)
}
if s.TokenText() != ident.text {
t.Errorf("text = %q want %q", s.TokenText(), ident.text)
}
}
}
func TestPosition(t *testing.T) {
// t.SkipNow()
// create artifical source code
buf := new(bytes.Buffer)
for _, listName := range orderedTokenLists {
for _, ident := range tokenLists[listName] {
fmt.Fprintf(buf, "\t\t\t\t%s\n", ident.text)
}
}
s, err := NewScanner(buf)
if err != nil {
t.Fatal(err)
}
pos := Position{"", 4, 1, 5}
for _, listName := range orderedTokenLists {
s.Scan()
for _, k := range tokenLists[listName] {
curPos := s.Pos()
// fmt.Printf("[%q] s = %+v:%+v\n", k.text, curPos.Offset, curPos.Column)
if curPos.Offset != pos.Offset {
t.Fatalf("offset = %d, want %d for %q", curPos.Offset, pos.Offset, k.text)
}
if curPos.Line != pos.Line {
t.Fatalf("line = %d, want %d for %q", curPos.Line, pos.Line, k.text)
}
if curPos.Column != pos.Column {
t.Fatalf("column = %d, want %d for %q", curPos.Column, pos.Column, k.text)
}
pos.Offset += 4 + len(k.text) + 1 // 4 tabs + token bytes + newline
pos.Line += countNewlines(k.text) + 1 // each token is on a new line
s.Scan()
}
}
// make sure there were no token-internal errors reported by scanner
if s.ErrorCount != 0 {
t.Errorf("%d errors", s.ErrorCount)
}
}
var orderedTokenLists = []string{
// "comment",
// "operator",
// "bool",
// "ident",
// "string",
"number",
// "float",
}
var tokenLists = map[string][]tokenPair{
"comment": []tokenPair{
{token.COMMENT, "//"},
@ -242,6 +164,59 @@ var tokenLists = map[string][]tokenPair{
},
}
var orderedTokenLists = []string{
"comment",
"operator",
"bool",
"ident",
"string",
"number",
"float",
}
func TestPosition(t *testing.T) {
// t.SkipNow()
// create artifical source code
buf := new(bytes.Buffer)
for _, listName := range orderedTokenLists {
for _, ident := range tokenLists[listName] {
fmt.Fprintf(buf, "\t\t\t\t%s\n", ident.text)
}
}
s, err := NewScanner(buf)
if err != nil {
t.Fatal(err)
}
pos := Position{"", 4, 1, 5}
s.Scan()
for _, listName := range orderedTokenLists {
for _, k := range tokenLists[listName] {
curPos := s.Pos()
// fmt.Printf("[%q] s = %+v:%+v\n", k.text, curPos.Offset, curPos.Column)
if curPos.Offset != pos.Offset {
t.Fatalf("offset = %d, want %d for %q", curPos.Offset, pos.Offset, k.text)
}
if curPos.Line != pos.Line {
t.Fatalf("line = %d, want %d for %q", curPos.Line, pos.Line, k.text)
}
if curPos.Column != pos.Column {
t.Fatalf("column = %d, want %d for %q", curPos.Column, pos.Column, k.text)
}
pos.Offset += 4 + len(k.text) + 1 // 4 tabs + token bytes + newline
pos.Line += countNewlines(k.text) + 1 // each token is on a new line
s.Scan()
}
}
// make sure there were no token-internal errors reported by scanner
if s.ErrorCount != 0 {
t.Errorf("%d errors", s.ErrorCount)
}
}
func TestComment(t *testing.T) {
testTokenList(t, tokenLists["comment"])
}
@ -270,6 +245,31 @@ func TestFloat(t *testing.T) {
testTokenList(t, tokenLists["float"])
}
func testTokenList(t *testing.T, tokenList []tokenPair) {
// create artifical source code
buf := new(bytes.Buffer)
for _, ident := range tokenList {
fmt.Fprintf(buf, "%s\n", ident.text)
}
s, err := NewScanner(buf)
if err != nil {
t.Fatal(err)
}
for _, ident := range tokenList {
tok := s.Scan()
if tok != ident.tok {
t.Errorf("tok = %q want %q for %q\n", tok, ident.tok, ident.text)
}
if s.TokenText() != ident.text {
t.Errorf("text = %q want %q", s.TokenText(), ident.text)
}
}
}
func countNewlines(s string) int {
n := 0
for _, ch := range s {