go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

scanner: simplify token text reading

Browse Source
This commit is contained in:
Fatih Arslan 2015-10-07 15:04:34 +03:00
parent fa991d3df2
commit 0728686f59
2 changed files with 13 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scanner/scanner.go
View File

@ -25,7 +25,6 @@ type Scanner struct {
lastCharLen int // length of last character in bytes
lastLineLen int // length of last line in characters (for correct column reporting)
tokBuf bytes.Buffer // token text buffer
tokStart int // token text start position
tokEnd int // token text end position
@ -135,7 +134,6 @@ func (s *Scanner) Scan() Token {
var tok TokenType
// token text markings
s.tokBuf.Reset()
s.tokStart = s.srcPos.Offset - s.lastCharLen
// token position, initial next() is moving the offset by one(size of rune
@ -201,35 +199,23 @@ func (s *Scanner) Scan() Token {
}
}
// finish token ending
s.tokEnd = s.srcPos.Offset
// create token literal
var tokenText string
if s.tokStart >= 0 {
tokenText = string(s.src[s.tokStart:s.tokEnd])
}
s.tokStart = s.tokEnd // ensure idempotency of tokenText() call
return Token{
token: tok,
pos: s.tokPos,
text: s.TokenText(),
text: tokenText,
}
}
// TokenText returns the literal string corresponding to the most recently
// scanned token.
func (s *Scanner) TokenText() string {
if s.tokStart < 0 {
// no token text
return ""
}
// part of the token text was saved in tokBuf: save the rest in
// tokBuf as well and return its content
s.tokBuf.Write(s.src[s.tokStart:s.tokEnd])
s.tokStart = s.tokEnd // ensure idempotency of TokenText() call
return s.tokBuf.String()
}
// Pos returns the successful position of the most recently scanned token.
func (s *Scanner) Pos() (pos Position) {
return s.tokPos
}
func (s *Scanner) scanComment(ch rune) {
// single line comments
if ch == '#' || (ch == '/' && s.peek() != '*') {

4
scanner/scanner_test.go
View File

@ -408,8 +408,8 @@ func testTokenList(t *testing.T, tokenList []tokenPair) {
t.Errorf("tok = %q want %q for %q\n", tok, ident.tok, ident.text)
}
if s.TokenText() != ident.text {
t.Errorf("text = %q want %q", s.TokenText(), ident.text)
if tok.String() != ident.text {
t.Errorf("text = %q want %q", tok.String(), ident.text)
}
}