scanner: simplify token text reading

scanner/scanner.go

@@ -25,7 +25,6 @@ type Scanner struct {
 	lastCharLen int // length of last character in bytes
 	lastLineLen int // length of last line in characters (for correct column reporting)
 
-	tokBuf   bytes.Buffer // token text buffer
 	tokStart int // token text start position
 	tokEnd   int // token text end  position
 
@@ -135,7 +134,6 @@ func (s *Scanner) Scan() Token {
 	var tok TokenType
 
 	// token text markings
-	s.tokBuf.Reset()
 	s.tokStart = s.srcPos.Offset - s.lastCharLen
 
 	// token position, initial next() is moving the offset by one(size of rune
@@ -201,35 +199,23 @@ func (s *Scanner) Scan() Token {
 		}
 	}
 
+	// finish token ending
 	s.tokEnd = s.srcPos.Offset
 
+	// create token literal
+	var tokenText string
+	if s.tokStart >= 0 {
+		tokenText = string(s.src[s.tokStart:s.tokEnd])
+	}
+	s.tokStart = s.tokEnd // ensure idempotency of tokenText() call
+
 	return Token{
 		token: tok,
 		pos:   s.tokPos,
-		text:  s.TokenText(),
+		text:  tokenText,
 	}
 }
 
-// TokenText returns the literal string corresponding to the most recently
-// scanned token.
-func (s *Scanner) TokenText() string {
-	if s.tokStart < 0 {
-		// no token text
-		return ""
-	}
-
-	// part of the token text was saved in tokBuf: save the rest in
-	// tokBuf as well and return its content
-	s.tokBuf.Write(s.src[s.tokStart:s.tokEnd])
-	s.tokStart = s.tokEnd // ensure idempotency of TokenText() call
-	return s.tokBuf.String()
-}
-
-// Pos returns the successful position of the most recently scanned token.
-func (s *Scanner) Pos() (pos Position) {
-	return s.tokPos
-}
-
 func (s *Scanner) scanComment(ch rune) {
 	// single line comments
 	if ch == '#' || (ch == '/' && s.peek() != '*') {

scanner/scanner_test.go

@@ -408,8 +408,8 @@ func testTokenList(t *testing.T, tokenList []tokenPair) {
 			t.Errorf("tok = %q want %q for %q\n", tok, ident.tok, ident.text)
 		}
 
-		if s.TokenText() != ident.text {
+		if tok.String() != ident.text {
-			t.Errorf("text = %q want %q", s.TokenText(), ident.text)
+			t.Errorf("text = %q want %q", tok.String(), ident.text)
 		}
 
 	}