diff --git a/scanner/scanner.go b/scanner/scanner.go index 6487eb1..385d7bb 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -45,7 +45,14 @@ type Scanner struct { tokPos Position } -// NewScanner returns a new instance of Scanner. +// NewScannerstring creates and initializes a new instance of Scanner using +// string src as its source content. +func NewScannerString(src string) *Scanner { + return NewScanner([]byte(src)) +} + +// NewScanner creates and initializes a new instance of Scanner using src as +// its source content. func NewScanner(src []byte) *Scanner { // even though we accept a src, we read from a io.Reader compatible type // (*bytes.Buffer). So in the future we might easily change it to streaming @@ -99,7 +106,7 @@ func (s *Scanner) next() rune { return ch } -// unread +// unread unreads the previous read Rune and updates the source position func (s *Scanner) unread() { if err := s.buf.UnreadRune(); err != nil { panic(err) // this is user fault, we should catch it @@ -119,7 +126,7 @@ func (s *Scanner) peek() rune { } // Scan scans the next token and returns the token. -func (s *Scanner) Scan() (tok token.Token) { +func (s *Scanner) Scan() token.Token { ch := s.next() // skip white space @@ -127,6 +134,8 @@ func (s *Scanner) Scan() (tok token.Token) { ch = s.next() } + var tok token.Token + // token text markings s.tokBuf.Reset() s.tokStart = s.srcPos.Offset - s.lastCharLen