scanner: various fixes and improvements around NUMBER and FLOAT
This commit is contained in:
Fatih Arslan
parent
d9a424d177
commit
bbf8cf2ac0
@ -16,17 +16,21 @@ const eof = rune(0)
|
||||
|
||||
// Scanner defines a lexical scanner
|
||||
type Scanner struct {
|
||||
src *bytes.Buffer
|
||||
srcBytes []byte
|
||||
src *bytes.Buffer
|
||||
|
||||
lastCharLen int // length of last character in bytes
|
||||
// Source Buffer
|
||||
srcBuf []byte
|
||||
|
||||
currPos Position // current position
|
||||
// Source Position
|
||||
srcPos Position // current position
|
||||
prevPos Position // previous position
|
||||
|
||||
tokBuf bytes.Buffer // token text buffer
|
||||
tokPos int // token text tail position (srcBuf index); valid if >= 0
|
||||
tokEnd int // token text tail end (srcBuf index)
|
||||
lastCharLen int // length of last character in bytes
|
||||
lastLineLen int // length of last line in characters (for correct column reporting)
|
||||
|
||||
tokBuf bytes.Buffer // token text buffer
|
||||
tokStart int // token text start position
|
||||
tokEnd int // token text end position
|
||||
|
||||
// Error is called for each error encountered. If no Error
|
||||
// function is set, the error is reported to os.Stderr.
|
||||
@ -34,6 +38,14 @@ type Scanner struct {
|
||||
|
||||
// ErrorCount is incremented by one for each error encountered.
|
||||
ErrorCount int
|
||||
|
||||
// Start position of most recently scanned token; set by Scan.
|
||||
// Calling Init or Next invalidates the position (Line == 0).
|
||||
// The Filename field is always left untouched by the Scanner.
|
||||
// If an error is reported (via Error) and Position is invalid,
|
||||
// the scanner is not inside a token. Call Pos to obtain an error
|
||||
// position in that case.
|
||||
tokPos Position
|
||||
}
|
||||
|
||||
// NewScanner returns a new instance of Lexer. Even though src is an io.Reader,
|
||||
@ -45,10 +57,12 @@ func NewScanner(src io.Reader) (*Scanner, error) {
|
||||
}
|
||||
|
||||
b := bytes.NewBuffer(buf)
|
||||
return &Scanner{
|
||||
src: b,
|
||||
srcBytes: b.Bytes(),
|
||||
}, nil
|
||||
s := &Scanner{
|
||||
src: b,
|
||||
srcBuf: b.Bytes(),
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// next reads the next rune from the bufferred reader. Returns the rune(0) if
|
||||
@ -60,15 +74,16 @@ func (s *Scanner) next() rune {
|
||||
}
|
||||
|
||||
// remember last position
|
||||
s.prevPos = s.currPos
|
||||
|
||||
s.prevPos = s.srcPos
|
||||
s.lastCharLen = size
|
||||
s.currPos.Offset += size
|
||||
s.currPos.Column += size
|
||||
|
||||
s.srcPos.Offset += size
|
||||
|
||||
s.srcPos.Column += size
|
||||
if ch == '\n' {
|
||||
s.currPos.Line++
|
||||
s.currPos.Column = 0
|
||||
s.srcPos.Line++
|
||||
s.srcPos.Column = 0
|
||||
s.lastLineLen = s.srcPos.Column
|
||||
}
|
||||
|
||||
return ch
|
||||
@ -78,7 +93,7 @@ func (s *Scanner) unread() {
|
||||
if err := s.src.UnreadRune(); err != nil {
|
||||
panic(err) // this is user fault, we should catch it
|
||||
}
|
||||
s.currPos = s.prevPos // put back last position
|
||||
s.srcPos = s.prevPos // put back last position
|
||||
}
|
||||
|
||||
func (s *Scanner) peek() rune {
|
||||
@ -93,16 +108,30 @@ func (s *Scanner) peek() rune {
|
||||
|
||||
// Scan scans the next token and returns the token.
|
||||
func (s *Scanner) Scan() (tok token.Token) {
|
||||
ch := s.next()
|
||||
ch := s.peek()
|
||||
|
||||
// skip white space
|
||||
for isWhitespace(ch) {
|
||||
ch = s.next()
|
||||
}
|
||||
|
||||
// start the token position
|
||||
// token text markings
|
||||
s.tokBuf.Reset()
|
||||
s.tokPos = s.currPos.Offset - s.lastCharLen
|
||||
s.tokStart = s.srcPos.Offset - s.lastCharLen
|
||||
|
||||
// token position
|
||||
s.tokPos.Offset = s.srcPos.Offset
|
||||
if s.srcPos.Column > 0 {
|
||||
// common case: last character was not a '\n'
|
||||
s.tokPos.Line = s.srcPos.Line
|
||||
s.tokPos.Column = s.srcPos.Column
|
||||
} else {
|
||||
// last character was a '\n'
|
||||
// (we cannot be at the beginning of the source
|
||||
// since we have called next() at least once)
|
||||
s.tokPos.Line = s.srcPos.Line - 1
|
||||
s.tokPos.Column = s.lastLineLen
|
||||
}
|
||||
|
||||
switch {
|
||||
case isLetter(ch):
|
||||
@ -150,7 +179,7 @@ func (s *Scanner) Scan() (tok token.Token) {
|
||||
}
|
||||
}
|
||||
|
||||
s.tokEnd = s.currPos.Offset
|
||||
s.tokEnd = s.srcPos.Offset
|
||||
return tok
|
||||
}
|
||||
|
||||
@ -219,10 +248,21 @@ func (s *Scanner) scanNumber(ch rune) token.Token {
|
||||
}
|
||||
s.unread()
|
||||
|
||||
if ch == '.' || ch == 'e' || ch == 'E' {
|
||||
ch = s.next()
|
||||
ch = s.scanFraction(ch)
|
||||
// literals of form 01e10 are treates as Numbers in HCL, which differs from Go.
|
||||
if ch == 'e' || ch == 'E' {
|
||||
ch = s.next() // seek forward
|
||||
ch = s.scanExponent(ch)
|
||||
return token.NUMBER
|
||||
}
|
||||
|
||||
if ch == '.' {
|
||||
ch = s.next() // seek forward
|
||||
ch = s.scanFraction(ch)
|
||||
|
||||
if ch == 'e' || ch == 'E' {
|
||||
ch = s.next()
|
||||
ch = s.scanExponent(ch)
|
||||
}
|
||||
return token.FLOAT
|
||||
}
|
||||
|
||||
@ -234,10 +274,20 @@ func (s *Scanner) scanNumber(ch rune) token.Token {
|
||||
}
|
||||
|
||||
ch = s.scanMantissa(ch)
|
||||
if ch == '.' || ch == 'e' || ch == 'E' {
|
||||
// literals of form 1e10 are treates as Numbers in HCL, which differs from Go.
|
||||
if ch == 'e' || ch == 'E' {
|
||||
ch = s.next()
|
||||
ch = s.scanExponent(ch)
|
||||
return token.NUMBER
|
||||
}
|
||||
|
||||
if ch == '.' {
|
||||
ch = s.next() // seek forward
|
||||
ch = s.scanFraction(ch)
|
||||
ch = s.scanExponent(ch)
|
||||
if ch == 'e' || ch == 'E' {
|
||||
ch = s.next()
|
||||
ch = s.scanExponent(ch)
|
||||
}
|
||||
return token.FLOAT
|
||||
}
|
||||
return token.NUMBER
|
||||
@ -344,46 +394,45 @@ func (s *Scanner) scanDigits(ch rune, base, n int) rune {
|
||||
|
||||
// scanIdentifier scans an identifier and returns the literal string
|
||||
func (s *Scanner) scanIdentifier() string {
|
||||
offs := s.currPos.Offset - s.lastCharLen
|
||||
offs := s.srcPos.Offset - s.lastCharLen
|
||||
ch := s.next()
|
||||
for isLetter(ch) || isDigit(ch) {
|
||||
ch = s.next()
|
||||
}
|
||||
s.unread() // we got identifier, put back latest char
|
||||
|
||||
// return string(s.srcBytes[offs:(s.currPos.Offset - s.lastCharLen)])
|
||||
return string(s.srcBytes[offs:s.currPos.Offset])
|
||||
return string(s.srcBuf[offs:s.srcPos.Offset])
|
||||
}
|
||||
|
||||
// TokenText returns the literal string corresponding to the most recently
|
||||
// scanned token.
|
||||
func (s *Scanner) TokenText() string {
|
||||
if s.tokPos < 0 {
|
||||
if s.tokStart < 0 {
|
||||
// no token text
|
||||
return ""
|
||||
}
|
||||
|
||||
// part of the token text was saved in tokBuf: save the rest in
|
||||
// tokBuf as well and return its content
|
||||
s.tokBuf.Write(s.srcBytes[s.tokPos:s.tokEnd])
|
||||
s.tokPos = s.tokEnd // ensure idempotency of TokenText() call
|
||||
s.tokBuf.Write(s.srcBuf[s.tokStart:s.tokEnd])
|
||||
s.tokStart = s.tokEnd // ensure idempotency of TokenText() call
|
||||
return s.tokBuf.String()
|
||||
}
|
||||
|
||||
// Pos returns the position of the character immediately after the character or
|
||||
// token returned by the last call to Scan.
|
||||
func (s *Scanner) Pos() Position {
|
||||
return s.currPos
|
||||
func (s *Scanner) Pos() (pos Position) {
|
||||
return s.tokPos
|
||||
}
|
||||
|
||||
func (s *Scanner) err(msg string) {
|
||||
s.ErrorCount++
|
||||
if s.Error != nil {
|
||||
s.Error(s.currPos, msg)
|
||||
s.Error(s.srcPos, msg)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", s.currPos, msg)
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", s.srcPos, msg)
|
||||
}
|
||||
|
||||
func isLetter(ch rune) bool {
|
||||
|
||||
@ -40,43 +40,84 @@ func testTokenList(t *testing.T, tokenList []tokenPair) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPosition(t *testing.T) {
|
||||
t.SkipNow()
|
||||
// create artifical source code
|
||||
buf := new(bytes.Buffer)
|
||||
for _, list := range tokenLists {
|
||||
for _, ident := range list {
|
||||
fmt.Fprintf(buf, "\t\t\t\t%s\n", ident.text)
|
||||
}
|
||||
}
|
||||
|
||||
s, err := NewScanner(buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s.Scan()
|
||||
pos := Position{"", 4, 1, 5}
|
||||
for _, list := range tokenLists {
|
||||
for _, k := range list {
|
||||
curPos := s.Pos()
|
||||
fmt.Printf("[%q] s = %+v:%+v\n", k.text, curPos.Offset, curPos.Column)
|
||||
if curPos.Offset != pos.Offset {
|
||||
t.Errorf("offset = %d, want %d for %q", curPos.Offset, pos.Offset, k.text)
|
||||
}
|
||||
if curPos.Line != pos.Line {
|
||||
t.Errorf("line = %d, want %d for %q", curPos.Line, pos.Line, k.text)
|
||||
}
|
||||
if curPos.Column != pos.Column {
|
||||
t.Errorf("column = %d, want %d for %q", curPos.Column, pos.Column, k.text)
|
||||
}
|
||||
pos.Offset += 4 + len(k.text) + 1 // 4 tabs + token bytes + newline
|
||||
pos.Line += countNewlines(k.text) + 1 // each token is on a new line
|
||||
s.Scan()
|
||||
}
|
||||
}
|
||||
// make sure there were no token-internal errors reported by scanner
|
||||
if s.ErrorCount != 0 {
|
||||
t.Errorf("%d errors", s.ErrorCount)
|
||||
}
|
||||
}
|
||||
|
||||
var tokenLists = map[string][]tokenPair{
|
||||
"comment": []tokenPair{
|
||||
{token.COMMENT, "//"},
|
||||
{token.COMMENT, "////"},
|
||||
{token.COMMENT, "// comment"},
|
||||
{token.COMMENT, "// /* comment */"},
|
||||
{token.COMMENT, "// // comment //"},
|
||||
{token.COMMENT, "//" + f100},
|
||||
{token.COMMENT, "#"},
|
||||
{token.COMMENT, "##"},
|
||||
{token.COMMENT, "# comment"},
|
||||
{token.COMMENT, "# /* comment */"},
|
||||
{token.COMMENT, "# # comment #"},
|
||||
{token.COMMENT, "#" + f100},
|
||||
{token.COMMENT, "/**/"},
|
||||
{token.COMMENT, "/***/"},
|
||||
{token.COMMENT, "/* comment */"},
|
||||
{token.COMMENT, "/* // comment */"},
|
||||
{token.COMMENT, "/* /* comment */"},
|
||||
{token.COMMENT, "/*\n comment\n*/"},
|
||||
{token.COMMENT, "/*" + f100 + "*/"},
|
||||
},
|
||||
"operator": []tokenPair{
|
||||
{token.LBRACK, "["},
|
||||
{token.LBRACE, "{"},
|
||||
{token.COMMA, ","},
|
||||
{token.PERIOD, "."},
|
||||
{token.RBRACK, "]"},
|
||||
{token.RBRACE, "}"},
|
||||
{token.ASSIGN, "="},
|
||||
{token.ADD, "+"},
|
||||
{token.SUB, "-"},
|
||||
},
|
||||
"bool": []tokenPair{
|
||||
{token.BOOL, "true"},
|
||||
{token.BOOL, "false"},
|
||||
},
|
||||
// "comment": []tokenPair{
|
||||
// {token.COMMENT, "//"},
|
||||
// {token.COMMENT, "////"},
|
||||
// {token.COMMENT, "// comment"},
|
||||
// {token.COMMENT, "// /* comment */"},
|
||||
// {token.COMMENT, "// // comment //"},
|
||||
// {token.COMMENT, "//" + f100},
|
||||
// {token.COMMENT, "#"},
|
||||
// {token.COMMENT, "##"},
|
||||
// {token.COMMENT, "# comment"},
|
||||
// {token.COMMENT, "# /* comment */"},
|
||||
// {token.COMMENT, "# # comment #"},
|
||||
// {token.COMMENT, "#" + f100},
|
||||
// {token.COMMENT, "/**/"},
|
||||
// {token.COMMENT, "/***/"},
|
||||
// {token.COMMENT, "/* comment */"},
|
||||
// {token.COMMENT, "/* // comment */"},
|
||||
// {token.COMMENT, "/* /* comment */"},
|
||||
// {token.COMMENT, "/*\n comment\n*/"},
|
||||
// {token.COMMENT, "/*" + f100 + "*/"},
|
||||
// },
|
||||
// "operator": []tokenPair{
|
||||
// {token.LBRACK, "["},
|
||||
// {token.LBRACE, "{"},
|
||||
// {token.COMMA, ","},
|
||||
// {token.PERIOD, "."},
|
||||
// {token.RBRACK, "]"},
|
||||
// {token.RBRACE, "}"},
|
||||
// {token.ASSIGN, "="},
|
||||
// {token.ADD, "+"},
|
||||
// {token.SUB, "-"},
|
||||
// },
|
||||
// "bool": []tokenPair{
|
||||
// {token.BOOL, "true"},
|
||||
// {token.BOOL, "false"},
|
||||
// },
|
||||
|
||||
"ident": []tokenPair{
|
||||
{token.IDENT, "a"},
|
||||
@ -88,36 +129,36 @@ var tokenLists = map[string][]tokenPair{
|
||||
{token.IDENT, "_abc123"},
|
||||
{token.IDENT, "abc123_"},
|
||||
{token.IDENT, "_abc_123_"},
|
||||
{token.IDENT, "_äöü"},
|
||||
{token.IDENT, "_本"},
|
||||
{token.IDENT, "äöü"},
|
||||
{token.IDENT, "本"},
|
||||
{token.IDENT, "a۰۱۸"},
|
||||
{token.IDENT, "foo६४"},
|
||||
{token.IDENT, "bar９８７６"},
|
||||
},
|
||||
"string": []tokenPair{
|
||||
{token.STRING, `" "`},
|
||||
{token.STRING, `"a"`},
|
||||
{token.STRING, `"本"`},
|
||||
{token.STRING, `"\a"`},
|
||||
{token.STRING, `"\b"`},
|
||||
{token.STRING, `"\f"`},
|
||||
{token.STRING, `"\n"`},
|
||||
{token.STRING, `"\r"`},
|
||||
{token.STRING, `"\t"`},
|
||||
{token.STRING, `"\v"`},
|
||||
{token.STRING, `"\""`},
|
||||
{token.STRING, `"\000"`},
|
||||
{token.STRING, `"\777"`},
|
||||
{token.STRING, `"\x00"`},
|
||||
{token.STRING, `"\xff"`},
|
||||
{token.STRING, `"\u0000"`},
|
||||
{token.STRING, `"\ufA16"`},
|
||||
{token.STRING, `"\U00000000"`},
|
||||
{token.STRING, `"\U0000ffAB"`},
|
||||
{token.STRING, `"` + f100 + `"`},
|
||||
// {token.IDENT, "_äöü"},
|
||||
// {token.IDENT, "_本"},
|
||||
// {token.IDENT, "äöü"},
|
||||
// {token.IDENT, "本"},
|
||||
// {token.IDENT, "a۰۱۸"},
|
||||
// {token.IDENT, "foo६४"},
|
||||
// {token.IDENT, "bar９８７６"},
|
||||
},
|
||||
// "string": []tokenPair{
|
||||
// {token.STRING, `" "`},
|
||||
// {token.STRING, `"a"`},
|
||||
// {token.STRING, `"本"`},
|
||||
// {token.STRING, `"\a"`},
|
||||
// {token.STRING, `"\b"`},
|
||||
// {token.STRING, `"\f"`},
|
||||
// {token.STRING, `"\n"`},
|
||||
// {token.STRING, `"\r"`},
|
||||
// {token.STRING, `"\t"`},
|
||||
// {token.STRING, `"\v"`},
|
||||
// {token.STRING, `"\""`},
|
||||
// {token.STRING, `"\000"`},
|
||||
// {token.STRING, `"\777"`},
|
||||
// {token.STRING, `"\x00"`},
|
||||
// {token.STRING, `"\xff"`},
|
||||
// {token.STRING, `"\u0000"`},
|
||||
// {token.STRING, `"\ufA16"`},
|
||||
// {token.STRING, `"\U00000000"`},
|
||||
// {token.STRING, `"\U0000ffAB"`},
|
||||
// {token.STRING, `"` + f100 + `"`},
|
||||
// },
|
||||
"number": []tokenPair{
|
||||
{token.NUMBER, "0"},
|
||||
{token.NUMBER, "1"},
|
||||
@ -141,6 +182,22 @@ var tokenLists = map[string][]tokenPair{
|
||||
{token.NUMBER, "0X42"},
|
||||
{token.NUMBER, "0X123456789abcDEF"},
|
||||
{token.NUMBER, "0X" + f100},
|
||||
{token.NUMBER, "0e0"},
|
||||
{token.NUMBER, "1e0"},
|
||||
{token.NUMBER, "42e0"},
|
||||
{token.NUMBER, "01234567890e0"},
|
||||
{token.NUMBER, "0E0"},
|
||||
{token.NUMBER, "1E0"},
|
||||
{token.NUMBER, "42E0"},
|
||||
{token.NUMBER, "01234567890E0"},
|
||||
{token.NUMBER, "0e+10"},
|
||||
{token.NUMBER, "1e-10"},
|
||||
{token.NUMBER, "42e+10"},
|
||||
{token.NUMBER, "01234567890e-10"},
|
||||
{token.NUMBER, "0E+10"},
|
||||
{token.NUMBER, "1E-10"},
|
||||
{token.NUMBER, "42E+10"},
|
||||
{token.NUMBER, "01234567890E-10"},
|
||||
},
|
||||
"float": []tokenPair{
|
||||
{token.FLOAT, "0."},
|
||||
@ -155,22 +212,22 @@ var tokenLists = map[string][]tokenPair{
|
||||
{token.FLOAT, "1.0"},
|
||||
{token.FLOAT, "42.0"},
|
||||
{token.FLOAT, "01234567890.0"},
|
||||
{token.FLOAT, "0e0"},
|
||||
{token.FLOAT, "1e0"},
|
||||
{token.FLOAT, "42e0"},
|
||||
{token.FLOAT, "01234567890e0"},
|
||||
{token.FLOAT, "0E0"},
|
||||
{token.FLOAT, "1E0"},
|
||||
{token.FLOAT, "42E0"},
|
||||
{token.FLOAT, "01234567890E0"},
|
||||
{token.FLOAT, "0e+10"},
|
||||
{token.FLOAT, "1e-10"},
|
||||
{token.FLOAT, "42e+10"},
|
||||
{token.FLOAT, "01234567890e-10"},
|
||||
{token.FLOAT, "0E+10"},
|
||||
{token.FLOAT, "1E-10"},
|
||||
{token.FLOAT, "42E+10"},
|
||||
{token.FLOAT, "01234567890E-10"},
|
||||
{token.FLOAT, "01.8e0"},
|
||||
{token.FLOAT, "1.4e0"},
|
||||
{token.FLOAT, "42.2e0"},
|
||||
{token.FLOAT, "01234567890.12e0"},
|
||||
{token.FLOAT, "0.E0"},
|
||||
{token.FLOAT, "1.12E0"},
|
||||
{token.FLOAT, "42.123E0"},
|
||||
{token.FLOAT, "01234567890.213E0"},
|
||||
{token.FLOAT, "0.2e+10"},
|
||||
{token.FLOAT, "1.2e-10"},
|
||||
{token.FLOAT, "42.54e+10"},
|
||||
{token.FLOAT, "01234567890.98e-10"},
|
||||
{token.FLOAT, "0.1E+10"},
|
||||
{token.FLOAT, "1.1E-10"},
|
||||
{token.FLOAT, "42.1E+10"},
|
||||
{token.FLOAT, "01234567890.1E-10"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -201,3 +258,13 @@ func TestNumber(t *testing.T) {
|
||||
func TestFloat(t *testing.T) {
|
||||
testTokenList(t, tokenLists["float"])
|
||||
}
|
||||
|
||||
func countNewlines(s string) int {
|
||||
n := 0
|
||||
for _, ch := range s {
|
||||
if ch == '\n' {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user