go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

scanner: improvements around Position

Browse Source
This commit is contained in:
Fatih Arslan 2015-10-05 17:43:29 +03:00
parent 89db79cf63
commit 81a8399ed1
2 changed files with 82 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scanner/scanner.go
View File

@ -62,6 +62,9 @@ func NewScanner(src io.Reader) (*Scanner, error) {
srcBuf: b.Bytes(),
}
// srcPosition always starts with 1
s.srcPos.Line = 1
return s, nil
}
@ -78,7 +81,6 @@ func (s *Scanner) next() rune {
s.lastCharLen = size
s.srcPos.Offset += size
s.srcPos.Column += size
if ch == '\n' {
s.srcPos.Line++
@ -119,8 +121,10 @@ func (s *Scanner) Scan() (tok token.Token) {
s.tokBuf.Reset()
s.tokStart = s.srcPos.Offset - s.lastCharLen
// token position
s.tokPos.Offset = s.srcPos.Offset
// token position, initial next() is moving the offset by one, though we
// are interested with the starting point
s.tokPos.Offset = s.srcPos.Offset - 1
if s.srcPos.Column > 0 {
// common case: last character was not a '\n'
s.tokPos.Line = s.srcPos.Line
@ -271,16 +275,15 @@ func (s *Scanner) scanNumber(ch rune) token.Token {
return token.NUMBER
}
ch = s.scanMantissa(ch)
s.scanMantissa(ch)
ch = s.next() // seek forward
// literals of form 1e10 are treates as Numbers in HCL, which differs from Go.
if ch == 'e' || ch == 'E' {
ch = s.next()
ch = s.scanExponent(ch)
return token.NUMBER
}
if ch == '.' {
ch = s.next() // seek forward
ch = s.scanFraction(ch)
if ch == 'e' || ch == 'E' {
ch = s.next()
@ -288,6 +291,8 @@ func (s *Scanner) scanNumber(ch rune) token.Token {
}
return token.FLOAT
}
s.unread()
return token.NUMBER
}

142
scanner/scanner_test.go
View File

@ -41,7 +41,7 @@ func testTokenList(t *testing.T, tokenList []tokenPair) {
}
func TestPosition(t *testing.T) {
t.SkipNow()
// t.SkipNow()
// create artifical source code
buf := new(bytes.Buffer)
for _, list := range tokenLists {
@ -159,76 +159,76 @@ var tokenLists = map[string][]tokenPair{
// {token.STRING, `"\U0000ffAB"`},
// {token.STRING, `"` + f100 + `"`},
// },
"number": []tokenPair{
{token.NUMBER, "0"},
{token.NUMBER, "1"},
{token.NUMBER, "9"},
{token.NUMBER, "42"},
{token.NUMBER, "1234567890"},
{token.NUMBER, "00"},
{token.NUMBER, "01"},
{token.NUMBER, "07"},
{token.NUMBER, "042"},
{token.NUMBER, "01234567"},
{token.NUMBER, "0x0"},
{token.NUMBER, "0x1"},
{token.NUMBER, "0xf"},
{token.NUMBER, "0x42"},
{token.NUMBER, "0x123456789abcDEF"},
{token.NUMBER, "0x" + f100},
{token.NUMBER, "0X0"},
{token.NUMBER, "0X1"},
{token.NUMBER, "0XF"},
{token.NUMBER, "0X42"},
{token.NUMBER, "0X123456789abcDEF"},
{token.NUMBER, "0X" + f100},
{token.NUMBER, "0e0"},
{token.NUMBER, "1e0"},
{token.NUMBER, "42e0"},
{token.NUMBER, "01234567890e0"},
{token.NUMBER, "0E0"},
{token.NUMBER, "1E0"},
{token.NUMBER, "42E0"},
{token.NUMBER, "01234567890E0"},
{token.NUMBER, "0e+10"},
{token.NUMBER, "1e-10"},
{token.NUMBER, "42e+10"},
{token.NUMBER, "01234567890e-10"},
{token.NUMBER, "0E+10"},
{token.NUMBER, "1E-10"},
{token.NUMBER, "42E+10"},
{token.NUMBER, "01234567890E-10"},
},
"float": []tokenPair{
{token.FLOAT, "0."},
{token.FLOAT, "1."},
{token.FLOAT, "42."},
{token.FLOAT, "01234567890."},
{token.FLOAT, ".0"},
{token.FLOAT, ".1"},
{token.FLOAT, ".42"},
{token.FLOAT, ".0123456789"},
{token.FLOAT, "0.0"},
{token.FLOAT, "1.0"},
{token.FLOAT, "42.0"},
{token.FLOAT, "01234567890.0"},
{token.FLOAT, "01.8e0"},
{token.FLOAT, "1.4e0"},
{token.FLOAT, "42.2e0"},
{token.FLOAT, "01234567890.12e0"},
{token.FLOAT, "0.E0"},
{token.FLOAT, "1.12E0"},
{token.FLOAT, "42.123E0"},
{token.FLOAT, "01234567890.213E0"},
{token.FLOAT, "0.2e+10"},
{token.FLOAT, "1.2e-10"},
{token.FLOAT, "42.54e+10"},
{token.FLOAT, "01234567890.98e-10"},
{token.FLOAT, "0.1E+10"},
{token.FLOAT, "1.1E-10"},
{token.FLOAT, "42.1E+10"},
{token.FLOAT, "01234567890.1E-10"},
},
// "number": []tokenPair{
// {token.NUMBER, "0"},
// {token.NUMBER, "1"},
// {token.NUMBER, "9"},
// {token.NUMBER, "42"},
// {token.NUMBER, "1234567890"},
// {token.NUMBER, "00"},
// {token.NUMBER, "01"},
// {token.NUMBER, "07"},
// {token.NUMBER, "042"},
// {token.NUMBER, "01234567"},
// {token.NUMBER, "0x0"},
// {token.NUMBER, "0x1"},
// {token.NUMBER, "0xf"},
// {token.NUMBER, "0x42"},
// {token.NUMBER, "0x123456789abcDEF"},
// {token.NUMBER, "0x" + f100},
// {token.NUMBER, "0X0"},
// {token.NUMBER, "0X1"},
// {token.NUMBER, "0XF"},
// {token.NUMBER, "0X42"},
// {token.NUMBER, "0X123456789abcDEF"},
// {token.NUMBER, "0X" + f100},
// {token.NUMBER, "0e0"},
// {token.NUMBER, "1e0"},
// {token.NUMBER, "42e0"},
// {token.NUMBER, "01234567890e0"},
// {token.NUMBER, "0E0"},
// {token.NUMBER, "1E0"},
// {token.NUMBER, "42E0"},
// {token.NUMBER, "01234567890E0"},
// {token.NUMBER, "0e+10"},
// {token.NUMBER, "1e-10"},
// {token.NUMBER, "42e+10"},
// {token.NUMBER, "01234567890e-10"},
// {token.NUMBER, "0E+10"},
// {token.NUMBER, "1E-10"},
// {token.NUMBER, "42E+10"},
// {token.NUMBER, "01234567890E-10"},
// },
// "float": []tokenPair{
// {token.FLOAT, "0."},
// {token.FLOAT, "1."},
// {token.FLOAT, "42."},
// {token.FLOAT, "01234567890."},
// {token.FLOAT, ".0"},
// {token.FLOAT, ".1"},
// {token.FLOAT, ".42"},
// {token.FLOAT, ".0123456789"},
// {token.FLOAT, "0.0"},
// {token.FLOAT, "1.0"},
// {token.FLOAT, "42.0"},
// {token.FLOAT, "01234567890.0"},
// {token.FLOAT, "01.8e0"},
// {token.FLOAT, "1.4e0"},
// {token.FLOAT, "42.2e0"},
// {token.FLOAT, "01234567890.12e0"},
// {token.FLOAT, "0.E0"},
// {token.FLOAT, "1.12E0"},
// {token.FLOAT, "42.123E0"},
// {token.FLOAT, "01234567890.213E0"},
// {token.FLOAT, "0.2e+10"},
// {token.FLOAT, "1.2e-10"},
// {token.FLOAT, "42.54e+10"},
// {token.FLOAT, "01234567890.98e-10"},
// {token.FLOAT, "0.1E+10"},
// {token.FLOAT, "1.1E-10"},
// {token.FLOAT, "42.1E+10"},
// {token.FLOAT, "01234567890.1E-10"},
// },
}
func TestComment(t *testing.T) {