go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zclsyntax: scanning of numeric literals

Browse Source
This commit is contained in:
Martin Atkins 2017-05-28 08:56:43 -07:00
parent 187d7b8045
commit 6362354c87
3 changed files with 180 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
zcl/zclsyntax/scan_tokens.go
View File

@ -11,62 +11,75 @@ import (
var _zcltok_actions []byte = []byte{
0, 1, 0, 1, 1, 1, 2, 1, 3,
1, 4, 1, 5, 1, 6, 1, 7,
1, 8,
1, 8, 1, 9, 1, 10,
}
var _zcltok_key_offsets []byte = []byte{
0, 2, 4, 16, 17, 18, 20, 22,
0, 4, 6, 8, 10, 24, 25, 26,
31, 33, 35,
}
var _zcltok_trans_keys []byte = []byte{
128, 191, 128, 191, 9, 32, 128, 191,
43, 45, 48, 57, 48, 57, 128, 191,
128, 191, 9, 32, 48, 57, 128, 191,
192, 223, 224, 239, 240, 247, 248, 255,
9, 32, 128, 191, 128, 191, 128, 191,
9, 32, 46, 69, 101, 48, 57, 128,
191, 128, 191, 128, 191,
}
var _zcltok_single_lengths []byte = []byte{
0, 0, 2, 1, 1, 0, 0, 0,
2, 0, 0, 0, 2, 1, 1, 3,
0, 0, 0,
}
var _zcltok_range_lengths []byte = []byte{
1, 1, 5, 0, 0, 1, 1, 1,
1, 1, 1, 1, 6, 0, 0, 1,
1, 1, 1,
}
var _zcltok_index_offsets []byte = []byte{
0, 2, 4, 12, 14, 16, 18, 20,
0, 4, 6, 8, 10, 19, 21, 23,
28, 30, 32,
}
var _zcltok_trans_targs []byte = []byte{
2, 2, 0, 2, 3, 4, 2, 5,
6, 7, 2, 2, 3, 2, 4, 2,
2, 2, 0, 2, 1, 2, 2, 2,
2, 2, 2, 2, 2,
1, 1, 7, 4, 7, 4, 4, 4,
2, 4, 5, 6, 7, 4, 8, 9,
10, 4, 4, 5, 4, 6, 4, 7,
0, 0, 7, 4, 4, 4, 2, 4,
3, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4,
}
var _zcltok_trans_actions []byte = []byte{
7, 17, 0, 17, 0, 0, 9, 0,
5, 5, 9, 7, 0, 13, 0, 11,
7, 15, 0, 15, 0, 15, 17, 17,
13, 11, 15, 15, 15,
0, 0, 5, 19, 5, 19, 7, 21,
0, 21, 0, 0, 5, 9, 0, 5,
5, 9, 7, 0, 15, 0, 11, 5,
0, 0, 5, 13, 7, 17, 0, 17,
0, 17, 19, 19, 21, 21, 15, 11,
13, 17, 17, 17,
}
var _zcltok_to_state_actions []byte = []byte{
0, 0, 1, 0, 0, 0, 0, 0,
0, 0, 0, 0, 1, 0, 0, 0,
0, 0, 0,
}
var _zcltok_from_state_actions []byte = []byte{
0, 0, 3, 0, 0, 0, 0, 0,
0, 0, 0, 0, 3, 0, 0, 0,
0, 0, 0,
}
var _zcltok_eof_trans []byte = []byte{
24, 24, 0, 25, 26, 29, 29, 29,
36, 36, 38, 38, 0, 39, 40, 41,
44, 44, 44,
}
const zcltok_start int = 2
const zcltok_first_final int = 2
const zcltok_start int = 4
const zcltok_first_final int = 4
const zcltok_error int = -1
const zcltok_en_main int = 2
const zcltok_en_main int = 4
// line 13 "scan_tokens.rl"
@ -77,7 +90,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
Pos: start,
}
// line 47 "scan_tokens.rl"
// line 50 "scan_tokens.rl"
// Ragel state
cs := 0 // Current State
@ -98,7 +111,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
f.emitToken(ty, ts, te)
}
// line 109 "scan_tokens.go"
// line 121 "scan_tokens.go"
{
cs = zcltok_start
ts = 0
@ -106,7 +119,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
act = 0
}
// line 117 "scan_tokens.go"
// line 129 "scan_tokens.go"
{
var _klen int
var _trans int
@ -128,7 +141,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
ts = p
// line 138 "scan_tokens.go"
// line 150 "scan_tokens.go"
}
}
@ -204,49 +217,64 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
te = p + 1
case 3:
// line 43 "scan_tokens.rl"
// line 46 "scan_tokens.rl"
te = p + 1
{
token(TokenInvalid)
}
case 4:
// line 44 "scan_tokens.rl"
// line 47 "scan_tokens.rl"
te = p + 1
{
token(TokenBadUTF8)
}
case 5:
// line 41 "scan_tokens.rl"
// line 43 "scan_tokens.rl"
te = p
p--
case 6:
// line 42 "scan_tokens.rl"
// line 44 "scan_tokens.rl"
te = p
p--
{
token(TokenNumberLit)
}
case 7:
// line 45 "scan_tokens.rl"
te = p
p--
{
token(TokenTabs)
}
case 7:
// line 44 "scan_tokens.rl"
case 8:
// line 47 "scan_tokens.rl"
te = p
p--
{
token(TokenBadUTF8)
}
case 8:
case 9:
// line 44 "scan_tokens.rl"
p = (te) - 1
{
token(TokenNumberLit)
}
case 10:
// line 47 "scan_tokens.rl"
p = (te) - 1
{
token(TokenBadUTF8)
}
// line 245 "scan_tokens.go"
// line 268 "scan_tokens.go"
}
}
@ -262,7 +290,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
ts = 0
// line 260 "scan_tokens.go"
// line 283 "scan_tokens.go"
}
}
@ -282,7 +310,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
}
// line 71 "scan_tokens.rl"
// line 74 "scan_tokens.rl"
// If we fall out here without being in a final state then we've
// encountered something that the scanner can't match, which we'll

11
zcl/zclsyntax/scan_tokens.rl
View File

@ -30,6 +30,8 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
);
BrokenUTF8 = any - AnyUTF8;
NumberLit = digit (digit|'.'|('e'|'E') ('+'|'-')? digit)*;
# Tabs are not valid, but we accept them in the scanner and mark them
# as tokens so that we can produce diagnostics advising the user to
# use spaces instead.
@ -38,10 +40,11 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
Spaces = ' '+;
main := |*
Spaces => {};
Tabs => { token(TokenTabs) };
AnyUTF8 => { token(TokenInvalid) };
BrokenUTF8 => { token(TokenBadUTF8) };
Spaces => {};
NumberLit => { token(TokenNumberLit) };
Tabs => { token(TokenTabs) };
AnyUTF8 => { token(TokenInvalid) };
BrokenUTF8 => { token(TokenBadUTF8) };
*|;
}%%

110
zcl/zclsyntax/scan_tokens_test.go
View File

@ -13,6 +13,7 @@ func TestScanTokens(t *testing.T) {
input string
want []Token
}{
// Empty input
{
``,
[]Token{
@ -39,6 +40,115 @@ func TestScanTokens(t *testing.T) {
},
},
},
// TokenNumberLit
{
`1`,
[]Token{
{
Type: TokenNumberLit,
Bytes: []byte(`1`),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
},
},
},
},
{
`12`,
[]Token{
{
Type: TokenNumberLit,
Bytes: []byte(`12`),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 2, Line: 1, Column: 3},
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
},
},
},
},
{
`12.3`,
[]Token{
{
Type: TokenNumberLit,
Bytes: []byte(`12.3`),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
},
},
},
},
{
`1e2`,
[]Token{
{
Type: TokenNumberLit,
Bytes: []byte(`1e2`),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
},
},
},
},
{
`1e+2`,
[]Token{
{
Type: TokenNumberLit,
Bytes: []byte(`1e+2`),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
},
},
},
},
// Invalid things
{
`|`,
[]Token{