diff --git a/zcl/zclsyntax/scan_tokens.go b/zcl/zclsyntax/scan_tokens.go index e24accb..4a0abea 100644 --- a/zcl/zclsyntax/scan_tokens.go +++ b/zcl/zclsyntax/scan_tokens.go @@ -11,62 +11,75 @@ import ( var _zcltok_actions []byte = []byte{ 0, 1, 0, 1, 1, 1, 2, 1, 3, 1, 4, 1, 5, 1, 6, 1, 7, - 1, 8, + 1, 8, 1, 9, 1, 10, } var _zcltok_key_offsets []byte = []byte{ - 0, 2, 4, 16, 17, 18, 20, 22, + 0, 4, 6, 8, 10, 24, 25, 26, + 31, 33, 35, } var _zcltok_trans_keys []byte = []byte{ - 128, 191, 128, 191, 9, 32, 128, 191, + 43, 45, 48, 57, 48, 57, 128, 191, + 128, 191, 9, 32, 48, 57, 128, 191, 192, 223, 224, 239, 240, 247, 248, 255, - 9, 32, 128, 191, 128, 191, 128, 191, + 9, 32, 46, 69, 101, 48, 57, 128, + 191, 128, 191, 128, 191, } var _zcltok_single_lengths []byte = []byte{ - 0, 0, 2, 1, 1, 0, 0, 0, + 2, 0, 0, 0, 2, 1, 1, 3, + 0, 0, 0, } var _zcltok_range_lengths []byte = []byte{ - 1, 1, 5, 0, 0, 1, 1, 1, + 1, 1, 1, 1, 6, 0, 0, 1, + 1, 1, 1, } var _zcltok_index_offsets []byte = []byte{ - 0, 2, 4, 12, 14, 16, 18, 20, + 0, 4, 6, 8, 10, 19, 21, 23, + 28, 30, 32, } var _zcltok_trans_targs []byte = []byte{ - 2, 2, 0, 2, 3, 4, 2, 5, - 6, 7, 2, 2, 3, 2, 4, 2, - 2, 2, 0, 2, 1, 2, 2, 2, - 2, 2, 2, 2, 2, + 1, 1, 7, 4, 7, 4, 4, 4, + 2, 4, 5, 6, 7, 4, 8, 9, + 10, 4, 4, 5, 4, 6, 4, 7, + 0, 0, 7, 4, 4, 4, 2, 4, + 3, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, } var _zcltok_trans_actions []byte = []byte{ - 7, 17, 0, 17, 0, 0, 9, 0, - 5, 5, 9, 7, 0, 13, 0, 11, - 7, 15, 0, 15, 0, 15, 17, 17, - 13, 11, 15, 15, 15, + 0, 0, 5, 19, 5, 19, 7, 21, + 0, 21, 0, 0, 5, 9, 0, 5, + 5, 9, 7, 0, 15, 0, 11, 5, + 0, 0, 5, 13, 7, 17, 0, 17, + 0, 17, 19, 19, 21, 21, 15, 11, + 13, 17, 17, 17, } var _zcltok_to_state_actions []byte = []byte{ - 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, } var _zcltok_from_state_actions []byte = []byte{ - 0, 0, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 0, } var _zcltok_eof_trans []byte = []byte{ - 24, 24, 0, 25, 26, 29, 29, 29, + 36, 36, 38, 38, 0, 39, 40, 41, + 44, 44, 44, } -const zcltok_start int = 2 -const zcltok_first_final int = 2 +const zcltok_start int = 4 +const zcltok_first_final int = 4 const zcltok_error int = -1 -const zcltok_en_main int = 2 +const zcltok_en_main int = 4 // line 13 "scan_tokens.rl" @@ -77,7 +90,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token { Pos: start, } - // line 47 "scan_tokens.rl" + // line 50 "scan_tokens.rl" // Ragel state cs := 0 // Current State @@ -98,7 +111,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token { f.emitToken(ty, ts, te) } - // line 109 "scan_tokens.go" + // line 121 "scan_tokens.go" { cs = zcltok_start ts = 0 @@ -106,7 +119,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token { act = 0 } - // line 117 "scan_tokens.go" + // line 129 "scan_tokens.go" { var _klen int var _trans int @@ -128,7 +141,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token { ts = p - // line 138 "scan_tokens.go" + // line 150 "scan_tokens.go" } } @@ -204,49 +217,64 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token { te = p + 1 case 3: - // line 43 "scan_tokens.rl" + // line 46 "scan_tokens.rl" te = p + 1 { token(TokenInvalid) } case 4: - // line 44 "scan_tokens.rl" + // line 47 "scan_tokens.rl" te = p + 1 { token(TokenBadUTF8) } case 5: - // line 41 "scan_tokens.rl" + // line 43 "scan_tokens.rl" te = p p-- case 6: - // line 42 "scan_tokens.rl" + // line 44 "scan_tokens.rl" + + te = p + p-- + { + token(TokenNumberLit) + } + case 7: + // line 45 "scan_tokens.rl" te = p p-- { token(TokenTabs) } - case 7: - // line 44 "scan_tokens.rl" + case 8: + // line 47 "scan_tokens.rl" te = p p-- { token(TokenBadUTF8) } - case 8: + case 9: // line 44 "scan_tokens.rl" + p = (te) - 1 + { + token(TokenNumberLit) + } + case 10: + // line 47 "scan_tokens.rl" + p = (te) - 1 { token(TokenBadUTF8) } - // line 245 "scan_tokens.go" + // line 268 "scan_tokens.go" } } @@ -262,7 +290,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token { ts = 0 - // line 260 "scan_tokens.go" + // line 283 "scan_tokens.go" } } @@ -282,7 +310,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token { } - // line 71 "scan_tokens.rl" + // line 74 "scan_tokens.rl" // If we fall out here without being in a final state then we've // encountered something that the scanner can't match, which we'll diff --git a/zcl/zclsyntax/scan_tokens.rl b/zcl/zclsyntax/scan_tokens.rl index 43f48fb..7fcc62d 100644 --- a/zcl/zclsyntax/scan_tokens.rl +++ b/zcl/zclsyntax/scan_tokens.rl @@ -30,6 +30,8 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token { ); BrokenUTF8 = any - AnyUTF8; + NumberLit = digit (digit|'.'|('e'|'E') ('+'|'-')? digit)*; + # Tabs are not valid, but we accept them in the scanner and mark them # as tokens so that we can produce diagnostics advising the user to # use spaces instead. @@ -38,10 +40,11 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token { Spaces = ' '+; main := |* - Spaces => {}; - Tabs => { token(TokenTabs) }; - AnyUTF8 => { token(TokenInvalid) }; - BrokenUTF8 => { token(TokenBadUTF8) }; + Spaces => {}; + NumberLit => { token(TokenNumberLit) }; + Tabs => { token(TokenTabs) }; + AnyUTF8 => { token(TokenInvalid) }; + BrokenUTF8 => { token(TokenBadUTF8) }; *|; }%% diff --git a/zcl/zclsyntax/scan_tokens_test.go b/zcl/zclsyntax/scan_tokens_test.go index 72225f4..d138598 100644 --- a/zcl/zclsyntax/scan_tokens_test.go +++ b/zcl/zclsyntax/scan_tokens_test.go @@ -13,6 +13,7 @@ func TestScanTokens(t *testing.T) { input string want []Token }{ + // Empty input { ``, []Token{ @@ -39,6 +40,115 @@ func TestScanTokens(t *testing.T) { }, }, }, + + // TokenNumberLit + { + `1`, + []Token{ + { + Type: TokenNumberLit, + Bytes: []byte(`1`), + Range: zcl.Range{ + Start: zcl.Pos{Byte: 0, Line: 1, Column: 1}, + End: zcl.Pos{Byte: 1, Line: 1, Column: 2}, + }, + }, + { + Type: TokenEOF, + Bytes: []byte{}, + Range: zcl.Range{ + Start: zcl.Pos{Byte: 1, Line: 1, Column: 2}, + End: zcl.Pos{Byte: 1, Line: 1, Column: 2}, + }, + }, + }, + }, + { + `12`, + []Token{ + { + Type: TokenNumberLit, + Bytes: []byte(`12`), + Range: zcl.Range{ + Start: zcl.Pos{Byte: 0, Line: 1, Column: 1}, + End: zcl.Pos{Byte: 2, Line: 1, Column: 3}, + }, + }, + { + Type: TokenEOF, + Bytes: []byte{}, + Range: zcl.Range{ + Start: zcl.Pos{Byte: 2, Line: 1, Column: 3}, + End: zcl.Pos{Byte: 2, Line: 1, Column: 3}, + }, + }, + }, + }, + { + `12.3`, + []Token{ + { + Type: TokenNumberLit, + Bytes: []byte(`12.3`), + Range: zcl.Range{ + Start: zcl.Pos{Byte: 0, Line: 1, Column: 1}, + End: zcl.Pos{Byte: 4, Line: 1, Column: 5}, + }, + }, + { + Type: TokenEOF, + Bytes: []byte{}, + Range: zcl.Range{ + Start: zcl.Pos{Byte: 4, Line: 1, Column: 5}, + End: zcl.Pos{Byte: 4, Line: 1, Column: 5}, + }, + }, + }, + }, + { + `1e2`, + []Token{ + { + Type: TokenNumberLit, + Bytes: []byte(`1e2`), + Range: zcl.Range{ + Start: zcl.Pos{Byte: 0, Line: 1, Column: 1}, + End: zcl.Pos{Byte: 3, Line: 1, Column: 4}, + }, + }, + { + Type: TokenEOF, + Bytes: []byte{}, + Range: zcl.Range{ + Start: zcl.Pos{Byte: 3, Line: 1, Column: 4}, + End: zcl.Pos{Byte: 3, Line: 1, Column: 4}, + }, + }, + }, + }, + { + `1e+2`, + []Token{ + { + Type: TokenNumberLit, + Bytes: []byte(`1e+2`), + Range: zcl.Range{ + Start: zcl.Pos{Byte: 0, Line: 1, Column: 1}, + End: zcl.Pos{Byte: 4, Line: 1, Column: 5}, + }, + }, + { + Type: TokenEOF, + Bytes: []byte{}, + Range: zcl.Range{ + Start: zcl.Pos{Byte: 4, Line: 1, Column: 5}, + End: zcl.Pos{Byte: 4, Line: 1, Column: 5}, + }, + }, + }, + }, + + // Invalid things { `|`, []Token{