go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zclsyntax: allow scanner to support multiple modes

Browse Source 
A scanner "mode" decides which state it starts in, allowing us to start
in template mode for parsing top-level templates. However, currently the
only mode implemented is "normal" mode, which is the behavior we had
before.
This commit is contained in:
Martin Atkins 2017-05-28 15:44:22 -07:00
parent da874ba0ff
commit 4a939a2b46
4 changed files with 44 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
zcl/zclsyntax/scan_tokens.go
View File

@ -100,7 +100,7 @@ var _zcltok_key_offsets []int16 = []int16{
var _zcltok_trans_keys []byte = []byte{
43, 45, 48, 57, 48, 57, 170, 181,
183, 186, 128, 150, 152, 182, 184, 255,
192, 255, 0, 127, 173, 130, 133, 146,
192, 255, 128, 255, 173, 130, 133, 146,
159, 165, 171, 175, 255, 181, 190, 184,
185, 192, 255, 140, 134, 138, 142, 161,
163, 255, 182, 130, 136, 137, 176, 151,
@ -821,7 +821,7 @@ var _zcltok_index_offsets []int16 = []int16{
var _zcltok_indicies []int16 = []int16{
1, 1, 2, 0, 2, 0, 4, 4,
4, 4, 3, 4, 4, 4, 3, 4,
3, 4, 3, 4, 3, 3, 3, 3,
3, 4, 4, 3, 3, 3, 3, 3,
3, 4, 3, 3, 3, 3, 4, 4,
4, 4, 4, 3, 3, 4, 3, 3,
4, 3, 4, 3, 3, 4, 3, 3,
@ -1605,7 +1605,7 @@ const zcltok_en_main int = 521
// line 13 "scan_tokens.rl"
func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []Token {
f := &tokenAccum{
Filename: filename,
Bytes: data,
@ -1615,7 +1615,6 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
// line 156 "scan_tokens.rl"
// Ragel state
cs := 0 // Current State
p := 0 // "Pointer" into data
pe := len(data) // End-of-data "pointer"
ts := 0
@ -1625,10 +1624,22 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
var stack []int
var top int
var cs int // current state
switch mode {
case scanNormal:
cs = zcltok_en_main
case scanTemplate:
// scanTemplate is a variant of heredoc scanning, so will
// be implemented once that is implemented.
panic("scanTemplate not yet implemented")
default:
panic("invalid scanMode")
}
braces := 0
var retBraces []int // stack of brace levels that cause us to use fret
// line 179 "scan_tokens.rl"
// line 190 "scan_tokens.rl"
// Make Go compiler happy
_ = ts
@ -1648,16 +1659,15 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
f.emitToken(TokenType(b[0]), ts, te)
}
// line 1660 "scan_tokens.go"
// line 1671 "scan_tokens.go"
{
cs = zcltok_start
top = 0
ts = 0
te = 0
act = 0
}
// line 1669 "scan_tokens.go"
// line 1679 "scan_tokens.go"
{
var _klen int
var _trans int
@ -1679,7 +1689,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
ts = p
// line 1690 "scan_tokens.go"
// line 1700 "scan_tokens.go"
}
}
@ -2100,7 +2110,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
}
}
// line 2042 "scan_tokens.go"
// line 2052 "scan_tokens.go"
}
}
@ -2116,7 +2126,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
ts = 0
// line 2057 "scan_tokens.go"
// line 2067 "scan_tokens.go"
}
}
@ -2136,7 +2146,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
}
// line 202 "scan_tokens.rl"
// line 213 "scan_tokens.rl"
// If we fall out here without being in a final state then we've
// encountered something that the scanner can't match, which we'll

17
zcl/zclsyntax/scan_tokens.rl
View File

@ -12,7 +12,7 @@ import (
write data;
}%%
func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []Token {
f := &tokenAccum{
Filename: filename,
Bytes: data,
@ -156,7 +156,6 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
}%%
// Ragel state
cs := 0 // Current State
p := 0 // "Pointer" into data
pe := len(data) // End-of-data "pointer"
ts := 0
@ -166,6 +165,18 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
var stack []int
var top int
var cs int // current state
switch mode {
case scanNormal:
cs = zcltok_en_main
case scanTemplate:
// scanTemplate is a variant of heredoc scanning, so will
// be implemented once that is implemented.
panic("scanTemplate not yet implemented")
default:
panic("invalid scanMode")
}
braces := 0
var retBraces []int // stack of brace levels that cause us to use fret
@ -197,7 +208,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
}
%%{
write init;
write init nocs;
write exec;
}%%

2
zcl/zclsyntax/scan_tokens_test.go
View File

@ -884,7 +884,7 @@ func TestScanTokens(t *testing.T) {
for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
got := scanTokens([]byte(test.input), "", zcl.Pos{Byte: 0, Line: 1, Column: 1})
got := scanTokens([]byte(test.input), "", zcl.Pos{Byte: 0, Line: 1, Column: 1}, scanNormal)
if !reflect.DeepEqual(got, test.want) {
diff := prettyConfig.Compare(test.want, got)

7
zcl/zclsyntax/token.go
View File

@ -89,6 +89,13 @@ func (t TokenType) GoString() string {
return fmt.Sprintf("zclsyntax.%s", t.String())
}
type scanMode int
const (
scanNormal scanMode = iota
scanTemplate
)
type tokenAccum struct {
Filename string
Bytes []byte