go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

json: Generate EOF tokens marking the end of scanning

Browse Source 
This helps the parser include a good position when it needs to generate
"unexpected EOF" diagnostics.
This commit is contained in:
Martin Atkins 2017-05-15 07:15:53 -07:00
parent b5ce4360cd
commit 45d426b99c
3 changed files with 275 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
zcl/json/scanner.go
View File

@ -19,6 +19,7 @@ const (
tokenKeyword tokenType = 'K'
tokenString tokenType = 'S'
tokenNumber tokenType = 'N'
tokenEOF tokenType = '␄'
tokenInvalid tokenType = 0
)
@ -40,12 +41,22 @@ func scan(buf []byte, start pos) []token {
p := start
for {
if len(buf) == 0 {
tokens = append(tokens, token{
Type: tokenEOF,
Bytes: nil,
Range: posRange(p, p),
})
return tokens
}
buf, p = skipWhitespace(buf, p)
if len(buf) == 0 {
tokens = append(tokens, token{
Type: tokenEOF,
Bytes: nil,
Range: posRange(p, p),
})
return tokens
}

248
zcl/json/scanner_test.go
View File

@ -16,7 +16,43 @@ func TestScan(t *testing.T) {
}{
{
``,
nil,
[]token{
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 0,
Line: 1,
Column: 1,
},
End: zcl.Pos{
Byte: 0,
Line: 1,
Column: 1,
},
},
},
},
},
{
` `,
[]token{
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 3,
Line: 1,
Column: 4,
},
End: zcl.Pos{
Byte: 3,
Line: 1,
Column: 4,
},
},
},
},
},
{
`{}`,
@ -53,6 +89,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 2,
Line: 1,
Column: 3,
},
End: zcl.Pos{
Byte: 2,
Line: 1,
Column: 3,
},
},
},
},
},
{
@ -90,6 +141,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 2,
Line: 1,
Column: 3,
},
End: zcl.Pos{
Byte: 2,
Line: 1,
Column: 3,
},
},
},
},
},
{
@ -127,6 +193,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 2,
Line: 1,
Column: 3,
},
End: zcl.Pos{
Byte: 2,
Line: 1,
Column: 3,
},
},
},
},
},
{
@ -148,6 +229,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 1,
Line: 1,
Column: 2,
},
End: zcl.Pos{
Byte: 1,
Line: 1,
Column: 2,
},
},
},
},
},
{
@ -169,6 +265,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 3,
Line: 1,
Column: 4,
},
End: zcl.Pos{
Byte: 3,
Line: 1,
Column: 4,
},
},
},
},
},
{
@ -190,6 +301,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 4,
Line: 1,
Column: 5,
},
End: zcl.Pos{
Byte: 4,
Line: 1,
Column: 5,
},
},
},
},
},
{
@ -227,6 +353,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 3,
Line: 1,
Column: 4,
},
End: zcl.Pos{
Byte: 3,
Line: 1,
Column: 4,
},
},
},
},
},
{
@ -264,6 +405,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 5,
Line: 3,
Column: 3,
},
End: zcl.Pos{
Byte: 5,
Line: 3,
Column: 3,
},
},
},
},
},
{
@ -301,6 +457,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 6,
Line: 1,
Column: 7,
},
End: zcl.Pos{
Byte: 6,
Line: 1,
Column: 7,
},
},
},
},
},
{
@ -322,6 +493,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 4,
Line: 1,
Column: 5,
},
End: zcl.Pos{
Byte: 4,
Line: 1,
Column: 5,
},
},
},
},
},
{
@ -343,6 +529,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 2,
Line: 1,
Column: 3,
},
End: zcl.Pos{
Byte: 2,
Line: 1,
Column: 3,
},
},
},
},
},
{
@ -364,6 +565,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 7,
Line: 1,
Column: 8,
},
End: zcl.Pos{
Byte: 7,
Line: 1,
Column: 8,
},
},
},
},
},
{
@ -385,6 +601,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 9,
Line: 1,
Column: 10,
},
End: zcl.Pos{
Byte: 9,
Line: 1,
Column: 10,
},
},
},
},
},
{
@ -422,6 +653,21 @@ func TestScan(t *testing.T) {
},
},
},
{
Type: tokenEOF,
Range: zcl.Range{
Start: zcl.Pos{
Byte: 11,
Line: 1,
Column: 12,
},
End: zcl.Pos{
Byte: 11,
Line: 1,
Column: 12,
},
},
},
},
},
}

64
zcl/json/tokentype_string.go
View File

@ -4,55 +4,25 @@ package json
import "fmt"
const (
_tokenType_name_0 = "tokenInvalid"
_tokenType_name_1 = "tokenComma"
_tokenType_name_2 = "tokenColon"
_tokenType_name_3 = "tokenKeyword"
_tokenType_name_4 = "tokenNumber"
_tokenType_name_5 = "tokenString"
_tokenType_name_6 = "tokenBrackO"
_tokenType_name_7 = "tokenBrackC"
_tokenType_name_8 = "tokenBraceO"
_tokenType_name_9 = "tokenBraceC"
)
const _tokenType_name = "tokenInvalidtokenCommatokenColontokenKeywordtokenNumbertokenStringtokenBrackOtokenBrackCtokenBraceOtokenBraceCtokenEOF"
var (
_tokenType_index_0 = [...]uint8{0, 12}
_tokenType_index_1 = [...]uint8{0, 10}
_tokenType_index_2 = [...]uint8{0, 10}
_tokenType_index_3 = [...]uint8{0, 12}
_tokenType_index_4 = [...]uint8{0, 11}
_tokenType_index_5 = [...]uint8{0, 11}
_tokenType_index_6 = [...]uint8{0, 11}
_tokenType_index_7 = [...]uint8{0, 11}
_tokenType_index_8 = [...]uint8{0, 11}
_tokenType_index_9 = [...]uint8{0, 11}
)
var _tokenType_map = map[tokenType]string{
0: _tokenType_name[0:12],
44: _tokenType_name[12:22],
58: _tokenType_name[22:32],
75: _tokenType_name[32:44],
78: _tokenType_name[44:55],
83: _tokenType_name[55:66],
91: _tokenType_name[66:77],
93: _tokenType_name[77:88],
123: _tokenType_name[88:99],
125: _tokenType_name[99:110],
9220: _tokenType_name[110:118],
}
func (i tokenType) String() string {
switch {
case i == 0:
return _tokenType_name_0
case i == 44:
return _tokenType_name_1
case i == 58:
return _tokenType_name_2
case i == 75:
return _tokenType_name_3
case i == 78:
return _tokenType_name_4
case i == 83:
return _tokenType_name_5
case i == 91:
return _tokenType_name_6
case i == 93:
return _tokenType_name_7
case i == 123:
return _tokenType_name_8
case i == 125:
return _tokenType_name_9
default:
return fmt.Sprintf("tokenType(%d)", i)
if str, ok := _tokenType_map[i]; ok {
return str
}
return fmt.Sprintf("tokenType(%d)", i)
}