diff --git a/zcl/json/scanner.go b/zcl/json/scanner.go index 077b52b..c399443 100644 --- a/zcl/json/scanner.go +++ b/zcl/json/scanner.go @@ -19,6 +19,7 @@ const ( tokenKeyword tokenType = 'K' tokenString tokenType = 'S' tokenNumber tokenType = 'N' + tokenEOF tokenType = '␄' tokenInvalid tokenType = 0 ) @@ -40,12 +41,22 @@ func scan(buf []byte, start pos) []token { p := start for { if len(buf) == 0 { + tokens = append(tokens, token{ + Type: tokenEOF, + Bytes: nil, + Range: posRange(p, p), + }) return tokens } buf, p = skipWhitespace(buf, p) if len(buf) == 0 { + tokens = append(tokens, token{ + Type: tokenEOF, + Bytes: nil, + Range: posRange(p, p), + }) return tokens } diff --git a/zcl/json/scanner_test.go b/zcl/json/scanner_test.go index 205865f..e5471c2 100644 --- a/zcl/json/scanner_test.go +++ b/zcl/json/scanner_test.go @@ -16,7 +16,43 @@ func TestScan(t *testing.T) { }{ { ``, - nil, + []token{ + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 0, + Line: 1, + Column: 1, + }, + End: zcl.Pos{ + Byte: 0, + Line: 1, + Column: 1, + }, + }, + }, + }, + }, + { + ` `, + []token{ + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 3, + Line: 1, + Column: 4, + }, + End: zcl.Pos{ + Byte: 3, + Line: 1, + Column: 4, + }, + }, + }, + }, }, { `{}`, @@ -53,6 +89,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 2, + Line: 1, + Column: 3, + }, + End: zcl.Pos{ + Byte: 2, + Line: 1, + Column: 3, + }, + }, + }, }, }, { @@ -90,6 +141,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 2, + Line: 1, + Column: 3, + }, + End: zcl.Pos{ + Byte: 2, + Line: 1, + Column: 3, + }, + }, + }, }, }, { @@ -127,6 +193,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 2, + Line: 1, + Column: 3, + }, + End: zcl.Pos{ + Byte: 2, + Line: 1, + Column: 3, + }, + }, + }, }, }, { @@ -148,6 +229,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 1, + Line: 1, + Column: 2, + }, + End: zcl.Pos{ + Byte: 1, + Line: 1, + Column: 2, + }, + }, + }, }, }, { @@ -169,6 +265,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 3, + Line: 1, + Column: 4, + }, + End: zcl.Pos{ + Byte: 3, + Line: 1, + Column: 4, + }, + }, + }, }, }, { @@ -190,6 +301,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 4, + Line: 1, + Column: 5, + }, + End: zcl.Pos{ + Byte: 4, + Line: 1, + Column: 5, + }, + }, + }, }, }, { @@ -227,6 +353,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 3, + Line: 1, + Column: 4, + }, + End: zcl.Pos{ + Byte: 3, + Line: 1, + Column: 4, + }, + }, + }, }, }, { @@ -264,6 +405,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 5, + Line: 3, + Column: 3, + }, + End: zcl.Pos{ + Byte: 5, + Line: 3, + Column: 3, + }, + }, + }, }, }, { @@ -301,6 +457,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 6, + Line: 1, + Column: 7, + }, + End: zcl.Pos{ + Byte: 6, + Line: 1, + Column: 7, + }, + }, + }, }, }, { @@ -322,6 +493,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 4, + Line: 1, + Column: 5, + }, + End: zcl.Pos{ + Byte: 4, + Line: 1, + Column: 5, + }, + }, + }, }, }, { @@ -343,6 +529,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 2, + Line: 1, + Column: 3, + }, + End: zcl.Pos{ + Byte: 2, + Line: 1, + Column: 3, + }, + }, + }, }, }, { @@ -364,6 +565,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 7, + Line: 1, + Column: 8, + }, + End: zcl.Pos{ + Byte: 7, + Line: 1, + Column: 8, + }, + }, + }, }, }, { @@ -385,6 +601,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 9, + Line: 1, + Column: 10, + }, + End: zcl.Pos{ + Byte: 9, + Line: 1, + Column: 10, + }, + }, + }, }, }, { @@ -422,6 +653,21 @@ func TestScan(t *testing.T) { }, }, }, + { + Type: tokenEOF, + Range: zcl.Range{ + Start: zcl.Pos{ + Byte: 11, + Line: 1, + Column: 12, + }, + End: zcl.Pos{ + Byte: 11, + Line: 1, + Column: 12, + }, + }, + }, }, }, } diff --git a/zcl/json/tokentype_string.go b/zcl/json/tokentype_string.go index ed1b57e..e81b137 100644 --- a/zcl/json/tokentype_string.go +++ b/zcl/json/tokentype_string.go @@ -4,55 +4,25 @@ package json import "fmt" -const ( - _tokenType_name_0 = "tokenInvalid" - _tokenType_name_1 = "tokenComma" - _tokenType_name_2 = "tokenColon" - _tokenType_name_3 = "tokenKeyword" - _tokenType_name_4 = "tokenNumber" - _tokenType_name_5 = "tokenString" - _tokenType_name_6 = "tokenBrackO" - _tokenType_name_7 = "tokenBrackC" - _tokenType_name_8 = "tokenBraceO" - _tokenType_name_9 = "tokenBraceC" -) +const _tokenType_name = "tokenInvalidtokenCommatokenColontokenKeywordtokenNumbertokenStringtokenBrackOtokenBrackCtokenBraceOtokenBraceCtokenEOF" -var ( - _tokenType_index_0 = [...]uint8{0, 12} - _tokenType_index_1 = [...]uint8{0, 10} - _tokenType_index_2 = [...]uint8{0, 10} - _tokenType_index_3 = [...]uint8{0, 12} - _tokenType_index_4 = [...]uint8{0, 11} - _tokenType_index_5 = [...]uint8{0, 11} - _tokenType_index_6 = [...]uint8{0, 11} - _tokenType_index_7 = [...]uint8{0, 11} - _tokenType_index_8 = [...]uint8{0, 11} - _tokenType_index_9 = [...]uint8{0, 11} -) +var _tokenType_map = map[tokenType]string{ + 0: _tokenType_name[0:12], + 44: _tokenType_name[12:22], + 58: _tokenType_name[22:32], + 75: _tokenType_name[32:44], + 78: _tokenType_name[44:55], + 83: _tokenType_name[55:66], + 91: _tokenType_name[66:77], + 93: _tokenType_name[77:88], + 123: _tokenType_name[88:99], + 125: _tokenType_name[99:110], + 9220: _tokenType_name[110:118], +} func (i tokenType) String() string { - switch { - case i == 0: - return _tokenType_name_0 - case i == 44: - return _tokenType_name_1 - case i == 58: - return _tokenType_name_2 - case i == 75: - return _tokenType_name_3 - case i == 78: - return _tokenType_name_4 - case i == 83: - return _tokenType_name_5 - case i == 91: - return _tokenType_name_6 - case i == 93: - return _tokenType_name_7 - case i == 123: - return _tokenType_name_8 - case i == 125: - return _tokenType_name_9 - default: - return fmt.Sprintf("tokenType(%d)", i) + if str, ok := _tokenType_map[i]; ok { + return str } + return fmt.Sprintf("tokenType(%d)", i) }