json: Generate EOF tokens marking the end of scanning

This helps the parser include a good position when it needs to generate
"unexpected EOF" diagnostics.

zcl/json/scanner.go

@@ -19,6 +19,7 @@ const (
 	tokenKeyword tokenType = 'K'
 	tokenString  tokenType = 'S'
 	tokenNumber  tokenType = 'N'
+	tokenEOF     tokenType = '␄'
 	tokenInvalid tokenType = 0
 )
 
@@ -40,12 +41,22 @@ func scan(buf []byte, start pos) []token {
 	p := start
 	for {
 		if len(buf) == 0 {
+			tokens = append(tokens, token{
+				Type:  tokenEOF,
+				Bytes: nil,
+				Range: posRange(p, p),
+			})
 			return tokens
 		}
 
 		buf, p = skipWhitespace(buf, p)
 
 		if len(buf) == 0 {
+			tokens = append(tokens, token{
+				Type:  tokenEOF,
+				Bytes: nil,
+				Range: posRange(p, p),
+			})
 			return tokens
 		}
 

zcl/json/scanner_test.go

@@ -16,7 +16,43 @@ func TestScan(t *testing.T) {
 	}{
 		{
 			``,
-			nil,
+			[]token{
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   0,
+							Line:   1,
+							Column: 1,
+						},
+						End: zcl.Pos{
+							Byte:   0,
+							Line:   1,
+							Column: 1,
+						},
+					},
+				},
+			},
+		},
+		{
+			`   `,
+			[]token{
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   3,
+							Line:   1,
+							Column: 4,
+						},
+						End: zcl.Pos{
+							Byte:   3,
+							Line:   1,
+							Column: 4,
+						},
+					},
+				},
+			},
 		},
 		{
 			`{}`,
@@ -53,6 +89,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   2,
+							Line:   1,
+							Column: 3,
+						},
+						End: zcl.Pos{
+							Byte:   2,
+							Line:   1,
+							Column: 3,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -90,6 +141,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   2,
+							Line:   1,
+							Column: 3,
+						},
+						End: zcl.Pos{
+							Byte:   2,
+							Line:   1,
+							Column: 3,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -127,6 +193,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   2,
+							Line:   1,
+							Column: 3,
+						},
+						End: zcl.Pos{
+							Byte:   2,
+							Line:   1,
+							Column: 3,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -148,6 +229,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   1,
+							Line:   1,
+							Column: 2,
+						},
+						End: zcl.Pos{
+							Byte:   1,
+							Line:   1,
+							Column: 2,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -169,6 +265,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   3,
+							Line:   1,
+							Column: 4,
+						},
+						End: zcl.Pos{
+							Byte:   3,
+							Line:   1,
+							Column: 4,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -190,6 +301,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   4,
+							Line:   1,
+							Column: 5,
+						},
+						End: zcl.Pos{
+							Byte:   4,
+							Line:   1,
+							Column: 5,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -227,6 +353,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   3,
+							Line:   1,
+							Column: 4,
+						},
+						End: zcl.Pos{
+							Byte:   3,
+							Line:   1,
+							Column: 4,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -264,6 +405,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   5,
+							Line:   3,
+							Column: 3,
+						},
+						End: zcl.Pos{
+							Byte:   5,
+							Line:   3,
+							Column: 3,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -301,6 +457,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   6,
+							Line:   1,
+							Column: 7,
+						},
+						End: zcl.Pos{
+							Byte:   6,
+							Line:   1,
+							Column: 7,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -322,6 +493,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   4,
+							Line:   1,
+							Column: 5,
+						},
+						End: zcl.Pos{
+							Byte:   4,
+							Line:   1,
+							Column: 5,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -343,6 +529,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   2,
+							Line:   1,
+							Column: 3,
+						},
+						End: zcl.Pos{
+							Byte:   2,
+							Line:   1,
+							Column: 3,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -364,6 +565,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   7,
+							Line:   1,
+							Column: 8,
+						},
+						End: zcl.Pos{
+							Byte:   7,
+							Line:   1,
+							Column: 8,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -385,6 +601,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   9,
+							Line:   1,
+							Column: 10,
+						},
+						End: zcl.Pos{
+							Byte:   9,
+							Line:   1,
+							Column: 10,
+						},
+					},
+				},
 			},
 		},
 		{
@@ -422,6 +653,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: zcl.Range{
+						Start: zcl.Pos{
+							Byte:   11,
+							Line:   1,
+							Column: 12,
+						},
+						End: zcl.Pos{
+							Byte:   11,
+							Line:   1,
+							Column: 12,
+						},
+					},
+				},
 			},
 		},
 	}

zcl/json/tokentype_string.go

@@ -4,55 +4,25 @@ package json
 
 import "fmt"
 
-const (
+const _tokenType_name = "tokenInvalidtokenCommatokenColontokenKeywordtokenNumbertokenStringtokenBrackOtokenBrackCtokenBraceOtokenBraceCtokenEOF"
-	_tokenType_name_0 = "tokenInvalid"
-	_tokenType_name_1 = "tokenComma"
-	_tokenType_name_2 = "tokenColon"
-	_tokenType_name_3 = "tokenKeyword"
-	_tokenType_name_4 = "tokenNumber"
-	_tokenType_name_5 = "tokenString"
-	_tokenType_name_6 = "tokenBrackO"
-	_tokenType_name_7 = "tokenBrackC"
-	_tokenType_name_8 = "tokenBraceO"
-	_tokenType_name_9 = "tokenBraceC"
-)
 
-var (
+var _tokenType_map = map[tokenType]string{
-	_tokenType_index_0 = [...]uint8{0, 12}
+	0:    _tokenType_name[0:12],
-	_tokenType_index_1 = [...]uint8{0, 10}
+	44:   _tokenType_name[12:22],
-	_tokenType_index_2 = [...]uint8{0, 10}
+	58:   _tokenType_name[22:32],
-	_tokenType_index_3 = [...]uint8{0, 12}
+	75:   _tokenType_name[32:44],
-	_tokenType_index_4 = [...]uint8{0, 11}
+	78:   _tokenType_name[44:55],
-	_tokenType_index_5 = [...]uint8{0, 11}
+	83:   _tokenType_name[55:66],
-	_tokenType_index_6 = [...]uint8{0, 11}
+	91:   _tokenType_name[66:77],
-	_tokenType_index_7 = [...]uint8{0, 11}
+	93:   _tokenType_name[77:88],
-	_tokenType_index_8 = [...]uint8{0, 11}
+	123:  _tokenType_name[88:99],
-	_tokenType_index_9 = [...]uint8{0, 11}
+	125:  _tokenType_name[99:110],
-)
+	9220: _tokenType_name[110:118],
+}
 
 func (i tokenType) String() string {
-	switch {
+	if str, ok := _tokenType_map[i]; ok {
-	case i == 0:
+		return str
-		return _tokenType_name_0
-	case i == 44:
-		return _tokenType_name_1
-	case i == 58:
-		return _tokenType_name_2
-	case i == 75:
-		return _tokenType_name_3
-	case i == 78:
-		return _tokenType_name_4
-	case i == 83:
-		return _tokenType_name_5
-	case i == 91:
-		return _tokenType_name_6
-	case i == 93:
-		return _tokenType_name_7
-	case i == 123:
-		return _tokenType_name_8
-	case i == 125:
-		return _tokenType_name_9
-	default:
-		return fmt.Sprintf("tokenType(%d)", i)
 	}
+	return fmt.Sprintf("tokenType(%d)", i)
 }