json: Parsing of strings and objects

zcl/json/parser.go

@@ -1,6 +1,7 @@
 package json
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/apparentlymart/go-zcl/zcl"
@@ -16,7 +17,12 @@ func parseFileContent(buf []byte, filename string) (node, zcl.Diagnostics) {
 		},
 	})
 	p := newPeeker(tokens)
-	return parseValue(p)
+	node, diags := parseValue(p)
+	if diags.HasErrors() {
+		// Don't return a node if there were errors during parsing.
+		return nil, diags
+	}
+	return node, diags
 }
 
 func parseValue(p *peeker) (node, zcl.Diagnostics) {
@@ -103,7 +109,7 @@ Token:
 		key := keyStrNode.Value
 
 		colon := p.Read()
-		if colon.Type != tokenComma {
+		if colon.Type != tokenColon {
 			if colon.Type == tokenBraceC || colon.Type == tokenComma {
 				// Catch common mistake of using braces instead of brackets
 				// for an array.
@@ -129,6 +135,19 @@ Token:
 			return nil, diags
 		}
 
+		if existing := attrs[key]; existing != nil {
+			// Generate a diagnostic for the duplicate key, but continue parsing
+			// anyway since this is a semantic error we can recover from.
+			diags = diags.Append(&zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  "Duplicate object attribute",
+				Detail: fmt.Sprintf(
+					"An attribute named %q was previously introduced at %s",
+					key, existing.NameRange.String(),
+				),
+				Subject: &colon.Range,
+			})
+		}
 		attrs[key] = &objectAttr{
 			Name:      key,
 			Value:     valNode,
@@ -148,6 +167,20 @@ Token:
 				})
 			}
 			continue Token
+		case tokenEOF:
+			return nil, diags.Append(&zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  "Unclosed object",
+				Detail:   "No closing brace was found for this JSON object.",
+				Subject:  &open.Range,
+			})
+		case tokenBrackC:
+			return nil, diags.Append(&zcl.Diagnostic{
+				Severity: zcl.DiagError,
+				Summary:  "Mismatched braces",
+				Detail:   "A JSON object must be closed with a brace, not a bracket.",
+				Subject:  p.Peek().Range.Ptr(),
+			})
 		case tokenBraceC:
 			break Token
 		default:
@@ -178,7 +211,58 @@ func parseNumber(p *peeker) (node, zcl.Diagnostics) {
 }
 
 func parseString(p *peeker) (node, zcl.Diagnostics) {
-	return nil, nil
+	tok := p.Read()
+	var str string
+	err := json.Unmarshal(tok.Bytes, &str)
+
+	if err != nil {
+		var errRange zcl.Range
+		if serr, ok := err.(*json.SyntaxError); ok {
+			errOfs := serr.Offset
+			errPos := tok.Range.Start
+			errPos.Byte += int(errOfs)
+
+			// TODO: Use the byte offset to properly count unicode
+			// characters for the column, and mark the whole of the
+			// character that was wrong as part of our range.
+			errPos.Column += int(errOfs)
+
+			errEndPos := errPos
+			errEndPos.Byte++
+			errEndPos.Column++
+
+			errRange = zcl.Range{
+				Filename: tok.Range.Filename,
+				Start:    errPos,
+				End:      errEndPos,
+			}
+		} else {
+			errRange = tok.Range
+		}
+
+		var contextRange *zcl.Range
+		if errRange != tok.Range {
+			contextRange = &tok.Range
+		}
+
+		// FIXME: Eventually we should parse strings directly here so
+		// we can produce a more useful error message in the face fo things
+		// such as invalid escapes, etc.
+		return nil, zcl.Diagnostics{
+			{
+				Severity: zcl.DiagError,
+				Summary:  "Invalid JSON string",
+				Detail:   fmt.Sprintf("There is a syntax error in the given JSON string."),
+				Subject:  &errRange,
+				Context:  contextRange,
+			},
+		}
+	}
+
+	return &stringVal{
+		Value:    str,
+		SrcRange: tok.Range,
+	}, nil
 }
 
 func parseKeyword(p *peeker) (node, zcl.Diagnostics) {

zcl/json/parser_test.go

@@ -14,14 +14,14 @@ func TestParse(t *testing.T) {
 		Want      node
 		DiagCount int
 	}{
+		// Simple, single-token constructs
 		{
 			`true`,
 			&booleanVal{
 				Value: true,
 				SrcRange: zcl.Range{
-					Filename: "",
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
-					Start:    zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 5, Byte: 4},
-					End:      zcl.Pos{Line: 1, Column: 5, Byte: 4},
 				},
 			},
 			0,
@@ -31,9 +31,8 @@ func TestParse(t *testing.T) {
 			&booleanVal{
 				Value: false,
 				SrcRange: zcl.Range{
-					Filename: "",
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
-					Start:    zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 6, Byte: 5},
-					End:      zcl.Pos{Line: 1, Column: 6, Byte: 5},
 				},
 			},
 			0,
@@ -42,9 +41,8 @@ func TestParse(t *testing.T) {
 			`null`,
 			&nullVal{
 				SrcRange: zcl.Range{
-					Filename: "",
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
-					Start:    zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 5, Byte: 4},
-					End:      zcl.Pos{Line: 1, Column: 5, Byte: 4},
 				},
 			},
 			0,
@@ -59,6 +57,186 @@ func TestParse(t *testing.T) {
 			nil,
 			1,
 		},
+		{
+			`"hello"`,
+			&stringVal{
+				Value: "hello",
+				SrcRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 8, Byte: 7},
+				},
+			},
+			0,
+		},
+		{
+			`"hello\nworld"`,
+			&stringVal{
+				Value: "hello\nworld",
+				SrcRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 15, Byte: 14},
+				},
+			},
+			0,
+		},
+		{
+			`"hello \"world\""`,
+			&stringVal{
+				Value: `hello "world"`,
+				SrcRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 18, Byte: 17},
+				},
+			},
+			0,
+		},
+		{
+			`"hello \\"`,
+			&stringVal{
+				Value: "hello \\",
+				SrcRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 11, Byte: 10},
+				},
+			},
+			0,
+		},
+		{
+			`"hello`,
+			nil,
+			1,
+		},
+		{
+			`"he\llo"`,
+			nil,
+			1,
+		},
+
+		// Objects
+		{
+			`{"hello": true}`,
+			&objectVal{
+				Attrs: map[string]*objectAttr{
+					"hello": {
+						Name: "hello",
+						Value: &booleanVal{
+							Value: true,
+							SrcRange: zcl.Range{
+								Start: zcl.Pos{Line: 1, Column: 11, Byte: 10},
+								End:   zcl.Pos{Line: 1, Column: 15, Byte: 14},
+							},
+						},
+						NameRange: zcl.Range{
+							Start: zcl.Pos{Line: 1, Column: 2, Byte: 1},
+							End:   zcl.Pos{Line: 1, Column: 9, Byte: 8},
+						},
+					},
+				},
+				SrcRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 16, Byte: 15},
+				},
+				OpenRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 2, Byte: 1},
+				},
+			},
+			0,
+		},
+		{
+			`{"hello": true, "bye": false}`,
+			&objectVal{
+				Attrs: map[string]*objectAttr{
+					"hello": {
+						Name: "hello",
+						Value: &booleanVal{
+							Value: true,
+							SrcRange: zcl.Range{
+								Start: zcl.Pos{Line: 1, Column: 11, Byte: 10},
+								End:   zcl.Pos{Line: 1, Column: 15, Byte: 14},
+							},
+						},
+						NameRange: zcl.Range{
+							Start: zcl.Pos{Line: 1, Column: 2, Byte: 1},
+							End:   zcl.Pos{Line: 1, Column: 9, Byte: 8},
+						},
+					},
+					"bye": {
+						Name: "bye",
+						Value: &booleanVal{
+							Value: false,
+							SrcRange: zcl.Range{
+								Start: zcl.Pos{Line: 1, Column: 24, Byte: 23},
+								End:   zcl.Pos{Line: 1, Column: 29, Byte: 28},
+							},
+						},
+						NameRange: zcl.Range{
+							Start: zcl.Pos{Line: 1, Column: 17, Byte: 16},
+							End:   zcl.Pos{Line: 1, Column: 22, Byte: 21},
+						},
+					},
+				},
+				SrcRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 30, Byte: 29},
+				},
+				OpenRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 2, Byte: 1},
+				},
+			},
+			0,
+		},
+		{
+			`{}`,
+			&objectVal{
+				Attrs: map[string]*objectAttr{},
+				SrcRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 3, Byte: 2},
+				},
+				OpenRange: zcl.Range{
+					Start: zcl.Pos{Line: 1, Column: 1, Byte: 0},
+					End:   zcl.Pos{Line: 1, Column: 2, Byte: 1},
+				},
+			},
+			0,
+		},
+		{
+			`{"hello":true`,
+			nil,
+			1,
+		},
+		{
+			`{"hello":true]`,
+			nil,
+			1,
+		},
+		{
+			`{"hello":true,}`,
+			nil,
+			1,
+		},
+		{
+			`{true:false}`,
+			nil,
+			1,
+		},
+		{
+			`{"hello": true, "hello": true}`,
+			nil,
+			1,
+		},
+		{
+			`{"hello": true, "hello": true, "hello", true}`,
+			nil,
+			2,
+		},
+		{
+			`{"hello", "world"}`,
+			nil,
+			1,
+		},
 	}
 
 	for _, test := range tests {
@@ -66,13 +244,16 @@ func TestParse(t *testing.T) {
 			got, diag := parseFileContent([]byte(test.Input), "")
 
 			if len(diag) != test.DiagCount {
-				t.Errorf("got %d diagnostics; want %d\n%s", len(diag), test.DiagCount, spew.Sdump(diag))
+				t.Errorf("got %d diagnostics; want %d", len(diag), test.DiagCount)
+				for _, d := range diag {
+					t.Logf("  - %s", d.Error())
+				}
 			}
 
 			if !reflect.DeepEqual(got, test.Want) {
 				t.Errorf(
-					"wrong result\ninput: %s\ngot:   %#v\nwant:  %#v",
+					"wrong result\ninput: %s\ngot:   %s\nwant:  %s",
-					test.Input, got, test.Want,
+					test.Input, spew.Sdump(got), spew.Sdump(test.Want),
 				)
 			}
 		})