From 87a91d1019261ce436dd90f99ff1448228d24915 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 7 Nov 2015 18:14:32 -0800 Subject: [PATCH 01/13] json: scanner and token --- json/scanner/scanner.go | 449 +++++++++++++++++++++++++++++++++++ json/scanner/scanner_test.go | 411 ++++++++++++++++++++++++++++++++ json/token/position.go | 46 ++++ json/token/token.go | 139 +++++++++++ json/token/token_test.go | 57 +++++ 5 files changed, 1102 insertions(+) create mode 100644 json/scanner/scanner.go create mode 100644 json/scanner/scanner_test.go create mode 100644 json/token/position.go create mode 100644 json/token/token.go create mode 100644 json/token/token_test.go diff --git a/json/scanner/scanner.go b/json/scanner/scanner.go new file mode 100644 index 0000000..eca2c98 --- /dev/null +++ b/json/scanner/scanner.go @@ -0,0 +1,449 @@ +package scanner + +import ( + "bytes" + "fmt" + "os" + "unicode" + "unicode/utf8" + + "github.com/hashicorp/hcl/json/token" +) + +// eof represents a marker rune for the end of the reader. +const eof = rune(0) + +// Scanner defines a lexical scanner +type Scanner struct { + buf *bytes.Buffer // Source buffer for advancing and scanning + src []byte // Source buffer for immutable access + + // Source Position + srcPos token.Pos // current position + prevPos token.Pos // previous position, used for peek() method + + lastCharLen int // length of last character in bytes + lastLineLen int // length of last line in characters (for correct column reporting) + + tokStart int // token text start position + tokEnd int // token text end position + + // Error is called for each error encountered. If no Error + // function is set, the error is reported to os.Stderr. + Error func(pos token.Pos, msg string) + + // ErrorCount is incremented by one for each error encountered. + ErrorCount int + + // tokPos is the start position of most recently scanned token; set by + // Scan. The Filename field is always left untouched by the Scanner. If + // an error is reported (via Error) and Position is invalid, the scanner is + // not inside a token. + tokPos token.Pos +} + +// New creates and initializes a new instance of Scanner using src as +// its source content. +func New(src []byte) *Scanner { + // even though we accept a src, we read from a io.Reader compatible type + // (*bytes.Buffer). So in the future we might easily change it to streaming + // read. + b := bytes.NewBuffer(src) + s := &Scanner{ + buf: b, + src: src, + } + + // srcPosition always starts with 1 + s.srcPos.Line = 1 + return s +} + +// next reads the next rune from the bufferred reader. Returns the rune(0) if +// an error occurs (or io.EOF is returned). +func (s *Scanner) next() rune { + ch, size, err := s.buf.ReadRune() + if err != nil { + // advance for error reporting + s.srcPos.Column++ + s.srcPos.Offset += size + s.lastCharLen = size + return eof + } + + if ch == utf8.RuneError && size == 1 { + s.srcPos.Column++ + s.srcPos.Offset += size + s.lastCharLen = size + s.err("illegal UTF-8 encoding") + return ch + } + + // remember last position + s.prevPos = s.srcPos + + s.srcPos.Column++ + s.lastCharLen = size + s.srcPos.Offset += size + + if ch == '\n' { + s.srcPos.Line++ + s.lastLineLen = s.srcPos.Column + s.srcPos.Column = 0 + } + + // debug + // fmt.Printf("ch: %q, offset:column: %d:%d\n", ch, s.srcPos.Offset, s.srcPos.Column) + return ch +} + +// unread unreads the previous read Rune and updates the source position +func (s *Scanner) unread() { + if err := s.buf.UnreadRune(); err != nil { + panic(err) // this is user fault, we should catch it + } + s.srcPos = s.prevPos // put back last position +} + +// peek returns the next rune without advancing the reader. +func (s *Scanner) peek() rune { + peek, _, err := s.buf.ReadRune() + if err != nil { + return eof + } + + s.buf.UnreadRune() + return peek +} + +// Scan scans the next token and returns the token. +func (s *Scanner) Scan() token.Token { + ch := s.next() + + // skip white space + for isWhitespace(ch) { + ch = s.next() + } + + var tok token.Type + + // token text markings + s.tokStart = s.srcPos.Offset - s.lastCharLen + + // token position, initial next() is moving the offset by one(size of rune + // actually), though we are interested with the starting point + s.tokPos.Offset = s.srcPos.Offset - s.lastCharLen + if s.srcPos.Column > 0 { + // common case: last character was not a '\n' + s.tokPos.Line = s.srcPos.Line + s.tokPos.Column = s.srcPos.Column + } else { + // last character was a '\n' + // (we cannot be at the beginning of the source + // since we have called next() at least once) + s.tokPos.Line = s.srcPos.Line - 1 + s.tokPos.Column = s.lastLineLen + } + + switch { + case isLetter(ch): + lit := s.scanIdentifier() + if lit == "true" || lit == "false" { + tok = token.BOOL + } else if lit == "null" { + tok = token.NULL + } else { + s.err("illegal char") + } + case isDecimal(ch): + tok = s.scanNumber(ch) + default: + switch ch { + case eof: + tok = token.EOF + case '"': + tok = token.STRING + s.scanString() + case '.': + tok = token.PERIOD + ch = s.peek() + if isDecimal(ch) { + tok = token.FLOAT + ch = s.scanMantissa(ch) + ch = s.scanExponent(ch) + } + case '[': + tok = token.LBRACK + case ']': + tok = token.RBRACK + case '{': + tok = token.LBRACE + case '}': + tok = token.RBRACE + case ',': + tok = token.COMMA + case '-': + if isDecimal(s.peek()) { + ch := s.next() + tok = s.scanNumber(ch) + } else { + s.err("illegal char") + } + default: + s.err("illegal char") + } + } + + // finish token ending + s.tokEnd = s.srcPos.Offset + + // create token literal + var tokenText string + if s.tokStart >= 0 { + tokenText = string(s.src[s.tokStart:s.tokEnd]) + } + s.tokStart = s.tokEnd // ensure idempotency of tokenText() call + + return token.Token{ + Type: tok, + Pos: s.tokPos, + Text: tokenText, + } +} + +// scanNumber scans a HCL number definition starting with the given rune +func (s *Scanner) scanNumber(ch rune) token.Type { + zero := ch == '0' + pos := s.srcPos + + s.scanMantissa(ch) + ch = s.next() // seek forward + if ch == 'e' || ch == 'E' { + ch = s.scanExponent(ch) + return token.FLOAT + } + + if ch == '.' { + ch = s.scanFraction(ch) + if ch == 'e' || ch == 'E' { + ch = s.next() + ch = s.scanExponent(ch) + } + return token.FLOAT + } + + if ch != eof { + s.unread() + } + + // If we have a larger number and this is zero, error + if zero && pos != s.srcPos { + s.err("numbers cannot start with 0") + } + + return token.NUMBER +} + +// scanMantissa scans the mantissa begining from the rune. It returns the next +// non decimal rune. It's used to determine wheter it's a fraction or exponent. +func (s *Scanner) scanMantissa(ch rune) rune { + scanned := false + for isDecimal(ch) { + ch = s.next() + scanned = true + } + + if scanned && ch != eof { + s.unread() + } + return ch +} + +// scanFraction scans the fraction after the '.' rune +func (s *Scanner) scanFraction(ch rune) rune { + if ch == '.' { + ch = s.peek() // we peek just to see if we can move forward + ch = s.scanMantissa(ch) + } + return ch +} + +// scanExponent scans the remaining parts of an exponent after the 'e' or 'E' +// rune. +func (s *Scanner) scanExponent(ch rune) rune { + if ch == 'e' || ch == 'E' { + ch = s.next() + if ch == '-' || ch == '+' { + ch = s.next() + } + ch = s.scanMantissa(ch) + } + return ch +} + +// scanString scans a quoted string +func (s *Scanner) scanString() { + braces := 0 + for { + // '"' opening already consumed + // read character after quote + ch := s.next() + + if ch == '\n' || ch < 0 || ch == eof { + s.err("literal not terminated") + return + } + + if ch == '"' && braces == 0 { + break + } + + // If we're going into a ${} then we can ignore quotes for awhile + if braces == 0 && ch == '$' && s.peek() == '{' { + braces++ + s.next() + } else if braces > 0 && ch == '{' { + braces++ + } + if braces > 0 && ch == '}' { + braces-- + } + + if ch == '\\' { + s.scanEscape() + } + } + + return +} + +// scanEscape scans an escape sequence +func (s *Scanner) scanEscape() rune { + // http://en.cppreference.com/w/cpp/language/escape + ch := s.next() // read character after '/' + switch ch { + case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', '"': + // nothing to do + case '0', '1', '2', '3', '4', '5', '6', '7': + // octal notation + ch = s.scanDigits(ch, 8, 3) + case 'x': + // hexademical notation + ch = s.scanDigits(s.next(), 16, 2) + case 'u': + // universal character name + ch = s.scanDigits(s.next(), 16, 4) + case 'U': + // universal character name + ch = s.scanDigits(s.next(), 16, 8) + default: + s.err("illegal char escape") + } + return ch +} + +// scanDigits scans a rune with the given base for n times. For example an +// octal notation \184 would yield in scanDigits(ch, 8, 3) +func (s *Scanner) scanDigits(ch rune, base, n int) rune { + for n > 0 && digitVal(ch) < base { + ch = s.next() + n-- + } + if n > 0 { + s.err("illegal char escape") + } + + // we scanned all digits, put the last non digit char back + s.unread() + return ch +} + +// scanIdentifier scans an identifier and returns the literal string +func (s *Scanner) scanIdentifier() string { + offs := s.srcPos.Offset - s.lastCharLen + ch := s.next() + for isLetter(ch) || isDigit(ch) || ch == '-' { + ch = s.next() + } + + if ch != eof { + s.unread() // we got identifier, put back latest char + } + + return string(s.src[offs:s.srcPos.Offset]) +} + +// recentPosition returns the position of the character immediately after the +// character or token returned by the last call to Scan. +func (s *Scanner) recentPosition() (pos token.Pos) { + pos.Offset = s.srcPos.Offset - s.lastCharLen + switch { + case s.srcPos.Column > 0: + // common case: last character was not a '\n' + pos.Line = s.srcPos.Line + pos.Column = s.srcPos.Column + case s.lastLineLen > 0: + // last character was a '\n' + // (we cannot be at the beginning of the source + // since we have called next() at least once) + pos.Line = s.srcPos.Line - 1 + pos.Column = s.lastLineLen + default: + // at the beginning of the source + pos.Line = 1 + pos.Column = 1 + } + return +} + +// err prints the error of any scanning to s.Error function. If the function is +// not defined, by default it prints them to os.Stderr +func (s *Scanner) err(msg string) { + s.ErrorCount++ + pos := s.recentPosition() + + if s.Error != nil { + s.Error(pos, msg) + return + } + + fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg) +} + +// isHexadecimal returns true if the given rune is a letter +func isLetter(ch rune) bool { + return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch) +} + +// isHexadecimal returns true if the given rune is a decimal digit +func isDigit(ch rune) bool { + return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch) +} + +// isHexadecimal returns true if the given rune is a decimal number +func isDecimal(ch rune) bool { + return '0' <= ch && ch <= '9' +} + +// isHexadecimal returns true if the given rune is an hexadecimal number +func isHexadecimal(ch rune) bool { + return '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F' +} + +// isWhitespace returns true if the rune is a space, tab, newline or carriage return +func isWhitespace(ch rune) bool { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' +} + +// digitVal returns the integer value of a given octal,decimal or hexadecimal rune +func digitVal(ch rune) int { + switch { + case '0' <= ch && ch <= '9': + return int(ch - '0') + case 'a' <= ch && ch <= 'f': + return int(ch - 'a' + 10) + case 'A' <= ch && ch <= 'F': + return int(ch - 'A' + 10) + } + return 16 // larger than any legal digit val +} diff --git a/json/scanner/scanner_test.go b/json/scanner/scanner_test.go new file mode 100644 index 0000000..94c81d3 --- /dev/null +++ b/json/scanner/scanner_test.go @@ -0,0 +1,411 @@ +package scanner + +import ( + "bytes" + "fmt" + "testing" + + "github.com/hashicorp/hcl/json/token" +) + +var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + +type tokenPair struct { + tok token.Type + text string +} + +var tokenLists = map[string][]tokenPair{ + "operator": []tokenPair{ + {token.LBRACK, "["}, + {token.LBRACE, "{"}, + {token.COMMA, ","}, + {token.PERIOD, "."}, + {token.RBRACK, "]"}, + {token.RBRACE, "}"}, + }, + "bool": []tokenPair{ + {token.BOOL, "true"}, + {token.BOOL, "false"}, + }, + "string": []tokenPair{ + {token.STRING, `" "`}, + {token.STRING, `"a"`}, + {token.STRING, `"本"`}, + {token.STRING, `"${file("foo")}"`}, + {token.STRING, `"\a"`}, + {token.STRING, `"\b"`}, + {token.STRING, `"\f"`}, + {token.STRING, `"\n"`}, + {token.STRING, `"\r"`}, + {token.STRING, `"\t"`}, + {token.STRING, `"\v"`}, + {token.STRING, `"\""`}, + {token.STRING, `"\000"`}, + {token.STRING, `"\777"`}, + {token.STRING, `"\x00"`}, + {token.STRING, `"\xff"`}, + {token.STRING, `"\u0000"`}, + {token.STRING, `"\ufA16"`}, + {token.STRING, `"\U00000000"`}, + {token.STRING, `"\U0000ffAB"`}, + {token.STRING, `"` + f100 + `"`}, + }, + "number": []tokenPair{ + {token.NUMBER, "0"}, + {token.NUMBER, "1"}, + {token.NUMBER, "9"}, + {token.NUMBER, "42"}, + {token.NUMBER, "1234567890"}, + {token.NUMBER, "-0"}, + {token.NUMBER, "-1"}, + {token.NUMBER, "-9"}, + {token.NUMBER, "-42"}, + {token.NUMBER, "-1234567890"}, + }, + "float": []tokenPair{ + {token.FLOAT, "0."}, + {token.FLOAT, "1."}, + {token.FLOAT, "42."}, + {token.FLOAT, "01234567890."}, + {token.FLOAT, ".0"}, + {token.FLOAT, ".1"}, + {token.FLOAT, ".42"}, + {token.FLOAT, ".0123456789"}, + {token.FLOAT, "0.0"}, + {token.FLOAT, "1.0"}, + {token.FLOAT, "42.0"}, + {token.FLOAT, "01234567890.0"}, + {token.FLOAT, "0e0"}, + {token.FLOAT, "1e0"}, + {token.FLOAT, "42e0"}, + {token.FLOAT, "01234567890e0"}, + {token.FLOAT, "0E0"}, + {token.FLOAT, "1E0"}, + {token.FLOAT, "42E0"}, + {token.FLOAT, "01234567890E0"}, + {token.FLOAT, "0e+10"}, + {token.FLOAT, "1e-10"}, + {token.FLOAT, "42e+10"}, + {token.FLOAT, "01234567890e-10"}, + {token.FLOAT, "0E+10"}, + {token.FLOAT, "1E-10"}, + {token.FLOAT, "42E+10"}, + {token.FLOAT, "01234567890E-10"}, + {token.FLOAT, "01.8e0"}, + {token.FLOAT, "1.4e0"}, + {token.FLOAT, "42.2e0"}, + {token.FLOAT, "01234567890.12e0"}, + {token.FLOAT, "0.E0"}, + {token.FLOAT, "1.12E0"}, + {token.FLOAT, "42.123E0"}, + {token.FLOAT, "01234567890.213E0"}, + {token.FLOAT, "0.2e+10"}, + {token.FLOAT, "1.2e-10"}, + {token.FLOAT, "42.54e+10"}, + {token.FLOAT, "01234567890.98e-10"}, + {token.FLOAT, "0.1E+10"}, + {token.FLOAT, "1.1E-10"}, + {token.FLOAT, "42.1E+10"}, + {token.FLOAT, "01234567890.1E-10"}, + {token.FLOAT, "-0.0"}, + {token.FLOAT, "-1.0"}, + {token.FLOAT, "-42.0"}, + {token.FLOAT, "-01234567890.0"}, + {token.FLOAT, "-0e0"}, + {token.FLOAT, "-1e0"}, + {token.FLOAT, "-42e0"}, + {token.FLOAT, "-01234567890e0"}, + {token.FLOAT, "-0E0"}, + {token.FLOAT, "-1E0"}, + {token.FLOAT, "-42E0"}, + {token.FLOAT, "-01234567890E0"}, + {token.FLOAT, "-0e+10"}, + {token.FLOAT, "-1e-10"}, + {token.FLOAT, "-42e+10"}, + {token.FLOAT, "-01234567890e-10"}, + {token.FLOAT, "-0E+10"}, + {token.FLOAT, "-1E-10"}, + {token.FLOAT, "-42E+10"}, + {token.FLOAT, "-01234567890E-10"}, + {token.FLOAT, "-01.8e0"}, + {token.FLOAT, "-1.4e0"}, + {token.FLOAT, "-42.2e0"}, + {token.FLOAT, "-01234567890.12e0"}, + {token.FLOAT, "-0.E0"}, + {token.FLOAT, "-1.12E0"}, + {token.FLOAT, "-42.123E0"}, + {token.FLOAT, "-01234567890.213E0"}, + {token.FLOAT, "-0.2e+10"}, + {token.FLOAT, "-1.2e-10"}, + {token.FLOAT, "-42.54e+10"}, + {token.FLOAT, "-01234567890.98e-10"}, + {token.FLOAT, "-0.1E+10"}, + {token.FLOAT, "-1.1E-10"}, + {token.FLOAT, "-42.1E+10"}, + {token.FLOAT, "-01234567890.1E-10"}, + }, +} + +var orderedTokenLists = []string{ + "comment", + "operator", + "bool", + "string", + "number", + "float", +} + +func TestPosition(t *testing.T) { + // create artifical source code + buf := new(bytes.Buffer) + + for _, listName := range orderedTokenLists { + for _, ident := range tokenLists[listName] { + fmt.Fprintf(buf, "\t\t\t\t%s\n", ident.text) + } + } + + s := New(buf.Bytes()) + + pos := token.Pos{"", 4, 1, 5} + s.Scan() + for _, listName := range orderedTokenLists { + + for _, k := range tokenLists[listName] { + curPos := s.tokPos + // fmt.Printf("[%q] s = %+v:%+v\n", k.text, curPos.Offset, curPos.Column) + + if curPos.Offset != pos.Offset { + t.Fatalf("offset = %d, want %d for %q", curPos.Offset, pos.Offset, k.text) + } + if curPos.Line != pos.Line { + t.Fatalf("line = %d, want %d for %q", curPos.Line, pos.Line, k.text) + } + if curPos.Column != pos.Column { + t.Fatalf("column = %d, want %d for %q", curPos.Column, pos.Column, k.text) + } + pos.Offset += 4 + len(k.text) + 1 // 4 tabs + token bytes + newline + pos.Line += countNewlines(k.text) + 1 // each token is on a new line + + s.Error = func(pos token.Pos, msg string) { + t.Errorf("error %q for %q", msg, k.text) + } + + s.Scan() + } + } + // make sure there were no token-internal errors reported by scanner + if s.ErrorCount != 0 { + t.Errorf("%d errors", s.ErrorCount) + } +} + +func TestComment(t *testing.T) { + testTokenList(t, tokenLists["comment"]) +} + +func TestOperator(t *testing.T) { + testTokenList(t, tokenLists["operator"]) +} + +func TestBool(t *testing.T) { + testTokenList(t, tokenLists["bool"]) +} + +func TestIdent(t *testing.T) { + testTokenList(t, tokenLists["ident"]) +} + +func TestString(t *testing.T) { + testTokenList(t, tokenLists["string"]) +} + +func TestNumber(t *testing.T) { + testTokenList(t, tokenLists["number"]) +} + +func TestFloat(t *testing.T) { + testTokenList(t, tokenLists["float"]) +} + +/* +func TestRealExample(t *testing.T) { + complexHCL := `// This comes from Terraform, as a test + variable "foo" { + default = "bar" + description = "bar" + } + + provider "aws" { + access_key = "foo" + secret_key = "bar" + } + + resource "aws_security_group" "firewall" { + count = 5 + } + + resource aws_instance "web" { + ami = "${var.foo}" + security_groups = [ + "foo", + "${aws_security_group.firewall.foo}" + ] + + network_interface { + device_index = 0 + description = "Main network interface" + } + }` + + literals := []struct { + tokenType token.Type + literal string + }{ + {token.COMMENT, `// This comes from Terraform, as a test`}, + {token.IDENT, `variable`}, + {token.STRING, `"foo"`}, + {token.LBRACE, `{`}, + {token.IDENT, `default`}, + {token.ASSIGN, `=`}, + {token.STRING, `"bar"`}, + {token.IDENT, `description`}, + {token.ASSIGN, `=`}, + {token.STRING, `"bar"`}, + {token.RBRACE, `}`}, + {token.IDENT, `provider`}, + {token.STRING, `"aws"`}, + {token.LBRACE, `{`}, + {token.IDENT, `access_key`}, + {token.ASSIGN, `=`}, + {token.STRING, `"foo"`}, + {token.IDENT, `secret_key`}, + {token.ASSIGN, `=`}, + {token.STRING, `"bar"`}, + {token.RBRACE, `}`}, + {token.IDENT, `resource`}, + {token.STRING, `"aws_security_group"`}, + {token.STRING, `"firewall"`}, + {token.LBRACE, `{`}, + {token.IDENT, `count`}, + {token.ASSIGN, `=`}, + {token.NUMBER, `5`}, + {token.RBRACE, `}`}, + {token.IDENT, `resource`}, + {token.IDENT, `aws_instance`}, + {token.STRING, `"web"`}, + {token.LBRACE, `{`}, + {token.IDENT, `ami`}, + {token.ASSIGN, `=`}, + {token.STRING, `"${var.foo}"`}, + {token.IDENT, `security_groups`}, + {token.ASSIGN, `=`}, + {token.LBRACK, `[`}, + {token.STRING, `"foo"`}, + {token.COMMA, `,`}, + {token.STRING, `"${aws_security_group.firewall.foo}"`}, + {token.RBRACK, `]`}, + {token.IDENT, `network_interface`}, + {token.LBRACE, `{`}, + {token.IDENT, `device_index`}, + {token.ASSIGN, `=`}, + {token.NUMBER, `0`}, + {token.IDENT, `description`}, + {token.ASSIGN, `=`}, + {token.STRING, `"Main network interface"`}, + {token.RBRACE, `}`}, + {token.RBRACE, `}`}, + {token.EOF, ``}, + } + + s := New([]byte(complexHCL)) + for _, l := range literals { + tok := s.Scan() + if l.tokenType != tok.Type { + t.Errorf("got: %s want %s for %s\n", tok, l.tokenType, tok.String()) + } + + if l.literal != tok.Text { + t.Errorf("got: %s want %s\n", tok, l.literal) + } + } + +} +*/ + +func TestError(t *testing.T) { + testError(t, "\x80", "1:1", "illegal UTF-8 encoding", token.ILLEGAL) + testError(t, "\xff", "1:1", "illegal UTF-8 encoding", token.ILLEGAL) + + testError(t, `"ab`+"\x80", "1:4", "illegal UTF-8 encoding", token.STRING) + testError(t, `"abc`+"\xff", "1:5", "illegal UTF-8 encoding", token.STRING) + + testError(t, `01238`, "1:7", "numbers cannot start with 0", token.NUMBER) + testError(t, `01238123`, "1:10", "numbers cannot start with 0", token.NUMBER) + testError(t, `'aa'`, "1:1", "illegal char", token.ILLEGAL) + + testError(t, `"`, "1:2", "literal not terminated", token.STRING) + testError(t, `"abc`, "1:5", "literal not terminated", token.STRING) + testError(t, `"abc`+"\n", "1:5", "literal not terminated", token.STRING) +} + +func testError(t *testing.T, src, pos, msg string, tok token.Type) { + s := New([]byte(src)) + + errorCalled := false + s.Error = func(p token.Pos, m string) { + if !errorCalled { + if pos != p.String() { + t.Errorf("pos = %q, want %q for %q", p, pos, src) + } + + if m != msg { + t.Errorf("msg = %q, want %q for %q", m, msg, src) + } + errorCalled = true + } + } + + tk := s.Scan() + if tk.Type != tok { + t.Errorf("tok = %s, want %s for %q", tk, tok, src) + } + if !errorCalled { + t.Errorf("error handler not called for %q", src) + } + if s.ErrorCount == 0 { + t.Errorf("count = %d, want > 0 for %q", s.ErrorCount, src) + } +} + +func testTokenList(t *testing.T, tokenList []tokenPair) { + // create artifical source code + buf := new(bytes.Buffer) + for _, ident := range tokenList { + fmt.Fprintf(buf, "%s\n", ident.text) + } + + s := New(buf.Bytes()) + for _, ident := range tokenList { + tok := s.Scan() + if tok.Type != ident.tok { + t.Errorf("tok = %q want %q for %q\n", tok, ident.tok, ident.text) + } + + if tok.Text != ident.text { + t.Errorf("text = %q want %q", tok.String(), ident.text) + } + + } +} + +func countNewlines(s string) int { + n := 0 + for _, ch := range s { + if ch == '\n' { + n++ + } + } + return n +} diff --git a/json/token/position.go b/json/token/position.go new file mode 100644 index 0000000..59c1bb7 --- /dev/null +++ b/json/token/position.go @@ -0,0 +1,46 @@ +package token + +import "fmt" + +// Pos describes an arbitrary source position +// including the file, line, and column location. +// A Position is valid if the line number is > 0. +type Pos struct { + Filename string // filename, if any + Offset int // offset, starting at 0 + Line int // line number, starting at 1 + Column int // column number, starting at 1 (character count) +} + +// IsValid returns true if the position is valid. +func (p *Pos) IsValid() bool { return p.Line > 0 } + +// String returns a string in one of several forms: +// +// file:line:column valid position with file name +// line:column valid position without file name +// file invalid position with file name +// - invalid position without file name +func (p Pos) String() string { + s := p.Filename + if p.IsValid() { + if s != "" { + s += ":" + } + s += fmt.Sprintf("%d:%d", p.Line, p.Column) + } + if s == "" { + s = "-" + } + return s +} + +// Before reports whether the position p is before u. +func (p Pos) Before(u Pos) bool { + return u.Offset > p.Offset || u.Line > p.Line +} + +// After reports whether the position p is after u. +func (p Pos) After(u Pos) bool { + return u.Offset < p.Offset || u.Line < p.Line +} diff --git a/json/token/token.go b/json/token/token.go new file mode 100644 index 0000000..536fb60 --- /dev/null +++ b/json/token/token.go @@ -0,0 +1,139 @@ +package token + +import ( + "fmt" + "strconv" + + hclstrconv "github.com/hashicorp/hcl/hcl/strconv" +) + +// Token defines a single HCL token which can be obtained via the Scanner +type Token struct { + Type Type + Pos Pos + Text string +} + +// Type is the set of lexical tokens of the HCL (HashiCorp Configuration Language) +type Type int + +const ( + // Special tokens + ILLEGAL Type = iota + EOF + + identifier_beg + literal_beg + NUMBER // 12345 + FLOAT // 123.45 + BOOL // true,false + STRING // "abc" + NULL // null + literal_end + identifier_end + + operator_beg + LBRACK // [ + LBRACE // { + COMMA // , + PERIOD // . + + RBRACK // ] + RBRACE // } + + operator_end +) + +var tokens = [...]string{ + ILLEGAL: "ILLEGAL", + + EOF: "EOF", + + NUMBER: "NUMBER", + FLOAT: "FLOAT", + BOOL: "BOOL", + STRING: "STRING", + NULL: "NULL", + + LBRACK: "LBRACK", + LBRACE: "LBRACE", + COMMA: "COMMA", + PERIOD: "PERIOD", + + RBRACK: "RBRACK", + RBRACE: "RBRACE", +} + +// String returns the string corresponding to the token tok. +func (t Type) String() string { + s := "" + if 0 <= t && t < Type(len(tokens)) { + s = tokens[t] + } + if s == "" { + s = "token(" + strconv.Itoa(int(t)) + ")" + } + return s +} + +// IsIdentifier returns true for tokens corresponding to identifiers and basic +// type literals; it returns false otherwise. +func (t Type) IsIdentifier() bool { return identifier_beg < t && t < identifier_end } + +// IsLiteral returns true for tokens corresponding to basic type literals; it +// returns false otherwise. +func (t Type) IsLiteral() bool { return literal_beg < t && t < literal_end } + +// IsOperator returns true for tokens corresponding to operators and +// delimiters; it returns false otherwise. +func (t Type) IsOperator() bool { return operator_beg < t && t < operator_end } + +// String returns the token's literal text. Note that this is only +// applicable for certain token types, such as token.IDENT, +// token.STRING, etc.. +func (t Token) String() string { + return fmt.Sprintf("%s %s %s", t.Pos.String(), t.Type.String(), t.Text) +} + +// Value returns the properly typed value for this token. The type of +// the returned interface{} is guaranteed based on the Type field. +// +// This can only be called for literal types. If it is called for any other +// type, this will panic. +func (t Token) Value() interface{} { + switch t.Type { + case BOOL: + if t.Text == "true" { + return true + } else if t.Text == "false" { + return false + } + + panic("unknown bool value: " + t.Text) + case FLOAT: + v, err := strconv.ParseFloat(t.Text, 64) + if err != nil { + panic(err) + } + + return float64(v) + case NULL: + return nil + case NUMBER: + v, err := strconv.ParseInt(t.Text, 0, 64) + if err != nil { + panic(err) + } + + return int64(v) + case STRING: + v, err := hclstrconv.Unquote(t.Text) + if err != nil { + panic(fmt.Sprintf("unquote %s err: %s", t.Text, err)) + } + + return v + default: + panic(fmt.Sprintf("unimplemented Value for type: %s", t.Type)) + } +} diff --git a/json/token/token_test.go b/json/token/token_test.go new file mode 100644 index 0000000..ea39e40 --- /dev/null +++ b/json/token/token_test.go @@ -0,0 +1,57 @@ +package token + +import ( + "reflect" + "testing" +) + +func TestTypeString(t *testing.T) { + var tokens = []struct { + tt Type + str string + }{ + {ILLEGAL, "ILLEGAL"}, + {EOF, "EOF"}, + {NUMBER, "NUMBER"}, + {FLOAT, "FLOAT"}, + {BOOL, "BOOL"}, + {STRING, "STRING"}, + {NULL, "NULL"}, + {LBRACK, "LBRACK"}, + {LBRACE, "LBRACE"}, + {COMMA, "COMMA"}, + {PERIOD, "PERIOD"}, + {RBRACK, "RBRACK"}, + {RBRACE, "RBRACE"}, + } + + for _, token := range tokens { + if token.tt.String() != token.str { + t.Errorf("want: %q got:%q\n", token.str, token.tt) + + } + } + +} + +func TestTokenValue(t *testing.T) { + var tokens = []struct { + tt Token + v interface{} + }{ + {Token{Type: BOOL, Text: `true`}, true}, + {Token{Type: BOOL, Text: `false`}, false}, + {Token{Type: FLOAT, Text: `3.14`}, float64(3.14)}, + {Token{Type: NULL, Text: `null`}, nil}, + {Token{Type: NUMBER, Text: `42`}, int64(42)}, + {Token{Type: STRING, Text: `"foo"`}, "foo"}, + {Token{Type: STRING, Text: `"foo\nbar"`}, "foo\nbar"}, + } + + for _, token := range tokens { + if val := token.tt.Value(); !reflect.DeepEqual(val, token.v) { + t.Errorf("want: %v got:%v\n", token.v, val) + } + } + +} From 0fce9beccff7624e8047826ef8874f62183657f4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 14:39:43 -0800 Subject: [PATCH 02/13] json/scanner: beef up tests with real example --- json/scanner/scanner.go | 4 +- json/scanner/scanner_test.go | 105 ++++++++++------------------------- 2 files changed, 31 insertions(+), 78 deletions(-) diff --git a/json/scanner/scanner.go b/json/scanner/scanner.go index eca2c98..477f71f 100644 --- a/json/scanner/scanner.go +++ b/json/scanner/scanner.go @@ -182,6 +182,8 @@ func (s *Scanner) Scan() token.Token { tok = token.RBRACE case ',': tok = token.COMMA + case ':': + tok = token.COLON case '-': if isDecimal(s.peek()) { ch := s.next() @@ -190,7 +192,7 @@ func (s *Scanner) Scan() token.Token { s.err("illegal char") } default: - s.err("illegal char") + s.err("illegal char: " + string(ch)) } } diff --git a/json/scanner/scanner_test.go b/json/scanner/scanner_test.go index 94c81d3..c9772fe 100644 --- a/json/scanner/scanner_test.go +++ b/json/scanner/scanner_test.go @@ -229,97 +229,49 @@ func TestFloat(t *testing.T) { testTokenList(t, tokenLists["float"]) } -/* func TestRealExample(t *testing.T) { - complexHCL := `// This comes from Terraform, as a test - variable "foo" { - default = "bar" - description = "bar" - } - - provider "aws" { - access_key = "foo" - secret_key = "bar" - } - - resource "aws_security_group" "firewall" { - count = 5 - } - - resource aws_instance "web" { - ami = "${var.foo}" - security_groups = [ - "foo", - "${aws_security_group.firewall.foo}" - ] - - network_interface { - device_index = 0 - description = "Main network interface" - } - }` + complexReal := ` +{ + "variable": { + "foo": { + "default": "bar", + "description": "bar", + "depends_on": ["something"] + } + } +}` literals := []struct { tokenType token.Type literal string }{ - {token.COMMENT, `// This comes from Terraform, as a test`}, - {token.IDENT, `variable`}, + {token.LBRACE, `{`}, + {token.STRING, `"variable"`}, + {token.COLON, `:`}, + {token.LBRACE, `{`}, {token.STRING, `"foo"`}, + {token.COLON, `:`}, {token.LBRACE, `{`}, - {token.IDENT, `default`}, - {token.ASSIGN, `=`}, + {token.STRING, `"default"`}, + {token.COLON, `:`}, {token.STRING, `"bar"`}, - {token.IDENT, `description`}, - {token.ASSIGN, `=`}, - {token.STRING, `"bar"`}, - {token.RBRACE, `}`}, - {token.IDENT, `provider`}, - {token.STRING, `"aws"`}, - {token.LBRACE, `{`}, - {token.IDENT, `access_key`}, - {token.ASSIGN, `=`}, - {token.STRING, `"foo"`}, - {token.IDENT, `secret_key`}, - {token.ASSIGN, `=`}, - {token.STRING, `"bar"`}, - {token.RBRACE, `}`}, - {token.IDENT, `resource`}, - {token.STRING, `"aws_security_group"`}, - {token.STRING, `"firewall"`}, - {token.LBRACE, `{`}, - {token.IDENT, `count`}, - {token.ASSIGN, `=`}, - {token.NUMBER, `5`}, - {token.RBRACE, `}`}, - {token.IDENT, `resource`}, - {token.IDENT, `aws_instance`}, - {token.STRING, `"web"`}, - {token.LBRACE, `{`}, - {token.IDENT, `ami`}, - {token.ASSIGN, `=`}, - {token.STRING, `"${var.foo}"`}, - {token.IDENT, `security_groups`}, - {token.ASSIGN, `=`}, - {token.LBRACK, `[`}, - {token.STRING, `"foo"`}, {token.COMMA, `,`}, - {token.STRING, `"${aws_security_group.firewall.foo}"`}, + {token.STRING, `"description"`}, + {token.COLON, `:`}, + {token.STRING, `"bar"`}, + {token.COMMA, `,`}, + {token.STRING, `"depends_on"`}, + {token.COLON, `:`}, + {token.LBRACK, `[`}, + {token.STRING, `"something"`}, {token.RBRACK, `]`}, - {token.IDENT, `network_interface`}, - {token.LBRACE, `{`}, - {token.IDENT, `device_index`}, - {token.ASSIGN, `=`}, - {token.NUMBER, `0`}, - {token.IDENT, `description`}, - {token.ASSIGN, `=`}, - {token.STRING, `"Main network interface"`}, + {token.RBRACE, `}`}, {token.RBRACE, `}`}, {token.RBRACE, `}`}, {token.EOF, ``}, } - s := New([]byte(complexHCL)) + s := New([]byte(complexReal)) for _, l := range literals { tok := s.Scan() if l.tokenType != tok.Type { @@ -332,7 +284,6 @@ func TestRealExample(t *testing.T) { } } -*/ func TestError(t *testing.T) { testError(t, "\x80", "1:1", "illegal UTF-8 encoding", token.ILLEGAL) @@ -343,7 +294,7 @@ func TestError(t *testing.T) { testError(t, `01238`, "1:7", "numbers cannot start with 0", token.NUMBER) testError(t, `01238123`, "1:10", "numbers cannot start with 0", token.NUMBER) - testError(t, `'aa'`, "1:1", "illegal char", token.ILLEGAL) + testError(t, `'aa'`, "1:1", "illegal char: '", token.ILLEGAL) testError(t, `"`, "1:2", "literal not terminated", token.STRING) testError(t, `"abc`, "1:5", "literal not terminated", token.STRING) From ca0ea00fe9ace87e844aa77d1000efd691b559ad Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 14:39:58 -0800 Subject: [PATCH 03/13] json/token: add comma token --- json/token/token.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/json/token/token.go b/json/token/token.go index 536fb60..d6d23e0 100644 --- a/json/token/token.go +++ b/json/token/token.go @@ -37,6 +37,7 @@ const ( LBRACE // { COMMA // , PERIOD // . + COLON // : RBRACK // ] RBRACE // } @@ -59,6 +60,7 @@ var tokens = [...]string{ LBRACE: "LBRACE", COMMA: "COMMA", PERIOD: "PERIOD", + COLON: "COLON", RBRACK: "RBRACK", RBRACE: "RBRACE", From f5a27d4be4ee8ae949168902e16ddb849783d540 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 15:50:37 -0800 Subject: [PATCH 04/13] json/parser --- json/parser/parser.go | 286 ++++++++++++++++++++++++++ json/parser/parser_test.go | 261 +++++++++++++++++++++++ json/parser/test-fixtures/array.json | 4 + json/parser/test-fixtures/basic.json | 3 + json/parser/test-fixtures/object.json | 5 + json/parser/test-fixtures/types.json | 10 + json/token/token.go | 21 ++ 7 files changed, 590 insertions(+) create mode 100644 json/parser/parser.go create mode 100644 json/parser/parser_test.go create mode 100644 json/parser/test-fixtures/array.json create mode 100644 json/parser/test-fixtures/basic.json create mode 100644 json/parser/test-fixtures/object.json create mode 100644 json/parser/test-fixtures/types.json diff --git a/json/parser/parser.go b/json/parser/parser.go new file mode 100644 index 0000000..1f7db62 --- /dev/null +++ b/json/parser/parser.go @@ -0,0 +1,286 @@ +package parser + +import ( + "errors" + "fmt" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/json/scanner" + "github.com/hashicorp/hcl/json/token" +) + +type Parser struct { + sc *scanner.Scanner + + // Last read token + tok token.Token + commaPrev token.Token + + enableTrace bool + indent int + n int // buffer size (max = 1) +} + +func newParser(src []byte) *Parser { + return &Parser{ + sc: scanner.New(src), + } +} + +// Parse returns the fully parsed source and returns the abstract syntax tree. +func Parse(src []byte) (*ast.File, error) { + p := newParser(src) + return p.Parse() +} + +var errEofToken = errors.New("EOF token found") + +// Parse returns the fully parsed source and returns the abstract syntax tree. +func (p *Parser) Parse() (*ast.File, error) { + f := &ast.File{} + var err, scerr error + p.sc.Error = func(pos token.Pos, msg string) { + scerr = fmt.Errorf("%s: %s", pos, msg) + } + + // The root must be an object in JSON + object, err := p.object() + if scerr != nil { + return nil, scerr + } + if err != nil { + return nil, err + } + + // We make our final node an object list so it is more HCL compatible + f.Node = object.List + return f, nil +} + +func (p *Parser) objectList() (*ast.ObjectList, error) { + defer un(trace(p, "ParseObjectList")) + node := &ast.ObjectList{} + + for { + n, err := p.objectItem() + if err == errEofToken { + break // we are finished + } + + // we don't return a nil node, because might want to use already + // collected items. + if err != nil { + return node, err + } + + node.Add(n) + + // Check for a followup comma. If it isn't a comma, then we're done + if tok := p.scan(); tok.Type != token.COMMA { + p.unscan() + break + } + } + return node, nil +} + +// objectItem parses a single object item +func (p *Parser) objectItem() (*ast.ObjectItem, error) { + defer un(trace(p, "ParseObjectItem")) + + keys, err := p.objectKey() + if err != nil { + return nil, err + } + + o := &ast.ObjectItem{ + Keys: keys, + } + + switch p.tok.Type { + case token.COLON: + o.Val, err = p.objectValue() + if err != nil { + return nil, err + } + } + + return o, nil +} + +// objectKey parses an object key and returns a ObjectKey AST +func (p *Parser) objectKey() ([]*ast.ObjectKey, error) { + keyCount := 0 + keys := make([]*ast.ObjectKey, 0) + + for { + tok := p.scan() + switch tok.Type { + case token.EOF: + return nil, errEofToken + case token.STRING: + keyCount++ + keys = append(keys, &ast.ObjectKey{ + Token: p.tok.HCLToken(), + }) + case token.COLON: + // Done + return keys, nil + case token.ILLEGAL: + fmt.Println("illegal") + default: + return nil, fmt.Errorf("expected: STRING got: %s", p.tok.Type) + } + } +} + +// object parses any type of object, such as number, bool, string, object or +// list. +func (p *Parser) objectValue() (ast.Node, error) { + defer un(trace(p, "ParseObjectValue")) + tok := p.scan() + + switch tok.Type { + case token.NUMBER, token.FLOAT, token.BOOL, token.NULL, token.STRING: + return p.literalType() + case token.LBRACE: + return p.objectType() + case token.LBRACK: + return p.listType() + case token.EOF: + return nil, errEofToken + } + + return nil, fmt.Errorf("Expected object value, got unknown token: %+v", tok) +} + +// object parses any type of object, such as number, bool, string, object or +// list. +func (p *Parser) object() (*ast.ObjectType, error) { + defer un(trace(p, "ParseType")) + tok := p.scan() + + switch tok.Type { + case token.LBRACE: + return p.objectType() + case token.EOF: + return nil, errEofToken + } + + return nil, fmt.Errorf("Expected object, got unknown token: %+v", tok) +} + +// objectType parses an object type and returns a ObjectType AST +func (p *Parser) objectType() (*ast.ObjectType, error) { + defer un(trace(p, "ParseObjectType")) + + // we assume that the currently scanned token is a LBRACE + o := &ast.ObjectType{} + + l, err := p.objectList() + + // if we hit RBRACE, we are good to go (means we parsed all Items), if it's + // not a RBRACE, it's an syntax error and we just return it. + if err != nil && p.tok.Type != token.RBRACE { + return nil, err + } + + o.List = l + return o, nil +} + +// listType parses a list type and returns a ListType AST +func (p *Parser) listType() (*ast.ListType, error) { + defer un(trace(p, "ParseListType")) + + // we assume that the currently scanned token is a LBRACK + l := &ast.ListType{} + + for { + tok := p.scan() + switch tok.Type { + case token.NUMBER, token.FLOAT, token.STRING: + node, err := p.literalType() + if err != nil { + return nil, err + } + + l.Add(node) + case token.COMMA: + continue + case token.BOOL: + // TODO(arslan) should we support? not supported by HCL yet + case token.LBRACK: + // TODO(arslan) should we support nested lists? Even though it's + // written in README of HCL, it's not a part of the grammar + // (not defined in parse.y) + case token.RBRACK: + // finished + return l, nil + default: + return nil, fmt.Errorf("unexpected token while parsing list: %s", tok.Type) + } + + } +} + +// literalType parses a literal type and returns a LiteralType AST +func (p *Parser) literalType() (*ast.LiteralType, error) { + defer un(trace(p, "ParseLiteral")) + + return &ast.LiteralType{ + Token: p.tok.HCLToken(), + }, nil +} + +// scan returns the next token from the underlying scanner. If a token has +// been unscanned then read that instead. +func (p *Parser) scan() token.Token { + // If we have a token on the buffer, then return it. + if p.n != 0 { + p.n = 0 + return p.tok + } + + p.tok = p.sc.Scan() + return p.tok +} + +// unscan pushes the previously read token back onto the buffer. +func (p *Parser) unscan() { + p.n = 1 +} + +// ---------------------------------------------------------------------------- +// Parsing support + +func (p *Parser) printTrace(a ...interface{}) { + if !p.enableTrace { + return + } + + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = len(dots) + fmt.Printf("%5d:%3d: ", p.tok.Pos.Line, p.tok.Pos.Column) + + i := 2 * p.indent + for i > n { + fmt.Print(dots) + i -= n + } + // i <= n + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + +func trace(p *Parser, msg string) *Parser { + p.printTrace(msg, "(") + p.indent++ + return p +} + +// Usage pattern: defer un(trace(p, "...")) +func un(p *Parser) { + p.indent-- + p.printTrace(")") +} diff --git a/json/parser/parser_test.go b/json/parser/parser_test.go new file mode 100644 index 0000000..9c77daf --- /dev/null +++ b/json/parser/parser_test.go @@ -0,0 +1,261 @@ +package parser + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "reflect" + "runtime" + "testing" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/hcl/token" +) + +func TestType(t *testing.T) { + var literals = []struct { + typ token.Type + src string + }{ + {token.STRING, `"foo": "bar"`}, + {token.NUMBER, `"foo": 123`}, + {token.FLOAT, `"foo": 123.12`}, + {token.FLOAT, `"foo": -123.12`}, + {token.BOOL, `"foo": true`}, + {token.STRING, `"foo": null`}, + } + + for _, l := range literals { + t.Logf("Testing: %s", l.src) + + p := newParser([]byte(l.src)) + item, err := p.objectItem() + if err != nil { + t.Error(err) + } + + lit, ok := item.Val.(*ast.LiteralType) + if !ok { + t.Errorf("node should be of type LiteralType, got: %T", item.Val) + } + + if lit.Token.Type != l.typ { + t.Errorf("want: %s, got: %s", l.typ, lit.Token.Type) + } + } +} + +func TestListType(t *testing.T) { + var literals = []struct { + src string + tokens []token.Type + }{ + { + `"foo": ["123", 123]`, + []token.Type{token.STRING, token.NUMBER}, + }, + { + `"foo": [123, "123",]`, + []token.Type{token.NUMBER, token.STRING}, + }, + { + `"foo": []`, + []token.Type{}, + }, + { + `"foo": ["123", 123]`, + []token.Type{token.STRING, token.NUMBER}, + }, + } + + for _, l := range literals { + t.Logf("Testing: %s", l.src) + + p := newParser([]byte(l.src)) + item, err := p.objectItem() + if err != nil { + t.Error(err) + } + + list, ok := item.Val.(*ast.ListType) + if !ok { + t.Errorf("node should be of type LiteralType, got: %T", item.Val) + } + + tokens := []token.Type{} + for _, li := range list.List { + if tp, ok := li.(*ast.LiteralType); ok { + tokens = append(tokens, tp.Token.Type) + } + } + + equals(t, l.tokens, tokens) + } +} + +func TestObjectType(t *testing.T) { + var literals = []struct { + src string + nodeType []ast.Node + itemLen int + }{ + { + `"foo": {}`, + nil, + 0, + }, + { + `"foo": { + "bar": "fatih" + }`, + []ast.Node{&ast.LiteralType{}}, + 1, + }, + { + `"foo": { + "bar": "fatih", + "baz": ["arslan"] + }`, + []ast.Node{ + &ast.LiteralType{}, + &ast.ListType{}, + }, + 2, + }, + { + `"foo": { + "bar": {} + }`, + []ast.Node{ + &ast.ObjectType{}, + }, + 1, + }, + { + `"foo": { + "bar": {}, + "foo": true + }`, + []ast.Node{ + &ast.ObjectType{}, + &ast.LiteralType{}, + }, + 2, + }, + } + + for _, l := range literals { + t.Logf("Testing:\n%s\n", l.src) + + p := newParser([]byte(l.src)) + // p.enableTrace = true + item, err := p.objectItem() + if err != nil { + t.Error(err) + } + + // we know that the ObjectKey name is foo for all cases, what matters + // is the object + obj, ok := item.Val.(*ast.ObjectType) + if !ok { + t.Errorf("node should be of type LiteralType, got: %T", item.Val) + } + + // check if the total length of items are correct + equals(t, l.itemLen, len(obj.List.Items)) + + // check if the types are correct + for i, item := range obj.List.Items { + equals(t, reflect.TypeOf(l.nodeType[i]), reflect.TypeOf(item.Val)) + } + } +} + +func TestObjectKey(t *testing.T) { + keys := []struct { + exp []token.Type + src string + }{ + {[]token.Type{token.STRING}, `"foo": {}`}, + } + + for _, k := range keys { + p := newParser([]byte(k.src)) + keys, err := p.objectKey() + if err != nil { + t.Fatal(err) + } + + tokens := []token.Type{} + for _, o := range keys { + tokens = append(tokens, o.Token.Type) + } + + equals(t, k.exp, tokens) + } + + errKeys := []struct { + src string + }{ + {`foo 12 {}`}, + {`foo bar = {}`}, + {`foo []`}, + {`12 {}`}, + } + + for _, k := range errKeys { + p := newParser([]byte(k.src)) + _, err := p.objectKey() + if err == nil { + t.Errorf("case '%s' should give an error", k.src) + } + } +} + +// Official HCL tests +func TestParse(t *testing.T) { + cases := []struct { + Name string + Err bool + }{ + { + "basic.json", + false, + }, + { + "object.json", + false, + }, + { + "array.json", + false, + }, + { + "types.json", + false, + }, + } + + const fixtureDir = "./test-fixtures" + + for _, tc := range cases { + d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.Name)) + if err != nil { + t.Fatalf("err: %s", err) + } + + _, err = Parse(d) + if (err != nil) != tc.Err { + t.Fatalf("Input: %s\n\nError: %s", tc.Name, err) + } + } +} + +// equals fails the test if exp is not equal to act. +func equals(tb testing.TB, exp, act interface{}) { + if !reflect.DeepEqual(exp, act) { + _, file, line, _ := runtime.Caller(1) + fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) + tb.FailNow() + } +} diff --git a/json/parser/test-fixtures/array.json b/json/parser/test-fixtures/array.json new file mode 100644 index 0000000..e320f17 --- /dev/null +++ b/json/parser/test-fixtures/array.json @@ -0,0 +1,4 @@ +{ + "foo": [1, 2, "bar"], + "bar": "baz" +} diff --git a/json/parser/test-fixtures/basic.json b/json/parser/test-fixtures/basic.json new file mode 100644 index 0000000..b54bde9 --- /dev/null +++ b/json/parser/test-fixtures/basic.json @@ -0,0 +1,3 @@ +{ + "foo": "bar" +} diff --git a/json/parser/test-fixtures/object.json b/json/parser/test-fixtures/object.json new file mode 100644 index 0000000..72168a3 --- /dev/null +++ b/json/parser/test-fixtures/object.json @@ -0,0 +1,5 @@ +{ + "foo": { + "bar": [1,2] + } +} diff --git a/json/parser/test-fixtures/types.json b/json/parser/test-fixtures/types.json new file mode 100644 index 0000000..9a142a6 --- /dev/null +++ b/json/parser/test-fixtures/types.json @@ -0,0 +1,10 @@ +{ + "foo": "bar", + "bar": 7, + "baz": [1,2,3], + "foo": -12, + "bar": 3.14159, + "foo": true, + "bar": false, + "foo": null +} diff --git a/json/token/token.go b/json/token/token.go index d6d23e0..034adcc 100644 --- a/json/token/token.go +++ b/json/token/token.go @@ -5,6 +5,7 @@ import ( "strconv" hclstrconv "github.com/hashicorp/hcl/hcl/strconv" + hcltoken "github.com/hashicorp/hcl/hcl/token" ) // Token defines a single HCL token which can be obtained via the Scanner @@ -139,3 +140,23 @@ func (t Token) Value() interface{} { panic(fmt.Sprintf("unimplemented Value for type: %s", t.Type)) } } + +// HCLToken converts this token to an HCL token. +// +// The token type must be a literal type or this will panic. +func (t Token) HCLToken() hcltoken.Token { + switch t.Type { + case BOOL: + return hcltoken.Token{Type: hcltoken.BOOL, Text: t.Text} + case FLOAT: + return hcltoken.Token{Type: hcltoken.FLOAT, Text: t.Text} + case NULL: + return hcltoken.Token{Type: hcltoken.STRING, Text: ""} + case NUMBER: + return hcltoken.Token{Type: hcltoken.NUMBER, Text: t.Text} + case STRING: + return hcltoken.Token{Type: hcltoken.STRING, Text: t.Text} + default: + panic(fmt.Sprintf("unimplemented HCLToken for type: %s", t.Type)) + } +} From cf0da81e506abb0ead83bc9fe6eb185d22f4b362 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 15:52:57 -0800 Subject: [PATCH 05/13] Remove json files, add flatten.go to json/parse --- json/json_test.go | 4 - json/lex.go | 256 ---------- json/lex_test.go | 78 --- json/parse.y | 227 --------- json/parse_test.go | 43 -- json/{parse.go => parser/flatten.go} | 46 +- json/parser/parser.go | 5 + json/y.go | 713 --------------------------- 8 files changed, 7 insertions(+), 1365 deletions(-) delete mode 100644 json/json_test.go delete mode 100644 json/lex.go delete mode 100644 json/lex_test.go delete mode 100644 json/parse.y delete mode 100644 json/parse_test.go rename json/{parse.go => parser/flatten.go} (75%) delete mode 100644 json/y.go diff --git a/json/json_test.go b/json/json_test.go deleted file mode 100644 index 418582b..0000000 --- a/json/json_test.go +++ /dev/null @@ -1,4 +0,0 @@ -package json - -// This is the directory where our test fixtures are. -const fixtureDir = "./test-fixtures" diff --git a/json/lex.go b/json/lex.go deleted file mode 100644 index 0b07e36..0000000 --- a/json/lex.go +++ /dev/null @@ -1,256 +0,0 @@ -package json - -import ( - "bytes" - "fmt" - "strconv" - "unicode" - "unicode/utf8" -) - -//go:generate go tool yacc -p "json" parse.y - -// This marks the end of the lexer -const lexEOF = 0 - -// The parser uses the type Lex as a lexer. It must provide -// the methods Lex(*SymType) int and Error(string). -type jsonLex struct { - Input string - - pos int - width int - col, line int - err error -} - -// The parser calls this method to get each new token. -func (x *jsonLex) Lex(yylval *jsonSymType) int { - for { - c := x.next() - if c == lexEOF { - return lexEOF - } - - // Ignore all whitespace except a newline which we handle - // specially later. - if unicode.IsSpace(c) { - continue - } - - // If it is a number, lex the number - if c >= '0' && c <= '9' { - x.backup() - return x.lexNumber(yylval) - } - - switch c { - case 'e': - fallthrough - case 'E': - switch x.next() { - case '+': - return EPLUS - case '-': - return EMINUS - default: - x.backup() - return EPLUS - } - case '.': - return PERIOD - case '-': - return MINUS - case ':': - return COLON - case ',': - return COMMA - case '[': - return LEFTBRACKET - case ']': - return RIGHTBRACKET - case '{': - return LEFTBRACE - case '}': - return RIGHTBRACE - case '"': - return x.lexString(yylval) - default: - x.backup() - return x.lexId(yylval) - } - } -} - -// lexId lexes an identifier -func (x *jsonLex) lexId(yylval *jsonSymType) int { - var b bytes.Buffer - first := true - for { - c := x.next() - if c == lexEOF { - break - } - - if !unicode.IsDigit(c) && !unicode.IsLetter(c) && c != '_' && c != '-' { - x.backup() - - if first { - x.createErr("Invalid identifier") - return lexEOF - } - - break - } - - first = false - if _, err := b.WriteRune(c); err != nil { - return lexEOF - } - } - - switch v := b.String(); v { - case "true": - return TRUE - case "false": - return FALSE - case "null": - return NULL - default: - x.createErr(fmt.Sprintf("Invalid identifier: %s", v)) - return lexEOF - } -} - -// lexNumber lexes out a number -func (x *jsonLex) lexNumber(yylval *jsonSymType) int { - var b bytes.Buffer - gotPeriod := false - for { - c := x.next() - if c == lexEOF { - break - } - - if c == '.' { - if gotPeriod { - x.backup() - break - } - - gotPeriod = true - } else if c < '0' || c > '9' { - x.backup() - break - } - - if _, err := b.WriteRune(c); err != nil { - x.createErr(fmt.Sprintf("Internal error: %s", err)) - return lexEOF - } - } - - if !gotPeriod { - v, err := strconv.ParseInt(b.String(), 0, 0) - if err != nil { - x.createErr(fmt.Sprintf("Expected number: %s", err)) - return lexEOF - } - - yylval.num = int(v) - return NUMBER - } - - f, err := strconv.ParseFloat(b.String(), 64) - if err != nil { - x.createErr(fmt.Sprintf("Expected float: %s", err)) - return lexEOF - } - - yylval.f = float64(f) - return FLOAT -} - -// lexString extracts a string from the input -func (x *jsonLex) lexString(yylval *jsonSymType) int { - var b bytes.Buffer - for { - c := x.next() - if c == lexEOF { - break - } - - // String end - if c == '"' { - break - } - - // If we're escaping a quote, then escape the quote - if c == '\\' { - n := x.next() - switch n { - case '"': - c = n - case 'n': - c = '\n' - case '\\': - c = n - default: - x.backup() - } - } - - if _, err := b.WriteRune(c); err != nil { - return lexEOF - } - } - - yylval.str = b.String() - return STRING -} - -// Return the next rune for the lexer. -func (x *jsonLex) next() rune { - if int(x.pos) >= len(x.Input) { - x.width = 0 - return lexEOF - } - - r, w := utf8.DecodeRuneInString(x.Input[x.pos:]) - x.width = w - x.pos += x.width - - x.col += 1 - if x.line == 0 { - x.line = 1 - } - if r == '\n' { - x.line += 1 - x.col = 0 - } - - return r -} - -// peek returns but does not consume the next rune in the input -func (x *jsonLex) peek() rune { - r := x.next() - x.backup() - return r -} - -// backup steps back one rune. Can only be called once per next. -func (x *jsonLex) backup() { - x.col -= 1 - x.pos -= x.width -} - -// createErr records the given error -func (x *jsonLex) createErr(msg string) { - x.err = fmt.Errorf("Line %d, column %d: %s", x.line, x.col, msg) -} - -// The parser calls this method on a parse error. -func (x *jsonLex) Error(s string) { - x.createErr(s) -} diff --git a/json/lex_test.go b/json/lex_test.go deleted file mode 100644 index f573fba..0000000 --- a/json/lex_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package json - -import ( - "io/ioutil" - "path/filepath" - "reflect" - "testing" -) - -func TestLexJson(t *testing.T) { - cases := []struct { - Input string - Output []int - }{ - { - "basic.json", - []int{ - LEFTBRACE, - STRING, COLON, STRING, - RIGHTBRACE, - lexEOF, - }, - }, - { - "array.json", - []int{ - LEFTBRACE, - STRING, COLON, LEFTBRACKET, - NUMBER, COMMA, NUMBER, COMMA, STRING, - RIGHTBRACKET, COMMA, - STRING, COLON, STRING, - RIGHTBRACE, - lexEOF, - }, - }, - { - "object.json", - []int{ - LEFTBRACE, - STRING, COLON, LEFTBRACE, - STRING, COLON, LEFTBRACKET, - NUMBER, COMMA, NUMBER, - RIGHTBRACKET, - RIGHTBRACE, - RIGHTBRACE, - lexEOF, - }, - }, - } - - for _, tc := range cases { - d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.Input)) - if err != nil { - t.Fatalf("err: %s", err) - } - - l := &jsonLex{Input: string(d)} - var actual []int - for { - token := l.Lex(new(jsonSymType)) - actual = append(actual, token) - - if token == lexEOF { - break - } - - if len(actual) > 500 { - t.Fatalf("Input:%s\n\nExausted.", tc.Input) - } - } - - if !reflect.DeepEqual(actual, tc.Output) { - t.Fatalf( - "Input: %s\n\nBad: %#v\n\nExpected: %#v", - tc.Input, actual, tc.Output) - } - } -} diff --git a/json/parse.y b/json/parse.y deleted file mode 100644 index 918e435..0000000 --- a/json/parse.y +++ /dev/null @@ -1,227 +0,0 @@ -// This is the yacc input for creating the parser for HCL JSON. - -%{ -package json - -import ( - "fmt" - "strconv" - - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/hcl/token" -) - -%} - -%union { - f float64 - list []ast.Node - node ast.Node - num int - str string - obj *ast.ObjectType - objitem *ast.ObjectItem - objlist *ast.ObjectList -} - -%type float -%type array elements -%type number value -%type int -%type object -%type pair -%type members -%type exp - -%token FLOAT -%token NUMBER -%token COLON COMMA IDENTIFIER EQUAL NEWLINE STRING -%token LEFTBRACE RIGHTBRACE LEFTBRACKET RIGHTBRACKET -%token TRUE FALSE NULL MINUS PERIOD EPLUS EMINUS - -%% - -top: - object - { - jsonResult = &ast.File{ - Node: $1.List, - } - } - -object: - LEFTBRACE members RIGHTBRACE - { - $$ = &ast.ObjectType{ - List: $2, - } - } -| LEFTBRACE RIGHTBRACE - { - $$ = &ast.ObjectType{} - } - -members: - pair - { - $$ = &ast.ObjectList{ - Items: []*ast.ObjectItem{$1}, - } - } -| members COMMA pair - { - $1.Items = append($1.Items, $3) - $$ = $1 - } - -pair: - STRING COLON value - { - $$ = &ast.ObjectItem{ - Keys: []*ast.ObjectKey{ - &ast.ObjectKey{ - Token: token.Token{ - Type: token.IDENT, - Text: $1, - }, - }, - }, - - Val: $3, - } - } - -value: - STRING - { - $$ = &ast.LiteralType{ - Token: token.Token{ - Type: token.STRING, - Text: fmt.Sprintf(`"%s"`, $1), - }, - } - } -| number - { - $$ = $1 - } -| object - { - $$ = $1 - } -| array - { - $$ = &ast.ListType{ - List: $1, - } - } -| TRUE - { - $$ = &ast.LiteralType{ - Token: token.Token{Type: token.BOOL, Text: "true"}, - } - } -| FALSE - { - $$ = &ast.LiteralType{ - Token: token.Token{Type: token.BOOL, Text: "false"}, - } - } -| NULL - { - $$ = &ast.LiteralType{ - Token: token.Token{Type: token.STRING, Text: ""}, - } - } - -array: - LEFTBRACKET RIGHTBRACKET - { - $$ = nil - } -| LEFTBRACKET elements RIGHTBRACKET - { - $$ = $2 - } - -elements: - value - { - $$ = []ast.Node{$1} - } -| elements COMMA value - { - $$ = append($1, $3) - } - -number: - int - { - $$ = &ast.LiteralType{ - Token: token.Token{ - Type: token.NUMBER, - Text: fmt.Sprintf("%d", $1), - }, - } - } -| float - { - $$ = &ast.LiteralType{ - Token: token.Token{ - Type: token.FLOAT, - Text: fmt.Sprintf("%f", $1), - }, - } - } -| int exp - { - fs := fmt.Sprintf("%d%s", $1, $2) - $$ = &ast.LiteralType{ - Token: token.Token{ - Type: token.FLOAT, - Text: fs, - }, - } - } -| float exp - { - fs := fmt.Sprintf("%f%s", $1, $2) - $$ = &ast.LiteralType{ - Token: token.Token{ - Type: token.FLOAT, - Text: fs, - }, - } - } - -int: - MINUS int - { - $$ = $2 * -1 - } -| NUMBER - { - $$ = $1 - } - -float: - MINUS float - { - $$ = $2 * -1 - } -| FLOAT - { - $$ = $1 - } - -exp: - EPLUS NUMBER - { - $$ = "e" + strconv.FormatInt(int64($2), 10) - } -| EMINUS NUMBER - { - $$ = "e-" + strconv.FormatInt(int64($2), 10) - } - -%% diff --git a/json/parse_test.go b/json/parse_test.go deleted file mode 100644 index 806acb9..0000000 --- a/json/parse_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package json - -import ( - "io/ioutil" - "path/filepath" - "testing" -) - -func TestParse(t *testing.T) { - cases := []struct { - Name string - Err bool - }{ - { - "basic.json", - false, - }, - { - "object.json", - false, - }, - { - "array.json", - false, - }, - { - "types.json", - false, - }, - } - - for _, tc := range cases { - d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.Name)) - if err != nil { - t.Fatalf("err: %s", err) - } - - _, err = Parse(string(d)) - if (err != nil) != tc.Err { - t.Fatalf("Input: %s\n\nError: %s", tc.Name, err) - } - } -} diff --git a/json/parse.go b/json/parser/flatten.go similarity index 75% rename from json/parse.go rename to json/parser/flatten.go index 86d2660..64fae32 100644 --- a/json/parse.go +++ b/json/parser/flatten.go @@ -1,54 +1,12 @@ -package json +package parser import ( - "sync" - - "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl/hcl/ast" ) -// jsonErrors are the errors built up from parsing. These should not -// be accessed directly. -var jsonErrors []error -var jsonLock sync.Mutex -var jsonResult *ast.File - -// Parse parses the given string and returns the result. -func Parse(v string) (*ast.File, error) { - jsonLock.Lock() - defer jsonLock.Unlock() - jsonErrors = nil - jsonResult = nil - - // Parse - lex := &jsonLex{Input: v} - jsonParse(lex) - - // If we have an error in the lexer itself, return it - if lex.err != nil { - return nil, lex.err - } - - // If we have a result, flatten it. This is an operation we take on - // to make our AST look more like traditional HCL. This makes parsing - // it a lot easier later. - if jsonResult != nil { - flattenObjects(jsonResult) - } - - // Build up the errors - var err error - if len(jsonErrors) > 0 { - err = &multierror.Error{Errors: jsonErrors} - jsonResult = nil - } - - return jsonResult, err -} - // flattenObjects takes an AST node, walks it, and flattens func flattenObjects(node ast.Node) { - ast.Walk(jsonResult, func(n ast.Node) bool { + ast.Walk(node, func(n ast.Node) bool { // We only care about lists, because this is what we modify list, ok := n.(*ast.ObjectList) if !ok { diff --git a/json/parser/parser.go b/json/parser/parser.go index 1f7db62..19895a4 100644 --- a/json/parser/parser.go +++ b/json/parser/parser.go @@ -54,6 +54,11 @@ func (p *Parser) Parse() (*ast.File, error) { // We make our final node an object list so it is more HCL compatible f.Node = object.List + + // Flatten it, which finds patterns and turns them into more HCL-like + // AST trees. + flattenObjects(f.Node) + return f, nil } diff --git a/json/y.go b/json/y.go deleted file mode 100644 index 001c5e0..0000000 --- a/json/y.go +++ /dev/null @@ -1,713 +0,0 @@ -//line parse.y:3 -package json - -import __yyfmt__ "fmt" - -//line parse.y:5 -import ( - "fmt" - "strconv" - - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/hcl/token" -) - -//line parse.y:16 -type jsonSymType struct { - yys int - f float64 - list []ast.Node - node ast.Node - num int - str string - obj *ast.ObjectType - objitem *ast.ObjectItem - objlist *ast.ObjectList -} - -const FLOAT = 57346 -const NUMBER = 57347 -const COLON = 57348 -const COMMA = 57349 -const IDENTIFIER = 57350 -const EQUAL = 57351 -const NEWLINE = 57352 -const STRING = 57353 -const LEFTBRACE = 57354 -const RIGHTBRACE = 57355 -const LEFTBRACKET = 57356 -const RIGHTBRACKET = 57357 -const TRUE = 57358 -const FALSE = 57359 -const NULL = 57360 -const MINUS = 57361 -const PERIOD = 57362 -const EPLUS = 57363 -const EMINUS = 57364 - -var jsonToknames = [...]string{ - "$end", - "error", - "$unk", - "FLOAT", - "NUMBER", - "COLON", - "COMMA", - "IDENTIFIER", - "EQUAL", - "NEWLINE", - "STRING", - "LEFTBRACE", - "RIGHTBRACE", - "LEFTBRACKET", - "RIGHTBRACKET", - "TRUE", - "FALSE", - "NULL", - "MINUS", - "PERIOD", - "EPLUS", - "EMINUS", -} -var jsonStatenames = [...]string{} - -const jsonEofCode = 1 -const jsonErrCode = 2 -const jsonMaxDepth = 200 - -//line parse.y:227 - -//line yacctab:1 -var jsonExca = [...]int{ - -1, 1, - 1, -1, - -2, 0, -} - -const jsonNprod = 28 -const jsonPrivate = 57344 - -var jsonTokenNames []string -var jsonStates []string - -const jsonLast = 53 - -var jsonAct = [...]int{ - - 12, 25, 24, 3, 20, 27, 28, 7, 13, 3, - 21, 22, 30, 17, 18, 19, 23, 25, 24, 26, - 25, 24, 36, 32, 13, 3, 10, 22, 33, 17, - 18, 19, 23, 35, 34, 23, 38, 9, 7, 39, - 5, 29, 6, 8, 37, 15, 2, 1, 4, 14, - 31, 16, 11, -} -var jsonPact = [...]int{ - - -9, -1000, -1000, 27, 30, -1000, -1000, 20, -1000, -4, - 13, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -16, -16, -3, 16, -1000, -1000, -1000, 28, 17, -1000, - -1000, 29, -1000, -1000, -1000, -1000, -1000, -1000, 13, -1000, -} -var jsonPgo = [...]int{ - - 0, 10, 51, 50, 49, 0, 4, 45, 42, 48, - 19, 47, -} -var jsonR1 = [...]int{ - - 0, 11, 7, 7, 9, 9, 8, 5, 5, 5, - 5, 5, 5, 5, 2, 2, 3, 3, 4, 4, - 4, 4, 6, 6, 1, 1, 10, 10, -} -var jsonR2 = [...]int{ - - 0, 1, 3, 2, 1, 3, 3, 1, 1, 1, - 1, 1, 1, 1, 2, 3, 1, 3, 1, 1, - 2, 2, 2, 1, 2, 1, 2, 2, -} -var jsonChk = [...]int{ - - -1000, -11, -7, 12, -9, 13, -8, 11, 13, 7, - 6, -8, -5, 11, -4, -7, -2, 16, 17, 18, - -6, -1, 14, 19, 5, 4, -10, 21, 22, -10, - 15, -3, -5, -6, -1, 5, 5, 15, 7, -5, -} -var jsonDef = [...]int{ - - 0, -2, 1, 0, 0, 3, 4, 0, 2, 0, - 0, 5, 6, 7, 8, 9, 10, 11, 12, 13, - 18, 19, 0, 0, 23, 25, 20, 0, 0, 21, - 14, 0, 16, 22, 24, 26, 27, 15, 0, 17, -} -var jsonTok1 = [...]int{ - - 1, -} -var jsonTok2 = [...]int{ - - 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, - 22, -} -var jsonTok3 = [...]int{ - 0, -} - -var jsonErrorMessages = [...]struct { - state int - token int - msg string -}{} - -//line yaccpar:1 - -/* parser for yacc output */ - -var ( - jsonDebug = 0 - jsonErrorVerbose = false -) - -type jsonLexer interface { - Lex(lval *jsonSymType) int - Error(s string) -} - -type jsonParser interface { - Parse(jsonLexer) int - Lookahead() int -} - -type jsonParserImpl struct { - lookahead func() int -} - -func (p *jsonParserImpl) Lookahead() int { - return p.lookahead() -} - -func jsonNewParser() jsonParser { - p := &jsonParserImpl{ - lookahead: func() int { return -1 }, - } - return p -} - -const jsonFlag = -1000 - -func jsonTokname(c int) string { - if c >= 1 && c-1 < len(jsonToknames) { - if jsonToknames[c-1] != "" { - return jsonToknames[c-1] - } - } - return __yyfmt__.Sprintf("tok-%v", c) -} - -func jsonStatname(s int) string { - if s >= 0 && s < len(jsonStatenames) { - if jsonStatenames[s] != "" { - return jsonStatenames[s] - } - } - return __yyfmt__.Sprintf("state-%v", s) -} - -func jsonErrorMessage(state, lookAhead int) string { - const TOKSTART = 4 - - if !jsonErrorVerbose { - return "syntax error" - } - - for _, e := range jsonErrorMessages { - if e.state == state && e.token == lookAhead { - return "syntax error: " + e.msg - } - } - - res := "syntax error: unexpected " + jsonTokname(lookAhead) - - // To match Bison, suggest at most four expected tokens. - expected := make([]int, 0, 4) - - // Look for shiftable tokens. - base := jsonPact[state] - for tok := TOKSTART; tok-1 < len(jsonToknames); tok++ { - if n := base + tok; n >= 0 && n < jsonLast && jsonChk[jsonAct[n]] == tok { - if len(expected) == cap(expected) { - return res - } - expected = append(expected, tok) - } - } - - if jsonDef[state] == -2 { - i := 0 - for jsonExca[i] != -1 || jsonExca[i+1] != state { - i += 2 - } - - // Look for tokens that we accept or reduce. - for i += 2; jsonExca[i] >= 0; i += 2 { - tok := jsonExca[i] - if tok < TOKSTART || jsonExca[i+1] == 0 { - continue - } - if len(expected) == cap(expected) { - return res - } - expected = append(expected, tok) - } - - // If the default action is to accept or reduce, give up. - if jsonExca[i+1] != 0 { - return res - } - } - - for i, tok := range expected { - if i == 0 { - res += ", expecting " - } else { - res += " or " - } - res += jsonTokname(tok) - } - return res -} - -func jsonlex1(lex jsonLexer, lval *jsonSymType) (char, token int) { - token = 0 - char = lex.Lex(lval) - if char <= 0 { - token = jsonTok1[0] - goto out - } - if char < len(jsonTok1) { - token = jsonTok1[char] - goto out - } - if char >= jsonPrivate { - if char < jsonPrivate+len(jsonTok2) { - token = jsonTok2[char-jsonPrivate] - goto out - } - } - for i := 0; i < len(jsonTok3); i += 2 { - token = jsonTok3[i+0] - if token == char { - token = jsonTok3[i+1] - goto out - } - } - -out: - if token == 0 { - token = jsonTok2[1] /* unknown char */ - } - if jsonDebug >= 3 { - __yyfmt__.Printf("lex %s(%d)\n", jsonTokname(token), uint(char)) - } - return char, token -} - -func jsonParse(jsonlex jsonLexer) int { - return jsonNewParser().Parse(jsonlex) -} - -func (jsonrcvr *jsonParserImpl) Parse(jsonlex jsonLexer) int { - var jsonn int - var jsonlval jsonSymType - var jsonVAL jsonSymType - var jsonDollar []jsonSymType - _ = jsonDollar // silence set and not used - jsonS := make([]jsonSymType, jsonMaxDepth) - - Nerrs := 0 /* number of errors */ - Errflag := 0 /* error recovery flag */ - jsonstate := 0 - jsonchar := -1 - jsontoken := -1 // jsonchar translated into internal numbering - jsonrcvr.lookahead = func() int { return jsonchar } - defer func() { - // Make sure we report no lookahead when not parsing. - jsonstate = -1 - jsonchar = -1 - jsontoken = -1 - }() - jsonp := -1 - goto jsonstack - -ret0: - return 0 - -ret1: - return 1 - -jsonstack: - /* put a state and value onto the stack */ - if jsonDebug >= 4 { - __yyfmt__.Printf("char %v in %v\n", jsonTokname(jsontoken), jsonStatname(jsonstate)) - } - - jsonp++ - if jsonp >= len(jsonS) { - nyys := make([]jsonSymType, len(jsonS)*2) - copy(nyys, jsonS) - jsonS = nyys - } - jsonS[jsonp] = jsonVAL - jsonS[jsonp].yys = jsonstate - -jsonnewstate: - jsonn = jsonPact[jsonstate] - if jsonn <= jsonFlag { - goto jsondefault /* simple state */ - } - if jsonchar < 0 { - jsonchar, jsontoken = jsonlex1(jsonlex, &jsonlval) - } - jsonn += jsontoken - if jsonn < 0 || jsonn >= jsonLast { - goto jsondefault - } - jsonn = jsonAct[jsonn] - if jsonChk[jsonn] == jsontoken { /* valid shift */ - jsonchar = -1 - jsontoken = -1 - jsonVAL = jsonlval - jsonstate = jsonn - if Errflag > 0 { - Errflag-- - } - goto jsonstack - } - -jsondefault: - /* default state action */ - jsonn = jsonDef[jsonstate] - if jsonn == -2 { - if jsonchar < 0 { - jsonchar, jsontoken = jsonlex1(jsonlex, &jsonlval) - } - - /* look through exception table */ - xi := 0 - for { - if jsonExca[xi+0] == -1 && jsonExca[xi+1] == jsonstate { - break - } - xi += 2 - } - for xi += 2; ; xi += 2 { - jsonn = jsonExca[xi+0] - if jsonn < 0 || jsonn == jsontoken { - break - } - } - jsonn = jsonExca[xi+1] - if jsonn < 0 { - goto ret0 - } - } - if jsonn == 0 { - /* error ... attempt to resume parsing */ - switch Errflag { - case 0: /* brand new error */ - jsonlex.Error(jsonErrorMessage(jsonstate, jsontoken)) - Nerrs++ - if jsonDebug >= 1 { - __yyfmt__.Printf("%s", jsonStatname(jsonstate)) - __yyfmt__.Printf(" saw %s\n", jsonTokname(jsontoken)) - } - fallthrough - - case 1, 2: /* incompletely recovered error ... try again */ - Errflag = 3 - - /* find a state where "error" is a legal shift action */ - for jsonp >= 0 { - jsonn = jsonPact[jsonS[jsonp].yys] + jsonErrCode - if jsonn >= 0 && jsonn < jsonLast { - jsonstate = jsonAct[jsonn] /* simulate a shift of "error" */ - if jsonChk[jsonstate] == jsonErrCode { - goto jsonstack - } - } - - /* the current p has no shift on "error", pop stack */ - if jsonDebug >= 2 { - __yyfmt__.Printf("error recovery pops state %d\n", jsonS[jsonp].yys) - } - jsonp-- - } - /* there is no state on the stack with an error shift ... abort */ - goto ret1 - - case 3: /* no shift yet; clobber input char */ - if jsonDebug >= 2 { - __yyfmt__.Printf("error recovery discards %s\n", jsonTokname(jsontoken)) - } - if jsontoken == jsonEofCode { - goto ret1 - } - jsonchar = -1 - jsontoken = -1 - goto jsonnewstate /* try again in the same state */ - } - } - - /* reduction by production jsonn */ - if jsonDebug >= 2 { - __yyfmt__.Printf("reduce %v in:\n\t%v\n", jsonn, jsonStatname(jsonstate)) - } - - jsonnt := jsonn - jsonpt := jsonp - _ = jsonpt // guard against "declared and not used" - - jsonp -= jsonR2[jsonn] - // jsonp is now the index of $0. Perform the default action. Iff the - // reduced production is ε, $1 is possibly out of range. - if jsonp+1 >= len(jsonS) { - nyys := make([]jsonSymType, len(jsonS)*2) - copy(nyys, jsonS) - jsonS = nyys - } - jsonVAL = jsonS[jsonp+1] - - /* consult goto table to find next state */ - jsonn = jsonR1[jsonn] - jsong := jsonPgo[jsonn] - jsonj := jsong + jsonS[jsonp].yys + 1 - - if jsonj >= jsonLast { - jsonstate = jsonAct[jsong] - } else { - jsonstate = jsonAct[jsonj] - if jsonChk[jsonstate] != -jsonn { - jsonstate = jsonAct[jsong] - } - } - // dummy call; replaced with literal code - switch jsonnt { - - case 1: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:46 - { - jsonResult = &ast.File{ - Node: jsonDollar[1].obj.List, - } - } - case 2: - jsonDollar = jsonS[jsonpt-3 : jsonpt+1] - //line parse.y:54 - { - jsonVAL.obj = &ast.ObjectType{ - List: jsonDollar[2].objlist, - } - } - case 3: - jsonDollar = jsonS[jsonpt-2 : jsonpt+1] - //line parse.y:60 - { - jsonVAL.obj = &ast.ObjectType{} - } - case 4: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:66 - { - jsonVAL.objlist = &ast.ObjectList{ - Items: []*ast.ObjectItem{jsonDollar[1].objitem}, - } - } - case 5: - jsonDollar = jsonS[jsonpt-3 : jsonpt+1] - //line parse.y:72 - { - jsonDollar[1].objlist.Items = append(jsonDollar[1].objlist.Items, jsonDollar[3].objitem) - jsonVAL.objlist = jsonDollar[1].objlist - } - case 6: - jsonDollar = jsonS[jsonpt-3 : jsonpt+1] - //line parse.y:79 - { - jsonVAL.objitem = &ast.ObjectItem{ - Keys: []*ast.ObjectKey{ - &ast.ObjectKey{ - Token: token.Token{ - Type: token.IDENT, - Text: jsonDollar[1].str, - }, - }, - }, - - Val: jsonDollar[3].node, - } - } - case 7: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:96 - { - jsonVAL.node = &ast.LiteralType{ - Token: token.Token{ - Type: token.STRING, - Text: fmt.Sprintf(`"%s"`, jsonDollar[1].str), - }, - } - } - case 8: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:105 - { - jsonVAL.node = jsonDollar[1].node - } - case 9: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:109 - { - jsonVAL.node = jsonDollar[1].obj - } - case 10: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:113 - { - jsonVAL.node = &ast.ListType{ - List: jsonDollar[1].list, - } - } - case 11: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:119 - { - jsonVAL.node = &ast.LiteralType{ - Token: token.Token{Type: token.BOOL, Text: "true"}, - } - } - case 12: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:125 - { - jsonVAL.node = &ast.LiteralType{ - Token: token.Token{Type: token.BOOL, Text: "false"}, - } - } - case 13: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:131 - { - jsonVAL.node = &ast.LiteralType{ - Token: token.Token{Type: token.STRING, Text: ""}, - } - } - case 14: - jsonDollar = jsonS[jsonpt-2 : jsonpt+1] - //line parse.y:139 - { - jsonVAL.list = nil - } - case 15: - jsonDollar = jsonS[jsonpt-3 : jsonpt+1] - //line parse.y:143 - { - jsonVAL.list = jsonDollar[2].list - } - case 16: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:149 - { - jsonVAL.list = []ast.Node{jsonDollar[1].node} - } - case 17: - jsonDollar = jsonS[jsonpt-3 : jsonpt+1] - //line parse.y:153 - { - jsonVAL.list = append(jsonDollar[1].list, jsonDollar[3].node) - } - case 18: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:159 - { - jsonVAL.node = &ast.LiteralType{ - Token: token.Token{ - Type: token.NUMBER, - Text: fmt.Sprintf("%d", jsonDollar[1].num), - }, - } - } - case 19: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:168 - { - jsonVAL.node = &ast.LiteralType{ - Token: token.Token{ - Type: token.FLOAT, - Text: fmt.Sprintf("%f", jsonDollar[1].f), - }, - } - } - case 20: - jsonDollar = jsonS[jsonpt-2 : jsonpt+1] - //line parse.y:177 - { - fs := fmt.Sprintf("%d%s", jsonDollar[1].num, jsonDollar[2].str) - jsonVAL.node = &ast.LiteralType{ - Token: token.Token{ - Type: token.FLOAT, - Text: fs, - }, - } - } - case 21: - jsonDollar = jsonS[jsonpt-2 : jsonpt+1] - //line parse.y:187 - { - fs := fmt.Sprintf("%f%s", jsonDollar[1].f, jsonDollar[2].str) - jsonVAL.node = &ast.LiteralType{ - Token: token.Token{ - Type: token.FLOAT, - Text: fs, - }, - } - } - case 22: - jsonDollar = jsonS[jsonpt-2 : jsonpt+1] - //line parse.y:199 - { - jsonVAL.num = jsonDollar[2].num * -1 - } - case 23: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:203 - { - jsonVAL.num = jsonDollar[1].num - } - case 24: - jsonDollar = jsonS[jsonpt-2 : jsonpt+1] - //line parse.y:209 - { - jsonVAL.f = jsonDollar[2].f * -1 - } - case 25: - jsonDollar = jsonS[jsonpt-1 : jsonpt+1] - //line parse.y:213 - { - jsonVAL.f = jsonDollar[1].f - } - case 26: - jsonDollar = jsonS[jsonpt-2 : jsonpt+1] - //line parse.y:219 - { - jsonVAL.str = "e" + strconv.FormatInt(int64(jsonDollar[2].num), 10) - } - case 27: - jsonDollar = jsonS[jsonpt-2 : jsonpt+1] - //line parse.y:223 - { - jsonVAL.str = "e-" + strconv.FormatInt(int64(jsonDollar[2].num), 10) - } - } - goto jsonstack /* stack new state and value */ -} From b3a9867999e3e666a468fdbbbbbbb64d3b61dc35 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 15:53:33 -0800 Subject: [PATCH 06/13] replace parse with new json parser --- parse.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parse.go b/parse.go index 34267a3..d0719c2 100644 --- a/parse.go +++ b/parse.go @@ -5,7 +5,7 @@ import ( "github.com/hashicorp/hcl/hcl/ast" hclParser "github.com/hashicorp/hcl/hcl/parser" - "github.com/hashicorp/hcl/json" + jsonParser "github.com/hashicorp/hcl/json/parser" ) // Parse parses the given input and returns the root object. @@ -16,7 +16,7 @@ func Parse(input string) (*ast.File, error) { case lexModeHcl: return hclParser.Parse([]byte(input)) case lexModeJson: - return json.Parse(input) + return jsonParser.Parse([]byte(input)) } return nil, fmt.Errorf("unknown config format") From e9e082dff431af918f0e86842697c346e96089b3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 15:54:48 -0800 Subject: [PATCH 07/13] json/token: remove Value --- json/token/token.go | 44 ---------------------------------------- json/token/token_test.go | 23 --------------------- 2 files changed, 67 deletions(-) diff --git a/json/token/token.go b/json/token/token.go index 034adcc..96c8f67 100644 --- a/json/token/token.go +++ b/json/token/token.go @@ -4,7 +4,6 @@ import ( "fmt" "strconv" - hclstrconv "github.com/hashicorp/hcl/hcl/strconv" hcltoken "github.com/hashicorp/hcl/hcl/token" ) @@ -98,49 +97,6 @@ func (t Token) String() string { return fmt.Sprintf("%s %s %s", t.Pos.String(), t.Type.String(), t.Text) } -// Value returns the properly typed value for this token. The type of -// the returned interface{} is guaranteed based on the Type field. -// -// This can only be called for literal types. If it is called for any other -// type, this will panic. -func (t Token) Value() interface{} { - switch t.Type { - case BOOL: - if t.Text == "true" { - return true - } else if t.Text == "false" { - return false - } - - panic("unknown bool value: " + t.Text) - case FLOAT: - v, err := strconv.ParseFloat(t.Text, 64) - if err != nil { - panic(err) - } - - return float64(v) - case NULL: - return nil - case NUMBER: - v, err := strconv.ParseInt(t.Text, 0, 64) - if err != nil { - panic(err) - } - - return int64(v) - case STRING: - v, err := hclstrconv.Unquote(t.Text) - if err != nil { - panic(fmt.Sprintf("unquote %s err: %s", t.Text, err)) - } - - return v - default: - panic(fmt.Sprintf("unimplemented Value for type: %s", t.Type)) - } -} - // HCLToken converts this token to an HCL token. // // The token type must be a literal type or this will panic. diff --git a/json/token/token_test.go b/json/token/token_test.go index ea39e40..a83fdd5 100644 --- a/json/token/token_test.go +++ b/json/token/token_test.go @@ -1,7 +1,6 @@ package token import ( - "reflect" "testing" ) @@ -33,25 +32,3 @@ func TestTypeString(t *testing.T) { } } - -func TestTokenValue(t *testing.T) { - var tokens = []struct { - tt Token - v interface{} - }{ - {Token{Type: BOOL, Text: `true`}, true}, - {Token{Type: BOOL, Text: `false`}, false}, - {Token{Type: FLOAT, Text: `3.14`}, float64(3.14)}, - {Token{Type: NULL, Text: `null`}, nil}, - {Token{Type: NUMBER, Text: `42`}, int64(42)}, - {Token{Type: STRING, Text: `"foo"`}, "foo"}, - {Token{Type: STRING, Text: `"foo\nbar"`}, "foo\nbar"}, - } - - for _, token := range tokens { - if val := token.tt.Value(); !reflect.DeepEqual(val, token.v) { - t.Errorf("want: %v got:%v\n", token.v, val) - } - } - -} From 340f0af3c0f95770e6ffddb0db0e665907321f72 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 16:08:36 -0800 Subject: [PATCH 08/13] json/token: fix issues with unquoting sttrings from JSON --- hcl/token/token.go | 11 ++++++++++- json/parser/parser.go | 2 +- json/token/token.go | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/hcl/token/token.go b/hcl/token/token.go index e8f9635..c93f077 100644 --- a/hcl/token/token.go +++ b/hcl/token/token.go @@ -14,6 +14,7 @@ type Token struct { Type Type Pos Pos Text string + JSON bool } // Type is the set of lexical tokens of the HCL (HashiCorp Configuration Language) @@ -138,7 +139,15 @@ func (t Token) Value() interface{} { case IDENT: return t.Text case STRING: - v, err := hclstrconv.Unquote(t.Text) + // Determine the Unquote method to use. If it came from JSON, + // then we need to use the built-in unquote since we have to + // escape interpolations there. + f := hclstrconv.Unquote + if t.JSON { + f = strconv.Unquote + } + + v, err := f(t.Text) if err != nil { panic(fmt.Sprintf("unquote %s err: %s", t.Text, err)) } diff --git a/json/parser/parser.go b/json/parser/parser.go index 19895a4..450b875 100644 --- a/json/parser/parser.go +++ b/json/parser/parser.go @@ -57,7 +57,7 @@ func (p *Parser) Parse() (*ast.File, error) { // Flatten it, which finds patterns and turns them into more HCL-like // AST trees. - flattenObjects(f.Node) + // flattenObjects(f.Node) return f, nil } diff --git a/json/token/token.go b/json/token/token.go index 96c8f67..95a0c3e 100644 --- a/json/token/token.go +++ b/json/token/token.go @@ -111,7 +111,7 @@ func (t Token) HCLToken() hcltoken.Token { case NUMBER: return hcltoken.Token{Type: hcltoken.NUMBER, Text: t.Text} case STRING: - return hcltoken.Token{Type: hcltoken.STRING, Text: t.Text} + return hcltoken.Token{Type: hcltoken.STRING, Text: t.Text, JSON: true} default: panic(fmt.Sprintf("unimplemented HCLToken for type: %s", t.Type)) } From 7ccda83497cc7af4f35f8d33c54eb89e77544db0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 16:11:15 -0800 Subject: [PATCH 09/13] json/parser: can parse multiple objects --- json/parser/parser.go | 1 - 1 file changed, 1 deletion(-) diff --git a/json/parser/parser.go b/json/parser/parser.go index 450b875..b4ee36e 100644 --- a/json/parser/parser.go +++ b/json/parser/parser.go @@ -82,7 +82,6 @@ func (p *Parser) objectList() (*ast.ObjectList, error) { // Check for a followup comma. If it isn't a comma, then we're done if tok := p.scan(); tok.Type != token.COMMA { - p.unscan() break } } From 278bb105f21b5a494a6b3d609c83d98876638f50 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 16:20:22 -0800 Subject: [PATCH 10/13] hcl/ast: style --- hcl/ast/ast.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hcl/ast/ast.go b/hcl/ast/ast.go index 67acf96..ee31d7e 100644 --- a/hcl/ast/ast.go +++ b/hcl/ast/ast.go @@ -14,11 +14,10 @@ type Node interface { Pos() token.Pos } -func (File) node() {} -func (ObjectList) node() {} -func (ObjectKey) node() {} -func (ObjectItem) node() {} - +func (File) node() {} +func (ObjectList) node() {} +func (ObjectKey) node() {} +func (ObjectItem) node() {} func (Comment) node() {} func (CommentGroup) node() {} func (ObjectType) node() {} From 407a4ff94cb1a443097f736c1b047fb408aae29d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 16:20:34 -0800 Subject: [PATCH 11/13] json/parser: flatten --- json/parser/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json/parser/parser.go b/json/parser/parser.go index b4ee36e..0bbd821 100644 --- a/json/parser/parser.go +++ b/json/parser/parser.go @@ -57,7 +57,7 @@ func (p *Parser) Parse() (*ast.File, error) { // Flatten it, which finds patterns and turns them into more HCL-like // AST trees. - // flattenObjects(f.Node) + flattenObjects(f.Node) return f, nil } From eabb04dcd5540671368a9d37701afe0731fa599b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 16:25:40 -0800 Subject: [PATCH 12/13] hcl/ast: filter should compare value string --- hcl/ast/ast.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hcl/ast/ast.go b/hcl/ast/ast.go index ee31d7e..079d19e 100644 --- a/hcl/ast/ast.go +++ b/hcl/ast/ast.go @@ -61,7 +61,7 @@ func (o *ObjectList) Filter(keys ...string) *ObjectList { match := true for i, key := range item.Keys[:len(keys)] { - key := key.Token.Text + key := key.Token.Value().(string) if key != keys[i] && !strings.EqualFold(key, keys[i]) { match = false break From 637d86c42d9c969352384f98558d855a636b30fa Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 8 Nov 2015 16:28:18 -0800 Subject: [PATCH 13/13] json/parser: can parse lists of objects --- json/parser/parser.go | 7 +++++++ json/parser/parser_test.go | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/json/parser/parser.go b/json/parser/parser.go index 0bbd821..65d56c9 100644 --- a/json/parser/parser.go +++ b/json/parser/parser.go @@ -212,6 +212,13 @@ func (p *Parser) listType() (*ast.ListType, error) { l.Add(node) case token.COMMA: continue + case token.LBRACE: + node, err := p.objectType() + if err != nil { + return nil, err + } + + l.Add(node) case token.BOOL: // TODO(arslan) should we support? not supported by HCL yet case token.LBRACK: diff --git a/json/parser/parser_test.go b/json/parser/parser_test.go index 9c77daf..96f67ea 100644 --- a/json/parser/parser_test.go +++ b/json/parser/parser_test.go @@ -66,6 +66,10 @@ func TestListType(t *testing.T) { `"foo": ["123", 123]`, []token.Type{token.STRING, token.NUMBER}, }, + { + `"foo": ["123", {}]`, + []token.Type{token.STRING, token.LBRACE}, + }, } for _, l := range literals { @@ -84,8 +88,11 @@ func TestListType(t *testing.T) { tokens := []token.Type{} for _, li := range list.List { - if tp, ok := li.(*ast.LiteralType); ok { - tokens = append(tokens, tp.Token.Type) + switch v := li.(type) { + case *ast.LiteralType: + tokens = append(tokens, v.Token.Type) + case *ast.ObjectType: + tokens = append(tokens, token.LBRACE) } }