From 4de51957ef8d4aba6e285ddfc587633bbfc7c0e8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 16 Sep 2015 09:54:29 -0700 Subject: [PATCH] hcl: fix trailing comma + comment parse error https://github.com/hashicorp/terraform/issues/3249 --- hcl/lex.go | 23 +- hcl/lex_test.go | 9 + hcl/parse.y | 10 +- hcl/parse_test.go | 4 + hcl/test-fixtures/array_comment.hcl | 4 + hcl/y.go | 399 +++++++++++++++++++--------- json/y.go | 273 ++++++++++++++----- 7 files changed, 507 insertions(+), 215 deletions(-) create mode 100644 hcl/test-fixtures/array_comment.hcl diff --git a/hcl/lex.go b/hcl/lex.go index c141659..54eb1c8 100644 --- a/hcl/lex.go +++ b/hcl/lex.go @@ -85,7 +85,7 @@ func (x *hclLex) Lex(yylval *hclSymType) int { case '-': return MINUS case ',': - return x.lexComma() + return COMMA case '=': return EQUAL case '[': @@ -166,27 +166,6 @@ func (x *hclLex) consumeComment(c rune) bool { } } -// lexComma reads the comma -func (x *hclLex) lexComma() int { - for { - c := x.peek() - - // Consume space - if unicode.IsSpace(c) { - x.next() - continue - } - - if c == ']' { - return COMMAEND - } - - break - } - - return COMMA -} - // lexId lexes an identifier func (x *hclLex) lexId(yylval *hclSymType) int { var b bytes.Buffer diff --git a/hcl/lex_test.go b/hcl/lex_test.go index dcdd9b3..909c0e2 100644 --- a/hcl/lex_test.go +++ b/hcl/lex_test.go @@ -63,6 +63,15 @@ func TestLex(t *testing.T) { RIGHTBRACE, lexEOF, }, }, + { + "array_comment.hcl", + []int{ + IDENTIFIER, EQUAL, LEFTBRACKET, + STRING, COMMA, + STRING, COMMA, + RIGHTBRACKET, lexEOF, + }, + }, } for _, tc := range cases { diff --git a/hcl/parse.y b/hcl/parse.y index 4f42d34..6144a87 100644 --- a/hcl/parse.y +++ b/hcl/parse.y @@ -29,7 +29,7 @@ import ( %token BOOL %token FLOAT %token NUMBER -%token COMMA COMMAEND IDENTIFIER EQUAL NEWLINE STRING MINUS +%token COMMA IDENTIFIER EQUAL NEWLINE STRING MINUS %token LEFTBRACE RIGHTBRACE LEFTBRACKET RIGHTBRACKET PERIOD %token EPLUS EMINUS @@ -152,6 +152,10 @@ list: { $$ = $2 } +| LEFTBRACKET listitems COMMA RIGHTBRACKET + { + $$ = $2 + } | LEFTBRACKET RIGHTBRACKET { $$ = nil @@ -166,10 +170,6 @@ listitems: { $$ = append($1, $3) } -| listitems COMMAEND - { - $$ = $1 - } listitem: number diff --git a/hcl/parse_test.go b/hcl/parse_test.go index ea3047b..1666bb5 100644 --- a/hcl/parse_test.go +++ b/hcl/parse_test.go @@ -59,6 +59,10 @@ func TestParse(t *testing.T) { "types.hcl", false, }, + { + "array_comment.hcl", + false, + }, } for _, tc := range cases { diff --git a/hcl/test-fixtures/array_comment.hcl b/hcl/test-fixtures/array_comment.hcl new file mode 100644 index 0000000..78c2675 --- /dev/null +++ b/hcl/test-fixtures/array_comment.hcl @@ -0,0 +1,4 @@ +foo = [ + "1", + "2", # comment +] diff --git a/hcl/y.go b/hcl/y.go index f139a24..f3ae8f2 100644 --- a/hcl/y.go +++ b/hcl/y.go @@ -24,26 +24,27 @@ const BOOL = 57346 const FLOAT = 57347 const NUMBER = 57348 const COMMA = 57349 -const COMMAEND = 57350 -const IDENTIFIER = 57351 -const EQUAL = 57352 -const NEWLINE = 57353 -const STRING = 57354 -const MINUS = 57355 -const LEFTBRACE = 57356 -const RIGHTBRACE = 57357 -const LEFTBRACKET = 57358 -const RIGHTBRACKET = 57359 -const PERIOD = 57360 -const EPLUS = 57361 -const EMINUS = 57362 +const IDENTIFIER = 57350 +const EQUAL = 57351 +const NEWLINE = 57352 +const STRING = 57353 +const MINUS = 57354 +const LEFTBRACE = 57355 +const RIGHTBRACE = 57356 +const LEFTBRACKET = 57357 +const RIGHTBRACKET = 57358 +const PERIOD = 57359 +const EPLUS = 57360 +const EMINUS = 57361 -var hclToknames = []string{ +var hclToknames = [...]string{ + "$end", + "error", + "$unk", "BOOL", "FLOAT", "NUMBER", "COMMA", - "COMMAEND", "IDENTIFIER", "EQUAL", "NEWLINE", @@ -57,7 +58,7 @@ var hclToknames = []string{ "EPLUS", "EMINUS", } -var hclStatenames = []string{} +var hclStatenames = [...]string{} const hclEofCode = 1 const hclErrCode = 2 @@ -66,15 +67,15 @@ const hclMaxDepth = 200 //line parse.y:259 //line yacctab:1 -var hclExca = []int{ +var hclExca = [...]int{ -1, 1, 1, -1, -2, 0, -1, 6, - 10, 7, + 9, 7, -2, 17, -1, 7, - 10, 8, + 9, 8, -2, 18, } @@ -84,92 +85,120 @@ const hclPrivate = 57344 var hclTokenNames []string var hclStates []string -const hclLast = 62 +const hclLast = 64 -var hclAct = []int{ +var hclAct = [...]int{ 35, 3, 21, 22, 9, 30, 31, 29, 17, 26, - 25, 26, 25, 10, 26, 25, 18, 24, 13, 24, - 23, 37, 24, 44, 45, 42, 34, 38, 39, 9, - 32, 6, 6, 43, 7, 7, 2, 40, 28, 26, - 25, 6, 41, 11, 7, 46, 37, 24, 14, 36, - 27, 15, 5, 13, 19, 1, 4, 8, 33, 20, - 16, 12, + 25, 11, 44, 26, 25, 18, 24, 13, 10, 23, + 24, 43, 19, 2, 42, 26, 25, 38, 39, 9, + 32, 37, 24, 26, 25, 6, 45, 27, 7, 37, + 24, 40, 36, 6, 34, 46, 7, 14, 6, 28, + 15, 7, 13, 16, 5, 41, 1, 4, 8, 33, + 20, 0, 0, 12, } -var hclPact = []int{ +var hclPact = [...]int{ - 32, -1000, 32, -1000, 3, -1000, -1000, -1000, 39, -1000, - 4, -1000, -1000, 23, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -14, -14, 9, 6, -1000, -1000, 22, -1000, -1000, - 36, 19, -1000, 16, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 34, -1000, -1000, + 40, -1000, 40, -1000, 9, -1000, -1000, -1000, 39, -1000, + 4, -1000, -1000, 35, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -13, -13, 28, 8, -1000, -1000, 27, -1000, -1000, + 49, 18, -1000, 5, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 20, -1000, -1000, } -var hclPgo = []int{ +var hclPgo = [...]int{ - 0, 3, 2, 59, 58, 36, 52, 49, 43, 1, - 0, 57, 7, 56, 55, + 0, 3, 2, 60, 59, 23, 54, 42, 11, 1, + 0, 58, 7, 57, 56, } -var hclR1 = []int{ +var hclR1 = [...]int{ 0, 14, 14, 5, 5, 8, 8, 13, 13, 9, 9, 9, 9, 9, 9, 6, 6, 11, 11, 3, - 3, 4, 4, 4, 10, 10, 7, 7, 7, 7, + 3, 3, 4, 4, 10, 10, 7, 7, 7, 7, 2, 2, 1, 1, 12, 12, } -var hclR2 = []int{ +var hclR2 = [...]int{ 0, 0, 1, 1, 2, 3, 2, 1, 1, 3, 3, 3, 3, 3, 1, 2, 2, 1, 1, 3, - 2, 1, 3, 2, 1, 1, 1, 1, 2, 2, + 4, 2, 1, 3, 1, 1, 1, 1, 2, 2, 2, 1, 2, 1, 2, 2, } -var hclChk = []int{ +var hclChk = [...]int{ - -1000, -14, -5, -9, -13, -6, 9, 12, -11, -9, - 10, -8, -6, 14, 9, 12, -7, 4, 12, -8, - -3, -2, -1, 16, 13, 6, 5, -5, 15, -12, - 19, 20, -12, -4, 17, -10, -7, 12, -2, -1, - 15, 6, 6, 17, 7, 8, -10, + -1000, -14, -5, -9, -13, -6, 8, 11, -11, -9, + 9, -8, -6, 13, 8, 11, -7, 4, 11, -8, + -3, -2, -1, 15, 12, 6, 5, -5, 14, -12, + 18, 19, -12, -4, 16, -10, -7, 11, -2, -1, + 14, 6, 6, 16, 7, 16, -10, } -var hclDef = []int{ +var hclDef = [...]int{ 1, -2, 2, 3, 0, 14, -2, -2, 0, 4, 0, 15, 16, 0, 17, 18, 9, 10, 11, 12, 13, 26, 27, 0, 0, 31, 33, 0, 6, 28, - 0, 0, 29, 0, 20, 21, 24, 25, 30, 32, - 5, 34, 35, 19, 0, 23, 22, + 0, 0, 29, 0, 21, 22, 24, 25, 30, 32, + 5, 34, 35, 19, 0, 20, 23, } -var hclTok1 = []int{ +var hclTok1 = [...]int{ 1, } -var hclTok2 = []int{ +var hclTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 16, 17, 18, 19, 20, + 12, 13, 14, 15, 16, 17, 18, 19, } -var hclTok3 = []int{ +var hclTok3 = [...]int{ 0, } +var hclErrorMessages = [...]struct { + state int + token int + msg string +}{} + //line yaccpar:1 /* parser for yacc output */ -var hclDebug = 0 +var ( + hclDebug = 0 + hclErrorVerbose = false +) type hclLexer interface { Lex(lval *hclSymType) int Error(s string) } +type hclParser interface { + Parse(hclLexer) int + Lookahead() int +} + +type hclParserImpl struct { + lookahead func() int +} + +func (p *hclParserImpl) Lookahead() int { + return p.lookahead() +} + +func hclNewParser() hclParser { + p := &hclParserImpl{ + lookahead: func() int { return -1 }, + } + return p +} + const hclFlag = -1000 func hclTokname(c int) string { - // 4 is TOKSTART above - if c >= 4 && c-4 < len(hclToknames) { - if hclToknames[c-4] != "" { - return hclToknames[c-4] + if c >= 1 && c-1 < len(hclToknames) { + if hclToknames[c-1] != "" { + return hclToknames[c-1] } } return __yyfmt__.Sprintf("tok-%v", c) @@ -184,51 +213,129 @@ func hclStatname(s int) string { return __yyfmt__.Sprintf("state-%v", s) } -func hcllex1(lex hclLexer, lval *hclSymType) int { - c := 0 - char := lex.Lex(lval) +func hclErrorMessage(state, lookAhead int) string { + const TOKSTART = 4 + + if !hclErrorVerbose { + return "syntax error" + } + + for _, e := range hclErrorMessages { + if e.state == state && e.token == lookAhead { + return "syntax error: " + e.msg + } + } + + res := "syntax error: unexpected " + hclTokname(lookAhead) + + // To match Bison, suggest at most four expected tokens. + expected := make([]int, 0, 4) + + // Look for shiftable tokens. + base := hclPact[state] + for tok := TOKSTART; tok-1 < len(hclToknames); tok++ { + if n := base + tok; n >= 0 && n < hclLast && hclChk[hclAct[n]] == tok { + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + } + + if hclDef[state] == -2 { + i := 0 + for hclExca[i] != -1 || hclExca[i+1] != state { + i += 2 + } + + // Look for tokens that we accept or reduce. + for i += 2; hclExca[i] >= 0; i += 2 { + tok := hclExca[i] + if tok < TOKSTART || hclExca[i+1] == 0 { + continue + } + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + + // If the default action is to accept or reduce, give up. + if hclExca[i+1] != 0 { + return res + } + } + + for i, tok := range expected { + if i == 0 { + res += ", expecting " + } else { + res += " or " + } + res += hclTokname(tok) + } + return res +} + +func hcllex1(lex hclLexer, lval *hclSymType) (char, token int) { + token = 0 + char = lex.Lex(lval) if char <= 0 { - c = hclTok1[0] + token = hclTok1[0] goto out } if char < len(hclTok1) { - c = hclTok1[char] + token = hclTok1[char] goto out } if char >= hclPrivate { if char < hclPrivate+len(hclTok2) { - c = hclTok2[char-hclPrivate] + token = hclTok2[char-hclPrivate] goto out } } for i := 0; i < len(hclTok3); i += 2 { - c = hclTok3[i+0] - if c == char { - c = hclTok3[i+1] + token = hclTok3[i+0] + if token == char { + token = hclTok3[i+1] goto out } } out: - if c == 0 { - c = hclTok2[1] /* unknown char */ + if token == 0 { + token = hclTok2[1] /* unknown char */ } if hclDebug >= 3 { - __yyfmt__.Printf("lex %s(%d)\n", hclTokname(c), uint(char)) + __yyfmt__.Printf("lex %s(%d)\n", hclTokname(token), uint(char)) } - return c + return char, token } func hclParse(hcllex hclLexer) int { + return hclNewParser().Parse(hcllex) +} + +func (hclrcvr *hclParserImpl) Parse(hcllex hclLexer) int { var hcln int var hcllval hclSymType var hclVAL hclSymType + var hclDollar []hclSymType + _ = hclDollar // silence set and not used hclS := make([]hclSymType, hclMaxDepth) Nerrs := 0 /* number of errors */ Errflag := 0 /* error recovery flag */ hclstate := 0 hclchar := -1 + hcltoken := -1 // hclchar translated into internal numbering + hclrcvr.lookahead = func() int { return hclchar } + defer func() { + // Make sure we report no lookahead when not parsing. + hclstate = -1 + hclchar = -1 + hcltoken = -1 + }() hclp := -1 goto hclstack @@ -241,7 +348,7 @@ ret1: hclstack: /* put a state and value onto the stack */ if hclDebug >= 4 { - __yyfmt__.Printf("char %v in %v\n", hclTokname(hclchar), hclStatname(hclstate)) + __yyfmt__.Printf("char %v in %v\n", hclTokname(hcltoken), hclStatname(hclstate)) } hclp++ @@ -259,15 +366,16 @@ hclnewstate: goto hcldefault /* simple state */ } if hclchar < 0 { - hclchar = hcllex1(hcllex, &hcllval) + hclchar, hcltoken = hcllex1(hcllex, &hcllval) } - hcln += hclchar + hcln += hcltoken if hcln < 0 || hcln >= hclLast { goto hcldefault } hcln = hclAct[hcln] - if hclChk[hcln] == hclchar { /* valid shift */ + if hclChk[hcln] == hcltoken { /* valid shift */ hclchar = -1 + hcltoken = -1 hclVAL = hcllval hclstate = hcln if Errflag > 0 { @@ -281,7 +389,7 @@ hcldefault: hcln = hclDef[hclstate] if hcln == -2 { if hclchar < 0 { - hclchar = hcllex1(hcllex, &hcllval) + hclchar, hcltoken = hcllex1(hcllex, &hcllval) } /* look through exception table */ @@ -294,7 +402,7 @@ hcldefault: } for xi += 2; ; xi += 2 { hcln = hclExca[xi+0] - if hcln < 0 || hcln == hclchar { + if hcln < 0 || hcln == hcltoken { break } } @@ -307,11 +415,11 @@ hcldefault: /* error ... attempt to resume parsing */ switch Errflag { case 0: /* brand new error */ - hcllex.Error("syntax error") + hcllex.Error(hclErrorMessage(hclstate, hcltoken)) Nerrs++ if hclDebug >= 1 { __yyfmt__.Printf("%s", hclStatname(hclstate)) - __yyfmt__.Printf(" saw %s\n", hclTokname(hclchar)) + __yyfmt__.Printf(" saw %s\n", hclTokname(hcltoken)) } fallthrough @@ -339,12 +447,13 @@ hcldefault: case 3: /* no shift yet; clobber input char */ if hclDebug >= 2 { - __yyfmt__.Printf("error recovery discards %s\n", hclTokname(hclchar)) + __yyfmt__.Printf("error recovery discards %s\n", hclTokname(hcltoken)) } - if hclchar == hclEofCode { + if hcltoken == hclEofCode { goto ret1 } hclchar = -1 + hcltoken = -1 goto hclnewstate /* try again in the same state */ } } @@ -359,6 +468,13 @@ hcldefault: _ = hclpt // guard against "declared and not used" hclp -= hclR2[hcln] + // hclp is now the index of $0. Perform the default action. Iff the + // reduced production is ε, $1 is possibly out of range. + if hclp+1 >= len(hclS) { + nyys := make([]hclSymType, len(hclS)*2) + copy(nyys, hclS) + hclS = nyys + } hclVAL = hclS[hclp+1] /* consult goto table to find next state */ @@ -378,37 +494,43 @@ hcldefault: switch hclnt { case 1: + hclDollar = hclS[hclpt-0 : hclpt+1] //line parse.y:39 { hclResult = &Object{Type: ValueTypeObject} } case 2: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:43 { hclResult = &Object{ Type: ValueTypeObject, - Value: ObjectList(hclS[hclpt-0].objlist).Flat(), + Value: ObjectList(hclDollar[1].objlist).Flat(), } } case 3: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:52 { - hclVAL.objlist = []*Object{hclS[hclpt-0].obj} + hclVAL.objlist = []*Object{hclDollar[1].obj} } case 4: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:56 { - hclVAL.objlist = append(hclS[hclpt-1].objlist, hclS[hclpt-0].obj) + hclVAL.objlist = append(hclDollar[1].objlist, hclDollar[2].obj) } case 5: + hclDollar = hclS[hclpt-3 : hclpt+1] //line parse.y:62 { hclVAL.obj = &Object{ Type: ValueTypeObject, - Value: ObjectList(hclS[hclpt-1].objlist).Flat(), + Value: ObjectList(hclDollar[2].objlist).Flat(), } } case 6: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:69 { hclVAL.obj = &Object{ @@ -416,142 +538,164 @@ hcldefault: } } case 7: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:77 { - hclVAL.str = hclS[hclpt-0].str + hclVAL.str = hclDollar[1].str } case 8: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:81 { - hclVAL.str = hclS[hclpt-0].str + hclVAL.str = hclDollar[1].str } case 9: + hclDollar = hclS[hclpt-3 : hclpt+1] //line parse.y:87 { - hclVAL.obj = hclS[hclpt-0].obj - hclVAL.obj.Key = hclS[hclpt-2].str + hclVAL.obj = hclDollar[3].obj + hclVAL.obj.Key = hclDollar[1].str } case 10: + hclDollar = hclS[hclpt-3 : hclpt+1] //line parse.y:92 { hclVAL.obj = &Object{ - Key: hclS[hclpt-2].str, + Key: hclDollar[1].str, Type: ValueTypeBool, - Value: hclS[hclpt-0].b, + Value: hclDollar[3].b, } } case 11: + hclDollar = hclS[hclpt-3 : hclpt+1] //line parse.y:100 { hclVAL.obj = &Object{ - Key: hclS[hclpt-2].str, + Key: hclDollar[1].str, Type: ValueTypeString, - Value: hclS[hclpt-0].str, + Value: hclDollar[3].str, } } case 12: + hclDollar = hclS[hclpt-3 : hclpt+1] //line parse.y:108 { - hclS[hclpt-0].obj.Key = hclS[hclpt-2].str - hclVAL.obj = hclS[hclpt-0].obj + hclDollar[3].obj.Key = hclDollar[1].str + hclVAL.obj = hclDollar[3].obj } case 13: + hclDollar = hclS[hclpt-3 : hclpt+1] //line parse.y:113 { hclVAL.obj = &Object{ - Key: hclS[hclpt-2].str, + Key: hclDollar[1].str, Type: ValueTypeList, - Value: hclS[hclpt-0].objlist, + Value: hclDollar[3].objlist, } } case 14: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:121 { - hclVAL.obj = hclS[hclpt-0].obj + hclVAL.obj = hclDollar[1].obj } case 15: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:127 { - hclS[hclpt-0].obj.Key = hclS[hclpt-1].str - hclVAL.obj = hclS[hclpt-0].obj + hclDollar[2].obj.Key = hclDollar[1].str + hclVAL.obj = hclDollar[2].obj } case 16: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:132 { hclVAL.obj = &Object{ - Key: hclS[hclpt-1].str, + Key: hclDollar[1].str, Type: ValueTypeObject, - Value: []*Object{hclS[hclpt-0].obj}, + Value: []*Object{hclDollar[2].obj}, } } case 17: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:142 { - hclVAL.str = hclS[hclpt-0].str + hclVAL.str = hclDollar[1].str } case 18: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:146 { - hclVAL.str = hclS[hclpt-0].str + hclVAL.str = hclDollar[1].str } case 19: + hclDollar = hclS[hclpt-3 : hclpt+1] //line parse.y:152 { - hclVAL.objlist = hclS[hclpt-1].objlist + hclVAL.objlist = hclDollar[2].objlist } case 20: + hclDollar = hclS[hclpt-4 : hclpt+1] //line parse.y:156 + { + hclVAL.objlist = hclDollar[2].objlist + } + case 21: + hclDollar = hclS[hclpt-2 : hclpt+1] + //line parse.y:160 { hclVAL.objlist = nil } - case 21: - //line parse.y:162 - { - hclVAL.objlist = []*Object{hclS[hclpt-0].obj} - } case 22: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:166 { - hclVAL.objlist = append(hclS[hclpt-2].objlist, hclS[hclpt-0].obj) + hclVAL.objlist = []*Object{hclDollar[1].obj} } case 23: + hclDollar = hclS[hclpt-3 : hclpt+1] //line parse.y:170 { - hclVAL.objlist = hclS[hclpt-1].objlist + hclVAL.objlist = append(hclDollar[1].objlist, hclDollar[3].obj) } case 24: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:176 { - hclVAL.obj = hclS[hclpt-0].obj + hclVAL.obj = hclDollar[1].obj } case 25: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:180 { hclVAL.obj = &Object{ Type: ValueTypeString, - Value: hclS[hclpt-0].str, + Value: hclDollar[1].str, } } case 26: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:189 { hclVAL.obj = &Object{ Type: ValueTypeInt, - Value: hclS[hclpt-0].num, + Value: hclDollar[1].num, } } case 27: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:196 { hclVAL.obj = &Object{ Type: ValueTypeFloat, - Value: hclS[hclpt-0].f, + Value: hclDollar[1].f, } } case 28: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:203 { - fs := fmt.Sprintf("%d%s", hclS[hclpt-1].num, hclS[hclpt-0].str) + fs := fmt.Sprintf("%d%s", hclDollar[1].num, hclDollar[2].str) f, err := strconv.ParseFloat(fs, 64) if err != nil { panic(err) @@ -563,9 +707,10 @@ hcldefault: } } case 29: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:216 { - fs := fmt.Sprintf("%f%s", hclS[hclpt-1].f, hclS[hclpt-0].str) + fs := fmt.Sprintf("%f%s", hclDollar[1].f, hclDollar[2].str) f, err := strconv.ParseFloat(fs, 64) if err != nil { panic(err) @@ -577,34 +722,40 @@ hcldefault: } } case 30: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:231 { - hclVAL.num = hclS[hclpt-0].num * -1 + hclVAL.num = hclDollar[2].num * -1 } case 31: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:235 { - hclVAL.num = hclS[hclpt-0].num + hclVAL.num = hclDollar[1].num } case 32: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:241 { - hclVAL.f = hclS[hclpt-0].f * -1 + hclVAL.f = hclDollar[2].f * -1 } case 33: + hclDollar = hclS[hclpt-1 : hclpt+1] //line parse.y:245 { - hclVAL.f = hclS[hclpt-0].f + hclVAL.f = hclDollar[1].f } case 34: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:251 { - hclVAL.str = "e" + strconv.FormatInt(int64(hclS[hclpt-0].num), 10) + hclVAL.str = "e" + strconv.FormatInt(int64(hclDollar[2].num), 10) } case 35: + hclDollar = hclS[hclpt-2 : hclpt+1] //line parse.y:255 { - hclVAL.str = "e-" + strconv.FormatInt(int64(hclS[hclpt-0].num), 10) + hclVAL.str = "e-" + strconv.FormatInt(int64(hclDollar[2].num), 10) } } goto hclstack /* stack new state and value */ diff --git a/json/y.go b/json/y.go index 075270a..a57649e 100644 --- a/json/y.go +++ b/json/y.go @@ -41,7 +41,10 @@ const PERIOD = 57362 const EPLUS = 57363 const EMINUS = 57364 -var jsonToknames = []string{ +var jsonToknames = [...]string{ + "$end", + "error", + "$unk", "FLOAT", "NUMBER", "COLON", @@ -62,7 +65,7 @@ var jsonToknames = []string{ "EPLUS", "EMINUS", } -var jsonStatenames = []string{} +var jsonStatenames = [...]string{} const jsonEofCode = 1 const jsonErrCode = 2 @@ -71,7 +74,7 @@ const jsonMaxDepth = 200 //line parse.y:210 //line yacctab:1 -var jsonExca = []int{ +var jsonExca = [...]int{ -1, 1, 1, -1, -2, 0, @@ -85,7 +88,7 @@ var jsonStates []string const jsonLast = 53 -var jsonAct = []int{ +var jsonAct = [...]int{ 12, 25, 24, 3, 20, 27, 28, 7, 13, 3, 21, 22, 30, 17, 18, 19, 23, 25, 24, 26, @@ -94,76 +97,104 @@ var jsonAct = []int{ 5, 29, 6, 8, 37, 15, 2, 1, 4, 31, 16, 14, 11, } -var jsonPact = []int{ +var jsonPact = [...]int{ -9, -1000, -1000, 27, 30, -1000, -1000, 20, -1000, -4, 13, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -16, -16, -3, 16, -1000, -1000, -1000, 28, 17, -1000, -1000, 29, -1000, -1000, -1000, -1000, -1000, -1000, 13, -1000, } -var jsonPgo = []int{ +var jsonPgo = [...]int{ 0, 10, 4, 51, 45, 42, 0, 50, 49, 48, 19, 47, } -var jsonR1 = []int{ +var jsonR1 = [...]int{ 0, 11, 4, 4, 9, 9, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 8, 8, 3, 3, 3, 3, 2, 2, 1, 1, 10, 10, } -var jsonR2 = []int{ +var jsonR2 = [...]int{ 0, 1, 3, 2, 1, 3, 3, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 3, 1, 1, 2, 2, 2, 1, 2, 1, 2, 2, } -var jsonChk = []int{ +var jsonChk = [...]int{ -1000, -11, -4, 12, -9, 13, -5, 11, 13, 7, 6, -5, -6, 11, -3, -4, -7, 16, 17, 18, -2, -1, 14, 19, 5, 4, -10, 21, 22, -10, 15, -8, -6, -2, -1, 5, 5, 15, 7, -6, } -var jsonDef = []int{ +var jsonDef = [...]int{ 0, -2, 1, 0, 0, 3, 4, 0, 2, 0, 0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 18, 19, 0, 0, 23, 25, 20, 0, 0, 21, 14, 0, 16, 22, 24, 26, 27, 15, 0, 17, } -var jsonTok1 = []int{ +var jsonTok1 = [...]int{ 1, } -var jsonTok2 = []int{ +var jsonTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, } -var jsonTok3 = []int{ +var jsonTok3 = [...]int{ 0, } +var jsonErrorMessages = [...]struct { + state int + token int + msg string +}{} + //line yaccpar:1 /* parser for yacc output */ -var jsonDebug = 0 +var ( + jsonDebug = 0 + jsonErrorVerbose = false +) type jsonLexer interface { Lex(lval *jsonSymType) int Error(s string) } +type jsonParser interface { + Parse(jsonLexer) int + Lookahead() int +} + +type jsonParserImpl struct { + lookahead func() int +} + +func (p *jsonParserImpl) Lookahead() int { + return p.lookahead() +} + +func jsonNewParser() jsonParser { + p := &jsonParserImpl{ + lookahead: func() int { return -1 }, + } + return p +} + const jsonFlag = -1000 func jsonTokname(c int) string { - // 4 is TOKSTART above - if c >= 4 && c-4 < len(jsonToknames) { - if jsonToknames[c-4] != "" { - return jsonToknames[c-4] + if c >= 1 && c-1 < len(jsonToknames) { + if jsonToknames[c-1] != "" { + return jsonToknames[c-1] } } return __yyfmt__.Sprintf("tok-%v", c) @@ -178,51 +209,129 @@ func jsonStatname(s int) string { return __yyfmt__.Sprintf("state-%v", s) } -func jsonlex1(lex jsonLexer, lval *jsonSymType) int { - c := 0 - char := lex.Lex(lval) +func jsonErrorMessage(state, lookAhead int) string { + const TOKSTART = 4 + + if !jsonErrorVerbose { + return "syntax error" + } + + for _, e := range jsonErrorMessages { + if e.state == state && e.token == lookAhead { + return "syntax error: " + e.msg + } + } + + res := "syntax error: unexpected " + jsonTokname(lookAhead) + + // To match Bison, suggest at most four expected tokens. + expected := make([]int, 0, 4) + + // Look for shiftable tokens. + base := jsonPact[state] + for tok := TOKSTART; tok-1 < len(jsonToknames); tok++ { + if n := base + tok; n >= 0 && n < jsonLast && jsonChk[jsonAct[n]] == tok { + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + } + + if jsonDef[state] == -2 { + i := 0 + for jsonExca[i] != -1 || jsonExca[i+1] != state { + i += 2 + } + + // Look for tokens that we accept or reduce. + for i += 2; jsonExca[i] >= 0; i += 2 { + tok := jsonExca[i] + if tok < TOKSTART || jsonExca[i+1] == 0 { + continue + } + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + + // If the default action is to accept or reduce, give up. + if jsonExca[i+1] != 0 { + return res + } + } + + for i, tok := range expected { + if i == 0 { + res += ", expecting " + } else { + res += " or " + } + res += jsonTokname(tok) + } + return res +} + +func jsonlex1(lex jsonLexer, lval *jsonSymType) (char, token int) { + token = 0 + char = lex.Lex(lval) if char <= 0 { - c = jsonTok1[0] + token = jsonTok1[0] goto out } if char < len(jsonTok1) { - c = jsonTok1[char] + token = jsonTok1[char] goto out } if char >= jsonPrivate { if char < jsonPrivate+len(jsonTok2) { - c = jsonTok2[char-jsonPrivate] + token = jsonTok2[char-jsonPrivate] goto out } } for i := 0; i < len(jsonTok3); i += 2 { - c = jsonTok3[i+0] - if c == char { - c = jsonTok3[i+1] + token = jsonTok3[i+0] + if token == char { + token = jsonTok3[i+1] goto out } } out: - if c == 0 { - c = jsonTok2[1] /* unknown char */ + if token == 0 { + token = jsonTok2[1] /* unknown char */ } if jsonDebug >= 3 { - __yyfmt__.Printf("lex %s(%d)\n", jsonTokname(c), uint(char)) + __yyfmt__.Printf("lex %s(%d)\n", jsonTokname(token), uint(char)) } - return c + return char, token } func jsonParse(jsonlex jsonLexer) int { + return jsonNewParser().Parse(jsonlex) +} + +func (jsonrcvr *jsonParserImpl) Parse(jsonlex jsonLexer) int { var jsonn int var jsonlval jsonSymType var jsonVAL jsonSymType + var jsonDollar []jsonSymType + _ = jsonDollar // silence set and not used jsonS := make([]jsonSymType, jsonMaxDepth) Nerrs := 0 /* number of errors */ Errflag := 0 /* error recovery flag */ jsonstate := 0 jsonchar := -1 + jsontoken := -1 // jsonchar translated into internal numbering + jsonrcvr.lookahead = func() int { return jsonchar } + defer func() { + // Make sure we report no lookahead when not parsing. + jsonstate = -1 + jsonchar = -1 + jsontoken = -1 + }() jsonp := -1 goto jsonstack @@ -235,7 +344,7 @@ ret1: jsonstack: /* put a state and value onto the stack */ if jsonDebug >= 4 { - __yyfmt__.Printf("char %v in %v\n", jsonTokname(jsonchar), jsonStatname(jsonstate)) + __yyfmt__.Printf("char %v in %v\n", jsonTokname(jsontoken), jsonStatname(jsonstate)) } jsonp++ @@ -253,15 +362,16 @@ jsonnewstate: goto jsondefault /* simple state */ } if jsonchar < 0 { - jsonchar = jsonlex1(jsonlex, &jsonlval) + jsonchar, jsontoken = jsonlex1(jsonlex, &jsonlval) } - jsonn += jsonchar + jsonn += jsontoken if jsonn < 0 || jsonn >= jsonLast { goto jsondefault } jsonn = jsonAct[jsonn] - if jsonChk[jsonn] == jsonchar { /* valid shift */ + if jsonChk[jsonn] == jsontoken { /* valid shift */ jsonchar = -1 + jsontoken = -1 jsonVAL = jsonlval jsonstate = jsonn if Errflag > 0 { @@ -275,7 +385,7 @@ jsondefault: jsonn = jsonDef[jsonstate] if jsonn == -2 { if jsonchar < 0 { - jsonchar = jsonlex1(jsonlex, &jsonlval) + jsonchar, jsontoken = jsonlex1(jsonlex, &jsonlval) } /* look through exception table */ @@ -288,7 +398,7 @@ jsondefault: } for xi += 2; ; xi += 2 { jsonn = jsonExca[xi+0] - if jsonn < 0 || jsonn == jsonchar { + if jsonn < 0 || jsonn == jsontoken { break } } @@ -301,11 +411,11 @@ jsondefault: /* error ... attempt to resume parsing */ switch Errflag { case 0: /* brand new error */ - jsonlex.Error("syntax error") + jsonlex.Error(jsonErrorMessage(jsonstate, jsontoken)) Nerrs++ if jsonDebug >= 1 { __yyfmt__.Printf("%s", jsonStatname(jsonstate)) - __yyfmt__.Printf(" saw %s\n", jsonTokname(jsonchar)) + __yyfmt__.Printf(" saw %s\n", jsonTokname(jsontoken)) } fallthrough @@ -333,12 +443,13 @@ jsondefault: case 3: /* no shift yet; clobber input char */ if jsonDebug >= 2 { - __yyfmt__.Printf("error recovery discards %s\n", jsonTokname(jsonchar)) + __yyfmt__.Printf("error recovery discards %s\n", jsonTokname(jsontoken)) } - if jsonchar == jsonEofCode { + if jsontoken == jsonEofCode { goto ret1 } jsonchar = -1 + jsontoken = -1 goto jsonnewstate /* try again in the same state */ } } @@ -353,6 +464,13 @@ jsondefault: _ = jsonpt // guard against "declared and not used" jsonp -= jsonR2[jsonn] + // jsonp is now the index of $0. Perform the default action. Iff the + // reduced production is ε, $1 is possibly out of range. + if jsonp+1 >= len(jsonS) { + nyys := make([]jsonSymType, len(jsonS)*2) + copy(nyys, jsonS) + jsonS = nyys + } jsonVAL = jsonS[jsonp+1] /* consult goto table to find next state */ @@ -372,66 +490,77 @@ jsondefault: switch jsonnt { case 1: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:39 { - jsonResult = jsonS[jsonpt-0].obj + jsonResult = jsonDollar[1].obj } case 2: + jsonDollar = jsonS[jsonpt-3 : jsonpt+1] //line parse.y:45 { jsonVAL.obj = &hcl.Object{ Type: hcl.ValueTypeObject, - Value: hcl.ObjectList(jsonS[jsonpt-1].objlist).Flat(), + Value: hcl.ObjectList(jsonDollar[2].objlist).Flat(), } } case 3: + jsonDollar = jsonS[jsonpt-2 : jsonpt+1] //line parse.y:52 { jsonVAL.obj = &hcl.Object{Type: hcl.ValueTypeObject} } case 4: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:58 { - jsonVAL.objlist = []*hcl.Object{jsonS[jsonpt-0].obj} + jsonVAL.objlist = []*hcl.Object{jsonDollar[1].obj} } case 5: + jsonDollar = jsonS[jsonpt-3 : jsonpt+1] //line parse.y:62 { - jsonVAL.objlist = append(jsonS[jsonpt-2].objlist, jsonS[jsonpt-0].obj) + jsonVAL.objlist = append(jsonDollar[1].objlist, jsonDollar[3].obj) } case 6: + jsonDollar = jsonS[jsonpt-3 : jsonpt+1] //line parse.y:68 { - jsonS[jsonpt-0].obj.Key = jsonS[jsonpt-2].str - jsonVAL.obj = jsonS[jsonpt-0].obj + jsonDollar[3].obj.Key = jsonDollar[1].str + jsonVAL.obj = jsonDollar[3].obj } case 7: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:75 { jsonVAL.obj = &hcl.Object{ Type: hcl.ValueTypeString, - Value: jsonS[jsonpt-0].str, + Value: jsonDollar[1].str, } } case 8: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:82 { - jsonVAL.obj = jsonS[jsonpt-0].obj + jsonVAL.obj = jsonDollar[1].obj } case 9: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:86 { - jsonVAL.obj = jsonS[jsonpt-0].obj + jsonVAL.obj = jsonDollar[1].obj } case 10: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:90 { jsonVAL.obj = &hcl.Object{ Type: hcl.ValueTypeList, - Value: jsonS[jsonpt-0].objlist, + Value: jsonDollar[1].objlist, } } case 11: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:97 { jsonVAL.obj = &hcl.Object{ @@ -440,6 +569,7 @@ jsondefault: } } case 12: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:104 { jsonVAL.obj = &hcl.Object{ @@ -448,6 +578,7 @@ jsondefault: } } case 13: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:111 { jsonVAL.obj = &hcl.Object{ @@ -456,45 +587,52 @@ jsondefault: } } case 14: + jsonDollar = jsonS[jsonpt-2 : jsonpt+1] //line parse.y:120 { jsonVAL.objlist = nil } case 15: + jsonDollar = jsonS[jsonpt-3 : jsonpt+1] //line parse.y:124 { - jsonVAL.objlist = jsonS[jsonpt-1].objlist + jsonVAL.objlist = jsonDollar[2].objlist } case 16: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:130 { - jsonVAL.objlist = []*hcl.Object{jsonS[jsonpt-0].obj} + jsonVAL.objlist = []*hcl.Object{jsonDollar[1].obj} } case 17: + jsonDollar = jsonS[jsonpt-3 : jsonpt+1] //line parse.y:134 { - jsonVAL.objlist = append(jsonS[jsonpt-2].objlist, jsonS[jsonpt-0].obj) + jsonVAL.objlist = append(jsonDollar[1].objlist, jsonDollar[3].obj) } case 18: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:140 { jsonVAL.obj = &hcl.Object{ Type: hcl.ValueTypeInt, - Value: jsonS[jsonpt-0].num, + Value: jsonDollar[1].num, } } case 19: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:147 { jsonVAL.obj = &hcl.Object{ Type: hcl.ValueTypeFloat, - Value: jsonS[jsonpt-0].f, + Value: jsonDollar[1].f, } } case 20: + jsonDollar = jsonS[jsonpt-2 : jsonpt+1] //line parse.y:154 { - fs := fmt.Sprintf("%d%s", jsonS[jsonpt-1].num, jsonS[jsonpt-0].str) + fs := fmt.Sprintf("%d%s", jsonDollar[1].num, jsonDollar[2].str) f, err := strconv.ParseFloat(fs, 64) if err != nil { panic(err) @@ -506,9 +644,10 @@ jsondefault: } } case 21: + jsonDollar = jsonS[jsonpt-2 : jsonpt+1] //line parse.y:167 { - fs := fmt.Sprintf("%f%s", jsonS[jsonpt-1].f, jsonS[jsonpt-0].str) + fs := fmt.Sprintf("%f%s", jsonDollar[1].f, jsonDollar[2].str) f, err := strconv.ParseFloat(fs, 64) if err != nil { panic(err) @@ -520,34 +659,40 @@ jsondefault: } } case 22: + jsonDollar = jsonS[jsonpt-2 : jsonpt+1] //line parse.y:182 { - jsonVAL.num = jsonS[jsonpt-0].num * -1 + jsonVAL.num = jsonDollar[2].num * -1 } case 23: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:186 { - jsonVAL.num = jsonS[jsonpt-0].num + jsonVAL.num = jsonDollar[1].num } case 24: + jsonDollar = jsonS[jsonpt-2 : jsonpt+1] //line parse.y:192 { - jsonVAL.f = jsonS[jsonpt-0].f * -1 + jsonVAL.f = jsonDollar[2].f * -1 } case 25: + jsonDollar = jsonS[jsonpt-1 : jsonpt+1] //line parse.y:196 { - jsonVAL.f = jsonS[jsonpt-0].f + jsonVAL.f = jsonDollar[1].f } case 26: + jsonDollar = jsonS[jsonpt-2 : jsonpt+1] //line parse.y:202 { - jsonVAL.str = "e" + strconv.FormatInt(int64(jsonS[jsonpt-0].num), 10) + jsonVAL.str = "e" + strconv.FormatInt(int64(jsonDollar[2].num), 10) } case 27: + jsonDollar = jsonS[jsonpt-2 : jsonpt+1] //line parse.y:206 { - jsonVAL.str = "e-" + strconv.FormatInt(int64(jsonS[jsonpt-0].num), 10) + jsonVAL.str = "e-" + strconv.FormatInt(int64(jsonDollar[2].num), 10) } } goto jsonstack /* stack new state and value */