go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zclsyntax: differentiate quoted and unquoted string literals

Browse Source 
The context where a string literal was found affects what sort of escaping
it can have, so we need to distinguish these cases so that we will only
look for and handle backslash escapes in quoted strings.
This commit is contained in:
Martin Atkins 2017-05-30 19:03:25 -07:00
parent 57d8ff0c1b
commit f94c89a6a5
7 changed files with 127 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
zcl/zclsyntax/parser.go
View File

@ -276,7 +276,7 @@ Token:
cQuote = tok
break Token
case TokenStringLit:
case TokenQuotedLit:
// TODO: Remove any escape sequences from the string, once we
// have a function with which to do that.
ret.Write(tok.Bytes)

140
zcl/zclsyntax/scan_tokens.go
View File

@ -2752,7 +2752,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
Pos: start,
}
// line 254 "scan_tokens.rl"
// line 257 "scan_tokens.rl"
// Ragel state
p := 0 // "Pointer" into data
@ -2778,7 +2778,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
var retBraces []int // stack of brace levels that cause us to use fret
var heredocs []heredocInProgress // stack of heredocs we're currently processing
// line 287 "scan_tokens.rl"
// line 290 "scan_tokens.rl"
// Make Go compiler happy
_ = ts
@ -2908,7 +2908,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
te = p + 1
case 4:
// line 130 "scan_tokens.rl"
// line 133 "scan_tokens.rl"
te = p + 1
{
@ -2927,7 +2927,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 5:
// line 140 "scan_tokens.rl"
// line 143 "scan_tokens.rl"
te = p + 1
{
@ -2946,7 +2946,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 6:
// line 72 "scan_tokens.rl"
// line 75 "scan_tokens.rl"
te = p + 1
{
@ -2960,21 +2960,21 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
case 7:
// line 207 "scan_tokens.rl"
// line 210 "scan_tokens.rl"
te = p + 1
{
token(TokenInvalid)
}
case 8:
// line 208 "scan_tokens.rl"
// line 211 "scan_tokens.rl"
te = p + 1
{
token(TokenBadUTF8)
}
case 9:
// line 130 "scan_tokens.rl"
// line 133 "scan_tokens.rl"
te = p
p--
@ -2994,7 +2994,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 10:
// line 140 "scan_tokens.rl"
// line 143 "scan_tokens.rl"
te = p
p--
@ -3014,15 +3014,15 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 11:
// line 206 "scan_tokens.rl"
// line 209 "scan_tokens.rl"
te = p
p--
{
token(TokenStringLit)
token(TokenQuotedLit)
}
case 12:
// line 207 "scan_tokens.rl"
// line 210 "scan_tokens.rl"
te = p
p--
@ -3030,7 +3030,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenInvalid)
}
case 13:
// line 208 "scan_tokens.rl"
// line 211 "scan_tokens.rl"
te = p
p--
@ -3038,29 +3038,29 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenBadUTF8)
}
case 14:
// line 206 "scan_tokens.rl"
// line 209 "scan_tokens.rl"
p = (te) - 1
{
token(TokenStringLit)
token(TokenQuotedLit)
}
case 15:
// line 208 "scan_tokens.rl"
// line 211 "scan_tokens.rl"
p = (te) - 1
{
token(TokenBadUTF8)
}
case 16:
// line 118 "scan_tokens.rl"
// line 121 "scan_tokens.rl"
act = 10
case 17:
// line 216 "scan_tokens.rl"
// line 219 "scan_tokens.rl"
act = 11
case 18:
// line 130 "scan_tokens.rl"
// line 133 "scan_tokens.rl"
te = p + 1
{
@ -3079,7 +3079,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 19:
// line 140 "scan_tokens.rl"
// line 143 "scan_tokens.rl"
te = p + 1
{
@ -3098,7 +3098,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 20:
// line 99 "scan_tokens.rl"
// line 102 "scan_tokens.rl"
te = p + 1
{
@ -3126,14 +3126,14 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenStringLit)
}
case 21:
// line 216 "scan_tokens.rl"
// line 219 "scan_tokens.rl"
te = p + 1
{
token(TokenBadUTF8)
}
case 22:
// line 130 "scan_tokens.rl"
// line 133 "scan_tokens.rl"
te = p
p--
@ -3153,7 +3153,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 23:
// line 140 "scan_tokens.rl"
// line 143 "scan_tokens.rl"
te = p
p--
@ -3173,7 +3173,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 24:
// line 118 "scan_tokens.rl"
// line 121 "scan_tokens.rl"
te = p
p--
@ -3185,7 +3185,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenStringLit)
}
case 25:
// line 216 "scan_tokens.rl"
// line 219 "scan_tokens.rl"
te = p
p--
@ -3193,7 +3193,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenBadUTF8)
}
case 26:
// line 118 "scan_tokens.rl"
// line 121 "scan_tokens.rl"
p = (te) - 1
{
@ -3230,15 +3230,15 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
case 28:
// line 126 "scan_tokens.rl"
// line 129 "scan_tokens.rl"
act = 14
case 29:
// line 223 "scan_tokens.rl"
// line 226 "scan_tokens.rl"
act = 15
case 30:
// line 130 "scan_tokens.rl"
// line 133 "scan_tokens.rl"
te = p + 1
{
@ -3257,7 +3257,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 31:
// line 140 "scan_tokens.rl"
// line 143 "scan_tokens.rl"
te = p + 1
{
@ -3276,21 +3276,21 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 32:
// line 126 "scan_tokens.rl"
// line 129 "scan_tokens.rl"
te = p + 1
{
token(TokenStringLit)
}
case 33:
// line 223 "scan_tokens.rl"
// line 226 "scan_tokens.rl"
te = p + 1
{
token(TokenBadUTF8)
}
case 34:
// line 130 "scan_tokens.rl"
// line 133 "scan_tokens.rl"
te = p
p--
@ -3310,7 +3310,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 35:
// line 140 "scan_tokens.rl"
// line 143 "scan_tokens.rl"
te = p
p--
@ -3330,7 +3330,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 36:
// line 126 "scan_tokens.rl"
// line 129 "scan_tokens.rl"
te = p
p--
@ -3338,7 +3338,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenStringLit)
}
case 37:
// line 223 "scan_tokens.rl"
// line 226 "scan_tokens.rl"
te = p
p--
@ -3346,7 +3346,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenBadUTF8)
}
case 38:
// line 126 "scan_tokens.rl"
// line 129 "scan_tokens.rl"
p = (te) - 1
{
@ -3375,76 +3375,76 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
case 40:
// line 229 "scan_tokens.rl"
// line 232 "scan_tokens.rl"
act = 18
case 41:
// line 231 "scan_tokens.rl"
// line 234 "scan_tokens.rl"
act = 19
case 42:
// line 239 "scan_tokens.rl"
// line 242 "scan_tokens.rl"
act = 26
case 43:
// line 250 "scan_tokens.rl"
// line 253 "scan_tokens.rl"
act = 33
case 44:
// line 251 "scan_tokens.rl"
// line 254 "scan_tokens.rl"
act = 34
case 45:
// line 232 "scan_tokens.rl"
// line 235 "scan_tokens.rl"
te = p + 1
{
token(TokenNewline)
}
case 46:
// line 234 "scan_tokens.rl"
// line 237 "scan_tokens.rl"
te = p + 1
{
token(TokenNotEqual)
}
case 47:
// line 235 "scan_tokens.rl"
// line 238 "scan_tokens.rl"
te = p + 1
{
token(TokenGreaterThanEq)
}
case 48:
// line 236 "scan_tokens.rl"
// line 239 "scan_tokens.rl"
te = p + 1
{
token(TokenLessThanEq)
}
case 49:
// line 237 "scan_tokens.rl"
// line 240 "scan_tokens.rl"
te = p + 1
{
token(TokenAnd)
}
case 50:
// line 238 "scan_tokens.rl"
// line 241 "scan_tokens.rl"
te = p + 1
{
token(TokenOr)
}
case 51:
// line 239 "scan_tokens.rl"
// line 242 "scan_tokens.rl"
te = p + 1
{
selfToken()
}
case 52:
// line 150 "scan_tokens.rl"
// line 153 "scan_tokens.rl"
te = p + 1
{
@ -3452,7 +3452,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
braces++
}
case 53:
// line 155 "scan_tokens.rl"
// line 158 "scan_tokens.rl"
te = p + 1
{
@ -3473,7 +3473,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 54:
// line 167 "scan_tokens.rl"
// line 170 "scan_tokens.rl"
te = p + 1
{
@ -3496,7 +3496,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 55:
// line 67 "scan_tokens.rl"
// line 70 "scan_tokens.rl"
te = p + 1
{
@ -3510,7 +3510,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 56:
// line 77 "scan_tokens.rl"
// line 80 "scan_tokens.rl"
te = p + 1
{
@ -3541,27 +3541,27 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 57:
// line 250 "scan_tokens.rl"
// line 253 "scan_tokens.rl"
te = p + 1
{
token(TokenInvalid)
}
case 58:
// line 251 "scan_tokens.rl"
// line 254 "scan_tokens.rl"
te = p + 1
{
token(TokenBadUTF8)
}
case 59:
// line 227 "scan_tokens.rl"
// line 230 "scan_tokens.rl"
te = p
p--
case 60:
// line 228 "scan_tokens.rl"
// line 231 "scan_tokens.rl"
te = p
p--
@ -3569,7 +3569,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenNumberLit)
}
case 61:
// line 229 "scan_tokens.rl"
// line 232 "scan_tokens.rl"
te = p
p--
@ -3577,7 +3577,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenIdent)
}
case 62:
// line 231 "scan_tokens.rl"
// line 234 "scan_tokens.rl"
te = p
p--
@ -3585,7 +3585,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenComment)
}
case 63:
// line 239 "scan_tokens.rl"
// line 242 "scan_tokens.rl"
te = p
p--
@ -3593,7 +3593,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
selfToken()
}
case 64:
// line 249 "scan_tokens.rl"
// line 252 "scan_tokens.rl"
te = p
p--
@ -3601,7 +3601,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenTabs)
}
case 65:
// line 250 "scan_tokens.rl"
// line 253 "scan_tokens.rl"
te = p
p--
@ -3609,7 +3609,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenInvalid)
}
case 66:
// line 251 "scan_tokens.rl"
// line 254 "scan_tokens.rl"
te = p
p--
@ -3617,28 +3617,28 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenBadUTF8)
}
case 67:
// line 228 "scan_tokens.rl"
// line 231 "scan_tokens.rl"
p = (te) - 1
{
token(TokenNumberLit)
}
case 68:
// line 229 "scan_tokens.rl"
// line 232 "scan_tokens.rl"
p = (te) - 1
{
token(TokenIdent)
}
case 69:
// line 239 "scan_tokens.rl"
// line 242 "scan_tokens.rl"
p = (te) - 1
{
selfToken()
}
case 70:
// line 251 "scan_tokens.rl"
// line 254 "scan_tokens.rl"
p = (te) - 1
{
@ -3722,7 +3722,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
// line 310 "scan_tokens.rl"
// line 313 "scan_tokens.rl"
// If we fall out here without being in a final state then we've
// encountered something that the scanner can't match, which we'll

2
zcl/zclsyntax/scan_tokens.rl
View File

@ -206,7 +206,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
TemplateInterp => beginTemplateInterp;
TemplateControl => beginTemplateControl;
EndStringTmpl => endStringTemplate;
TemplateStringLiteral => { token(TokenStringLit); };
TemplateStringLiteral => { token(TokenQuotedLit); };
AnyUTF8 => { token(TokenInvalid); };
BrokenUTF8 => { token(TokenBadUTF8); };
*|;

4
zcl/zclsyntax/scan_tokens_test.go
View File

@ -284,7 +284,7 @@ func TestScanTokens_normal(t *testing.T) {
},
},
{
Type: TokenStringLit,
Type: TokenQuotedLit,
Bytes: []byte(`hello`),
Range: zcl.Range{
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
@ -676,7 +676,7 @@ func TestScanTokens_normal(t *testing.T) {
},
},
{
Type: TokenStringLit,
Type: TokenQuotedLit,
Bytes: []byte(` foo`),
Range: zcl.Range{
Start: zcl.Pos{Byte: 8, Line: 1, Column: 9},

3
zcl/zclsyntax/token.go
View File

@ -64,7 +64,8 @@ const (
TokenTemplateControl TokenType = 'λ'
TokenTemplateSeqEnd TokenType = '∎'
TokenStringLit TokenType = 'S'
TokenQuotedLit TokenType = 'Q' // might contain backslash escapes
TokenStringLit TokenType = 'S' // cannot contain backslash escapes
TokenNumberLit TokenType = 'N'
TokenIdent TokenType = 'I'

94
zcl/zclsyntax/token_type_string.go
View File

@ -4,54 +4,56 @@ package zclsyntax
import "fmt"
const _TokenType_name = "TokenNewlineTokenBangTokenBitwiseAndTokenOParenTokenCParenTokenStarTokenPlusTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenCommentTokenOHeredocTokenIdentTokenNumberLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenStarStarTokenInvalidTokenBadUTF8"
const _TokenType_name = "TokenNilTokenNewlineTokenBangTokenBitwiseAndTokenOParenTokenCParenTokenStarTokenPlusTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenCommentTokenOHeredocTokenIdentTokenNumberLitTokenQuotedLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenStarStarTokenInvalidTokenBadUTF8"
var _TokenType_map = map[TokenType]string{
10: _TokenType_name[0:12],
33: _TokenType_name[12:21],
38: _TokenType_name[21:36],
40: _TokenType_name[36:47],
41: _TokenType_name[47:58],
42: _TokenType_name[58:67],
43: _TokenType_name[67:76],
45: _TokenType_name[76:86],
46: _TokenType_name[86:94],
47: _TokenType_name[94:104],
58: _TokenType_name[104:114],
59: _TokenType_name[114:128],
60: _TokenType_name[128:141],
61: _TokenType_name[141:151],
62: _TokenType_name[151:167],
63: _TokenType_name[167:180],
67: _TokenType_name[180:192],
72: _TokenType_name[192:205],
73: _TokenType_name[205:215],
78: _TokenType_name[215:229],
83: _TokenType_name[229:243],
91: _TokenType_name[243:254],
93: _TokenType_name[254:265],
94: _TokenType_name[265:280],
96: _TokenType_name[280:293],
104: _TokenType_name[293:306],
123: _TokenType_name[306:317],
124: _TokenType_name[317:331],
125: _TokenType_name[331:342],
126: _TokenType_name[342:357],
171: _TokenType_name[357:368],
187: _TokenType_name[368:379],
955: _TokenType_name[379:399],
8718: _TokenType_name[399:418],
8743: _TokenType_name[418:426],
8744: _TokenType_name[426:433],
8747: _TokenType_name[433:452],
8800: _TokenType_name[452:465],
8804: _TokenType_name[465:480],
8805: _TokenType_name[480:498],
9220: _TokenType_name[498:506],
9225: _TokenType_name[506:515],
10138: _TokenType_name[515:528],
65533: _TokenType_name[528:540],
128169: _TokenType_name[540:552],
0: _TokenType_name[0:8],
10: _TokenType_name[8:20],
33: _TokenType_name[20:29],
38: _TokenType_name[29:44],
40: _TokenType_name[44:55],
41: _TokenType_name[55:66],
42: _TokenType_name[66:75],
43: _TokenType_name[75:84],
45: _TokenType_name[84:94],
46: _TokenType_name[94:102],
47: _TokenType_name[102:112],
58: _TokenType_name[112:122],
59: _TokenType_name[122:136],
60: _TokenType_name[136:149],
61: _TokenType_name[149:159],
62: _TokenType_name[159:175],
63: _TokenType_name[175:188],
67: _TokenType_name[188:200],
72: _TokenType_name[200:213],
73: _TokenType_name[213:223],
78: _TokenType_name[223:237],
81: _TokenType_name[237:251],
83: _TokenType_name[251:265],
91: _TokenType_name[265:276],
93: _TokenType_name[276:287],
94: _TokenType_name[287:302],
96: _TokenType_name[302:315],
104: _TokenType_name[315:328],
123: _TokenType_name[328:339],
124: _TokenType_name[339:353],
125: _TokenType_name[353:364],
126: _TokenType_name[364:379],
171: _TokenType_name[379:390],
187: _TokenType_name[390:401],
955: _TokenType_name[401:421],
8718: _TokenType_name[421:440],
8743: _TokenType_name[440:448],
8744: _TokenType_name[448:455],
8747: _TokenType_name[455:474],
8800: _TokenType_name[474:487],
8804: _TokenType_name[487:502],
8805: _TokenType_name[502:520],
9220: _TokenType_name[520:528],
9225: _TokenType_name[528:537],
10138: _TokenType_name[537:550],
65533: _TokenType_name[550:562],
128169: _TokenType_name[562:574],
}
func (i TokenType) String() string {

6
zclwrite/parser_test.go
View File

@ -56,7 +56,7 @@ foo "bar" "baz" {
SpacesBefore: 1,
},
{
Type: zclsyntax.TokenStringLit,
Type: zclsyntax.TokenQuotedLit,
Bytes: []byte(`bar`),
SpacesBefore: 0,
},
@ -71,7 +71,7 @@ foo "bar" "baz" {
SpacesBefore: 1,
},
{
Type: zclsyntax.TokenStringLit,
Type: zclsyntax.TokenQuotedLit,
Bytes: []byte(`baz`),
SpacesBefore: 0,
},
@ -106,7 +106,7 @@ foo "bar" "baz" {
SpacesBefore: 1,
},
{
Type: zclsyntax.TokenStringLit,
Type: zclsyntax.TokenQuotedLit,
Bytes: []byte(` cheese `),
SpacesBefore: 0,
},