go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zclsyntax: scanning of comments

Browse Source
This commit is contained in:
Martin Atkins 2017-05-29 09:13:35 -07:00
parent 2551856d22
commit 861d0a1b4e
5 changed files with 2504 additions and 2379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4746
zcl/zclsyntax/scan_tokens.go
View File

File diff suppressed because it is too large Load Diff

11
zcl/zclsyntax/scan_tokens.rl
View File

@ -46,10 +46,17 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
LogicalOr = "||";
Newline = '\r' ? '\n';
EndOfLine = Newline;
BeginStringTmpl = '"';
BeginHeredocTmpl = '<<' ('-')? Ident Newline;
Comment = (
("#" any* EndOfLine) |
("//" any* EndOfLine) |
("/*" any* "*/")
);
# Tabs are not valid, but we accept them in the scanner and mark them
# as tokens so that we can produce diagnostics advising the user to
# use spaces instead.
@ -195,7 +202,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
heredocTemplate := |*
TemplateInterp => beginTemplateInterp;
TemplateControl => beginTemplateControl;
HeredocStringLiteral Newline => heredocLiteralEOL;
HeredocStringLiteral EndOfLine => heredocLiteralEOL;
HeredocStringLiteral => heredocLiteralMidline;
BrokenUTF8 => { token(TokenBadUTF8); };
*|;
@ -205,6 +212,8 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
NumberLit => { token(TokenNumberLit) };
Ident => { token(TokenIdent) };
Comment => { token(TokenComment) };
NotEqual => { token(TokenNotEqual); };
GreaterThanEqual => { token(TokenGreaterThanEq); };
LessThanEqual => { token(TokenLessThanEq); };

65
zcl/zclsyntax/scan_tokens_test.go
View File

@ -1021,6 +1021,71 @@ EOF
},
},
// Comments
{
"# hello\n",
[]Token{
{
Type: TokenComment,
Bytes: []byte("# hello\n"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 8, Line: 2, Column: 1},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 8, Line: 2, Column: 1},
End: zcl.Pos{Byte: 8, Line: 2, Column: 1},
},
},
},
},
{
"// hello\n",
[]Token{
{
Type: TokenComment,
Bytes: []byte("// hello\n"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 9, Line: 2, Column: 1},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 9, Line: 2, Column: 1},
End: zcl.Pos{Byte: 9, Line: 2, Column: 1},
},
},
},
},
{
"/* hello */",
[]Token{
{
Type: TokenComment,
Bytes: []byte("/* hello */"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 11, Line: 1, Column: 12},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 11, Line: 1, Column: 12},
End: zcl.Pos{Byte: 11, Line: 1, Column: 12},
},
},
},
},
// Invalid things
{
`🌻`,

2
zcl/zclsyntax/token.go
View File

@ -65,6 +65,8 @@ const (
TokenNumberLit TokenType = 'N'
TokenIdent TokenType = 'I'
TokenComment TokenType = 'C'
TokenNewline TokenType = '\n'
TokenEOF TokenType = '␄'

59
zcl/zclsyntax/token_type_string.go
View File

@ -4,7 +4,7 @@ package zclsyntax
import "fmt"
const _TokenType_name = "TokenNewlineTokenBangTokenBitwiseAndTokenOParenTokenCParenTokenStarTokenPlusTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenOHeredocTokenIdentTokenNumberLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenStarStarTokenInvalidTokenBadUTF8"
const _TokenType_name = "TokenNewlineTokenBangTokenBitwiseAndTokenOParenTokenCParenTokenStarTokenPlusTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenCommentTokenOHeredocTokenIdentTokenNumberLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenStarStarTokenInvalidTokenBadUTF8"
var _TokenType_map = map[TokenType]string{
10: _TokenType_name[0:12],
@ -23,34 +23,35 @@ var _TokenType_map = map[TokenType]string{
61: _TokenType_name[141:151],
62: _TokenType_name[151:167],
63: _TokenType_name[167:180],
72: _TokenType_name[180:193],
73: _TokenType_name[193:203],
78: _TokenType_name[203:217],
83: _TokenType_name[217:231],
91: _TokenType_name[231:242],
93: _TokenType_name[242:253],
94: _TokenType_name[253:268],
96: _TokenType_name[268:281],
104: _TokenType_name[281:294],
123: _TokenType_name[294:305],
124: _TokenType_name[305:319],
125: _TokenType_name[319:330],
126: _TokenType_name[330:345],
171: _TokenType_name[345:356],
187: _TokenType_name[356:367],
955: _TokenType_name[367:387],
8718: _TokenType_name[387:406],
8743: _TokenType_name[406:414],
8744: _TokenType_name[414:421],
8747: _TokenType_name[421:440],
8800: _TokenType_name[440:453],
8804: _TokenType_name[453:468],
8805: _TokenType_name[468:486],
9220: _TokenType_name[486:494],
9225: _TokenType_name[494:503],
10138: _TokenType_name[503:516],
65533: _TokenType_name[516:528],
128169: _TokenType_name[528:540],
67: _TokenType_name[180:192],
72: _TokenType_name[192:205],
73: _TokenType_name[205:215],
78: _TokenType_name[215:229],
83: _TokenType_name[229:243],
91: _TokenType_name[243:254],
93: _TokenType_name[254:265],
94: _TokenType_name[265:280],
96: _TokenType_name[280:293],
104: _TokenType_name[293:306],
123: _TokenType_name[306:317],
124: _TokenType_name[317:331],
125: _TokenType_name[331:342],
126: _TokenType_name[342:357],
171: _TokenType_name[357:368],
187: _TokenType_name[368:379],
955: _TokenType_name[379:399],
8718: _TokenType_name[399:418],
8743: _TokenType_name[418:426],
8744: _TokenType_name[426:433],
8747: _TokenType_name[433:452],
8800: _TokenType_name[452:465],
8804: _TokenType_name[465:480],
8805: _TokenType_name[480:498],
9220: _TokenType_name[498:506],
9225: _TokenType_name[506:515],
10138: _TokenType_name[515:528],
65533: _TokenType_name[528:540],
128169: _TokenType_name[540:552],
}
func (i TokenType) String() string {