go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zclsyntax: scanning of comments

Browse Source
This commit is contained in:
Martin Atkins 2017-05-29 09:13:35 -07:00
parent 2551856d22
commit 861d0a1b4e
5 changed files with 2504 additions and 2379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4746
zcl/zclsyntax/scan_tokens.go
View File

File diff suppressed because it is too large Load Diff

11
zcl/zclsyntax/scan_tokens.rl
View File

@ -46,10 +46,17 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
LogicalOr = "||"; LogicalOr = "||";
Newline = '\r' ? '\n'; Newline = '\r' ? '\n';
EndOfLine = Newline;
BeginStringTmpl = '"'; BeginStringTmpl = '"';
BeginHeredocTmpl = '<<' ('-')? Ident Newline; BeginHeredocTmpl = '<<' ('-')? Ident Newline;
Comment = (
("#" any* EndOfLine) |
("//" any* EndOfLine) |
("/*" any* "*/")
);
# Tabs are not valid, but we accept them in the scanner and mark them # Tabs are not valid, but we accept them in the scanner and mark them
# as tokens so that we can produce diagnostics advising the user to # as tokens so that we can produce diagnostics advising the user to
# use spaces instead. # use spaces instead.
@ -195,7 +202,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
heredocTemplate := |* heredocTemplate := |*
TemplateInterp => beginTemplateInterp; TemplateInterp => beginTemplateInterp;
TemplateControl => beginTemplateControl; TemplateControl => beginTemplateControl;
HeredocStringLiteral Newline => heredocLiteralEOL; HeredocStringLiteral EndOfLine => heredocLiteralEOL;
HeredocStringLiteral => heredocLiteralMidline; HeredocStringLiteral => heredocLiteralMidline;
BrokenUTF8 => { token(TokenBadUTF8); }; BrokenUTF8 => { token(TokenBadUTF8); };
*|; *|;
@ -205,6 +212,8 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
NumberLit => { token(TokenNumberLit) }; NumberLit => { token(TokenNumberLit) };
Ident => { token(TokenIdent) }; Ident => { token(TokenIdent) };
Comment => { token(TokenComment) };
NotEqual => { token(TokenNotEqual); }; NotEqual => { token(TokenNotEqual); };
GreaterThanEqual => { token(TokenGreaterThanEq); }; GreaterThanEqual => { token(TokenGreaterThanEq); };
LessThanEqual => { token(TokenLessThanEq); }; LessThanEqual => { token(TokenLessThanEq); };

65
zcl/zclsyntax/scan_tokens_test.go
View File

@ -1021,6 +1021,71 @@ EOF
}, },
}, },
// Comments
{
"# hello\n",
[]Token{
{
Type: TokenComment,
Bytes: []byte("# hello\n"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 8, Line: 2, Column: 1},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 8, Line: 2, Column: 1},
End: zcl.Pos{Byte: 8, Line: 2, Column: 1},
},
},
},
},
{
"// hello\n",
[]Token{
{
Type: TokenComment,
Bytes: []byte("// hello\n"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 9, Line: 2, Column: 1},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 9, Line: 2, Column: 1},
End: zcl.Pos{Byte: 9, Line: 2, Column: 1},
},
},
},
},
{
"/* hello */",
[]Token{
{
Type: TokenComment,
Bytes: []byte("/* hello */"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 11, Line: 1, Column: 12},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 11, Line: 1, Column: 12},
End: zcl.Pos{Byte: 11, Line: 1, Column: 12},
},
},
},
},
// Invalid things // Invalid things
{ {
`🌻`, `🌻`,

2
zcl/zclsyntax/token.go
View File

@ -65,6 +65,8 @@ const (
TokenNumberLit TokenType = 'N' TokenNumberLit TokenType = 'N'
TokenIdent TokenType = 'I' TokenIdent TokenType = 'I'
TokenComment TokenType = 'C'
TokenNewline TokenType = '\n' TokenNewline TokenType = '\n'
TokenEOF TokenType = '␄' TokenEOF TokenType = '␄'

59
zcl/zclsyntax/token_type_string.go
View File

@ -4,7 +4,7 @@ package zclsyntax
import "fmt" import "fmt"
const _TokenType_name = "TokenNewlineTokenBangTokenBitwiseAndTokenOParenTokenCParenTokenStarTokenPlusTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenOHeredocTokenIdentTokenNumberLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenStarStarTokenInvalidTokenBadUTF8" const _TokenType_name = "TokenNewlineTokenBangTokenBitwiseAndTokenOParenTokenCParenTokenStarTokenPlusTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenCommentTokenOHeredocTokenIdentTokenNumberLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenStarStarTokenInvalidTokenBadUTF8"
var _TokenType_map = map[TokenType]string{ var _TokenType_map = map[TokenType]string{
10: _TokenType_name[0:12], 10: _TokenType_name[0:12],
@ -23,34 +23,35 @@ var _TokenType_map = map[TokenType]string{
61: _TokenType_name[141:151], 61: _TokenType_name[141:151],
62: _TokenType_name[151:167], 62: _TokenType_name[151:167],
63: _TokenType_name[167:180], 63: _TokenType_name[167:180],
72: _TokenType_name[180:193], 67: _TokenType_name[180:192],
73: _TokenType_name[193:203], 72: _TokenType_name[192:205],
78: _TokenType_name[203:217], 73: _TokenType_name[205:215],
83: _TokenType_name[217:231], 78: _TokenType_name[215:229],
91: _TokenType_name[231:242], 83: _TokenType_name[229:243],
93: _TokenType_name[242:253], 91: _TokenType_name[243:254],
94: _TokenType_name[253:268], 93: _TokenType_name[254:265],
96: _TokenType_name[268:281], 94: _TokenType_name[265:280],
104: _TokenType_name[281:294], 96: _TokenType_name[280:293],
123: _TokenType_name[294:305], 104: _TokenType_name[293:306],
124: _TokenType_name[305:319], 123: _TokenType_name[306:317],
125: _TokenType_name[319:330], 124: _TokenType_name[317:331],
126: _TokenType_name[330:345], 125: _TokenType_name[331:342],
171: _TokenType_name[345:356], 126: _TokenType_name[342:357],
187: _TokenType_name[356:367], 171: _TokenType_name[357:368],
955: _TokenType_name[367:387], 187: _TokenType_name[368:379],
8718: _TokenType_name[387:406], 955: _TokenType_name[379:399],
8743: _TokenType_name[406:414], 8718: _TokenType_name[399:418],
8744: _TokenType_name[414:421], 8743: _TokenType_name[418:426],
8747: _TokenType_name[421:440], 8744: _TokenType_name[426:433],
8800: _TokenType_name[440:453], 8747: _TokenType_name[433:452],
8804: _TokenType_name[453:468], 8800: _TokenType_name[452:465],
8805: _TokenType_name[468:486], 8804: _TokenType_name[465:480],
9220: _TokenType_name[486:494], 8805: _TokenType_name[480:498],
9225: _TokenType_name[494:503], 9220: _TokenType_name[498:506],
10138: _TokenType_name[503:516], 9225: _TokenType_name[506:515],
65533: _TokenType_name[516:528], 10138: _TokenType_name[515:528],
128169: _TokenType_name[528:540], 65533: _TokenType_name[528:540],
128169: _TokenType_name[540:552],
} }
func (i TokenType) String() string { func (i TokenType) String() string {