hcl/hclsyntax: Fix scanning of multi-line comments
The pattern here was being too greedy because by default the longest match is taken, and "*/" followed by more content is always longer than just the "*/" to terminate the match. The :>> symbol is a "finish guard" which tells Ragel to prefer to exit from any* as soon as "*/" matches, making the match ungreedy. The result of this is that input files containing more than one multi-line comment will now tokenize each one separately, whereas before we would create one long comment including everything from the first /* to the final */ in the file, effectively causing parts of the file to be ignored entirely.
This commit is contained in:
parent
4d82d52bfa
commit
cd67ba1b25
File diff suppressed because it is too large
Load Diff
@ -60,7 +60,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To
|
|||||||
Comment = (
|
Comment = (
|
||||||
("#" (any - EndOfLine)* EndOfLine) |
|
("#" (any - EndOfLine)* EndOfLine) |
|
||||||
("//" (any - EndOfLine)* EndOfLine) |
|
("//" (any - EndOfLine)* EndOfLine) |
|
||||||
("/*" any* "*/")
|
("/*" any* :>> "*/")
|
||||||
);
|
);
|
||||||
|
|
||||||
# Note: hclwrite assumes that only ASCII spaces appear between tokens,
|
# Note: hclwrite assumes that only ASCII spaces appear between tokens,
|
||||||
|
@ -1526,6 +1526,43 @@ EOF
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"/* hello */ howdy /* hey */",
|
||||||
|
[]Token{
|
||||||
|
{
|
||||||
|
Type: TokenComment,
|
||||||
|
Bytes: []byte("/* hello */"),
|
||||||
|
Range: hcl.Range{
|
||||||
|
Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
|
||||||
|
End: hcl.Pos{Byte: 11, Line: 1, Column: 12},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: TokenIdent,
|
||||||
|
Bytes: []byte("howdy"),
|
||||||
|
Range: hcl.Range{
|
||||||
|
Start: hcl.Pos{Byte: 12, Line: 1, Column: 13},
|
||||||
|
End: hcl.Pos{Byte: 17, Line: 1, Column: 18},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: TokenComment,
|
||||||
|
Bytes: []byte("/* hey */"),
|
||||||
|
Range: hcl.Range{
|
||||||
|
Start: hcl.Pos{Byte: 18, Line: 1, Column: 19},
|
||||||
|
End: hcl.Pos{Byte: 27, Line: 1, Column: 28},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: TokenEOF,
|
||||||
|
Bytes: []byte{},
|
||||||
|
Range: hcl.Range{
|
||||||
|
Start: hcl.Pos{Byte: 27, Line: 1, Column: 28},
|
||||||
|
End: hcl.Pos{Byte: 27, Line: 1, Column: 28},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
// Invalid things
|
// Invalid things
|
||||||
{
|
{
|
||||||
|
Loading…
x
Reference in New Issue
Block a user