go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

hclsyntax: Allow single-line comments at EOF without newline

Browse Source 
Previously we were using the EndOfLine as a mandatory marker to end the
comment, but that meant that if a comment appeared immediately before EOF
without a newline on the end it would fail to match.

Now we use the :>> operator similarly to how we previously fixed
greediness in the multi-line comment case: it tells Ragel to end the
Comment production if the following pattern matches (if EndOfLine is found)
but also allows the point before EndOfLine to be a final state, in case
EOF shows up there.
This commit is contained in:
Martin Atkins 2019-05-15 15:13:57 -07:00
parent 6a61d80ae3
commit d1ed8ee699
3 changed files with 3464 additions and 3348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6612
hcl/hclsyntax/scan_tokens.go
View File

File diff suppressed because it is too large Load Diff

12
hcl/hclsyntax/scan_tokens.rl
View File

@ -63,8 +63,16 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To
BeginHeredocTmpl = '<<' ('-')? Ident Newline;
Comment = (
("#" (any - EndOfLine)* EndOfLine) |
("//" (any - EndOfLine)* EndOfLine) |
# The :>> operator in these is a "finish-guarded concatenation",
# which terminates the sequence on its left when it completes
# the sequence on its right.
# In the single-line comment cases this is allowing us to make
# the trailing EndOfLine optional while still having the overall
# pattern terminate. In the multi-line case it ensures that
# the first comment in the file ends at the first */, rather than
# gobbling up all of the "any*" until the _final_ */ in the file.
("#" (any - EndOfLine)* :>> EndOfLine?) |
("//" (any - EndOfLine)* :>> EndOfLine?) |
("/*" any* :>> "*/")
);

188
hcl/hclsyntax/scan_tokens_test.go
View File

@ -1,13 +1,10 @@
package hclsyntax
import (
"reflect"
"testing"
"github.com/go-test/deep"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/hcl2/hcl"
"github.com/kylelemons/godebug/pretty"
)
func TestScanTokens_normal(t *testing.T) {
@ -1814,6 +1811,154 @@ EOF
},
},
},
{
"// hello\n// hello",
[]Token{
{
Type: TokenComment,
Bytes: []byte("// hello\n"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
End: hcl.Pos{Byte: 9, Line: 2, Column: 1},
},
},
{
Type: TokenComment,
Bytes: []byte("// hello"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 9, Line: 2, Column: 1},
End: hcl.Pos{Byte: 17, Line: 2, Column: 9},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: hcl.Range{
Start: hcl.Pos{Byte: 17, Line: 2, Column: 9},
End: hcl.Pos{Byte: 17, Line: 2, Column: 9},
},
},
},
},
{
"// hello\nfoo\n// hello",
[]Token{
{
Type: TokenComment,
Bytes: []byte("// hello\n"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
End: hcl.Pos{Byte: 9, Line: 2, Column: 1},
},
},
{
Type: TokenIdent,
Bytes: []byte("foo"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 9, Line: 2, Column: 1},
End: hcl.Pos{Byte: 12, Line: 2, Column: 4},
},
},
{
Type: TokenNewline,
Bytes: []byte("\n"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 12, Line: 2, Column: 4},
End: hcl.Pos{Byte: 13, Line: 3, Column: 1},
},
},
{
Type: TokenComment,
Bytes: []byte("// hello"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 13, Line: 3, Column: 1},
End: hcl.Pos{Byte: 21, Line: 3, Column: 9},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: hcl.Range{
Start: hcl.Pos{Byte: 21, Line: 3, Column: 9},
End: hcl.Pos{Byte: 21, Line: 3, Column: 9},
},
},
},
},
{
"# hello\n# hello",
[]Token{
{
Type: TokenComment,
Bytes: []byte("# hello\n"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
End: hcl.Pos{Byte: 8, Line: 2, Column: 1},
},
},
{
Type: TokenComment,
Bytes: []byte("# hello"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 8, Line: 2, Column: 1},
End: hcl.Pos{Byte: 15, Line: 2, Column: 8},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: hcl.Range{
Start: hcl.Pos{Byte: 15, Line: 2, Column: 8},
End: hcl.Pos{Byte: 15, Line: 2, Column: 8},
},
},
},
},
{
"# hello\nfoo\n# hello",
[]Token{
{
Type: TokenComment,
Bytes: []byte("# hello\n"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
End: hcl.Pos{Byte: 8, Line: 2, Column: 1},
},
},
{
Type: TokenIdent,
Bytes: []byte("foo"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 8, Line: 2, Column: 1},
End: hcl.Pos{Byte: 11, Line: 2, Column: 4},
},
},
{
Type: TokenNewline,
Bytes: []byte("\n"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 11, Line: 2, Column: 4},
End: hcl.Pos{Byte: 12, Line: 3, Column: 1},
},
},
{
Type: TokenComment,
Bytes: []byte("# hello"),
Range: hcl.Range{
Start: hcl.Pos{Byte: 12, Line: 3, Column: 1},
End: hcl.Pos{Byte: 19, Line: 3, Column: 8},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: hcl.Range{
Start: hcl.Pos{Byte: 19, Line: 3, Column: 8},
End: hcl.Pos{Byte: 19, Line: 3, Column: 8},
},
},
},
},
{
"/* hello */",
[]Token{
@ -2132,30 +2277,13 @@ EOF
},
}
prettyConfig := &pretty.Config{
Diffable: true,
IncludeUnexported: true,
PrintStringers: true,
}
for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
got := scanTokens([]byte(test.input), "", hcl.Pos{Byte: 0, Line: 1, Column: 1}, scanNormal)
if !reflect.DeepEqual(got, test.want) {
diff := prettyConfig.Compare(test.want, got)
t.Errorf(
"wrong result\ninput: %s\ndiff: %s",
test.input, diff,
)
if diff := cmp.Diff(test.want, got); diff != "" {
t.Errorf("wrong result\n%s", diff)
}
// "pretty" diff output is not helpful for all differences, so
// we'll also print out a list of specific differences.
for _, problem := range deep.Equal(got, test.want) {
t.Error(problem)
}
})
}
}
@ -2339,22 +2467,12 @@ func TestScanTokens_template(t *testing.T) {
},
}
prettyConfig := &pretty.Config{
Diffable: true,
IncludeUnexported: true,
PrintStringers: true,
}
for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
got := scanTokens([]byte(test.input), "", hcl.Pos{Byte: 0, Line: 1, Column: 1}, scanTemplate)
if !reflect.DeepEqual(got, test.want) {
diff := prettyConfig.Compare(test.want, got)
t.Errorf(
"wrong result\ninput: %s\ndiff: %s",
test.input, diff,
)
if diff := cmp.Diff(test.want, got); diff != "" {
t.Errorf("wrong result\n%s", diff)
}
})
}