go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

hclsyntax: count \r\n line endings properly in source ranges

Browse Source 
Previously we were only counting a \n as starting a new line, so input
using \r\n endings would get treated as one long line for source-range
purposes.

Now we also consider \r\n to be a newline marker, resetting the column
count to zero and incrementing the line just as we would do for a single
\n. This is made easier because the unicode definition of "grapheme
cluster" considers \r\n to be a single character, so we don't need to
do anything special in order to match it.
This commit is contained in:
Martin Atkins 2018-03-08 08:30:58 -08:00
parent 7d6ed4d8f3
commit 5f8ed954ab
2 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hcl/hclsyntax/scan_tokens_test.go
View File

@ -1012,39 +1012,39 @@ EOT
Bytes: []byte("<<EOT\r\n"), Bytes: []byte("<<EOT\r\n"),
Range: hcl.Range{ Range: hcl.Range{
Start: hcl.Pos{Byte: 0, Line: 1, Column: 1}, Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
End: hcl.Pos{Byte: 7, Line: 1, Column: 7}, End: hcl.Pos{Byte: 7, Line: 2, Column: 1},
}, },
}, },
{ {
Type: TokenStringLit, Type: TokenStringLit,
Bytes: []byte("hello world\r\n"), Bytes: []byte("hello world\r\n"),
Range: hcl.Range{ Range: hcl.Range{
Start: hcl.Pos{Byte: 7, Line: 1, Column: 7}, Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
End: hcl.Pos{Byte: 20, Line: 1, Column: 19}, End: hcl.Pos{Byte: 20, Line: 3, Column: 1},
}, },
}, },
{ {
Type: TokenCHeredoc, Type: TokenCHeredoc,
Bytes: []byte("EOT"), Bytes: []byte("EOT"),
Range: hcl.Range{ Range: hcl.Range{
Start: hcl.Pos{Byte: 20, Line: 1, Column: 19}, Start: hcl.Pos{Byte: 20, Line: 3, Column: 1},
End: hcl.Pos{Byte: 23, Line: 1, Column: 22}, End: hcl.Pos{Byte: 23, Line: 3, Column: 4},
}, },
}, },
{ {
Type: TokenNewline, Type: TokenNewline,
Bytes: []byte("\r\n"), Bytes: []byte("\r\n"),
Range: hcl.Range{ Range: hcl.Range{
Start: hcl.Pos{Byte: 23, Line: 1, Column: 22}, Start: hcl.Pos{Byte: 23, Line: 3, Column: 4},
End: hcl.Pos{Byte: 25, Line: 1, Column: 23}, End: hcl.Pos{Byte: 25, Line: 4, Column: 1},
}, },
}, },
{ {
Type: TokenEOF, Type: TokenEOF,
Bytes: []byte{}, Bytes: []byte{},
Range: hcl.Range{ Range: hcl.Range{
Start: hcl.Pos{Byte: 25, Line: 1, Column: 23}, Start: hcl.Pos{Byte: 25, Line: 4, Column: 1},
End: hcl.Pos{Byte: 25, Line: 1, Column: 23}, End: hcl.Pos{Byte: 25, Line: 4, Column: 1},
}, },
}, },
}, },

2
hcl/hclsyntax/token.go
View File

@ -133,7 +133,7 @@ func (f *tokenAccum) emitToken(ty TokenType, startOfs, endOfs int) {
b := f.Bytes[startOfs:endOfs] b := f.Bytes[startOfs:endOfs]
for len(b) > 0 { for len(b) > 0 {
advance, seq, _ := textseg.ScanGraphemeClusters(b, true) advance, seq, _ := textseg.ScanGraphemeClusters(b, true)
if len(seq) == 1 && seq[0] == '\n' { if (len(seq) == 1 && seq[0] == '\n') || (len(seq) == 2 && seq[0] == '\r' && seq[1] == '\n') {
end.Line++ end.Line++
end.Column = 1 end.Column = 1
} else { } else {