e100bf4723
There are certain tokens that are _never_ valid, so we might as well catch them early in the Lex... functions rather than having to handle them in many different contexts within the parser. Unfortunately for now when such errors occur they tend to be echoed by more confusing errors coming from the parser, but we'll accept that for now.
35 lines
1014 B
Go
35 lines
1014 B
Go
package zclwrite
|
|
|
|
import (
|
|
"github.com/zclconf/go-zcl/zcl"
|
|
"github.com/zclconf/go-zcl/zcl/zclsyntax"
|
|
)
|
|
|
|
// lexConfig uses the zclsyntax scanner to get a token stream and then
|
|
// rewrites it into this package's token model.
|
|
func lexConfig(src []byte) Tokens {
|
|
mainTokens, _ := zclsyntax.LexConfig(src, "", zcl.Pos{Byte: 0, Line: 1, Column: 1})
|
|
ret := make(Tokens, len(mainTokens))
|
|
var lastByteOffset int
|
|
for i, mainToken := range mainTokens {
|
|
// Create a copy of the bytes so that we can mutate without
|
|
// corrupting the original token stream.
|
|
bytes := make([]byte, len(mainToken.Bytes))
|
|
copy(bytes, mainToken.Bytes)
|
|
|
|
ret[i] = &Token{
|
|
Type: mainToken.Type,
|
|
Bytes: bytes,
|
|
|
|
// We assume here that spaces are always ASCII spaces, since
|
|
// that's what the scanner also assumes, and thus the number
|
|
// of bytes skipped is also the number of space characters.
|
|
SpacesBefore: mainToken.Range.Start.Byte - lastByteOffset,
|
|
}
|
|
|
|
lastByteOffset = mainToken.Range.End.Byte
|
|
}
|
|
|
|
return ret
|
|
}
|