476e2c127e
In zclwrite we throw away the absolute source position information and instead just retain the number of spaces before each token. This different model allows us to rewrite parts of the token sequence without needing to re-adjust all of the positions, and it also allows us to do simple indentation and spacing adjustments just by walking through the token list and adjusting these numbers.
35 lines
1011 B
Go
35 lines
1011 B
Go
package zclwrite
|
|
|
|
import (
|
|
"github.com/zclconf/go-zcl/zcl"
|
|
"github.com/zclconf/go-zcl/zcl/zclsyntax"
|
|
)
|
|
|
|
// lexConfig uses the zclsyntax scanner to get a token stream and then
|
|
// rewrites it into this package's token model.
|
|
func lexConfig(src []byte) Tokens {
|
|
mainTokens := zclsyntax.LexConfig(src, "", zcl.Pos{Byte: 0, Line: 1, Column: 1})
|
|
ret := make(Tokens, len(mainTokens))
|
|
var lastByteOffset int
|
|
for i, mainToken := range mainTokens {
|
|
// Create a copy of the bytes so that we can mutate without
|
|
// corrupting the original token stream.
|
|
bytes := make([]byte, len(mainToken.Bytes))
|
|
copy(bytes, mainToken.Bytes)
|
|
|
|
ret[i] = &Token{
|
|
Type: mainToken.Type,
|
|
Bytes: bytes,
|
|
|
|
// We assume here that spaces are always ASCII spaces, since
|
|
// that's what the scanner also assumes, and thus the number
|
|
// of bytes skipped is also the number of space characters.
|
|
SpacesBefore: mainToken.Range.Start.Byte - lastByteOffset,
|
|
}
|
|
|
|
lastByteOffset = mainToken.Range.End.Byte
|
|
}
|
|
|
|
return ret
|
|
}
|