18e45ec05c
This requires some extra state-keeping because we allow templates to be nested inside templates. This takes us outside of the world of regular languages, but we accept that here because it makes things easier to deal with down the line in the parser. The methodology is to keep track of how many braces are open at a given time and then, when a nested template interpolation begins, record the current brace level. Then, when a closing brace is encountered, if its nesting level is at the top of the stack then we pop off the stack and return to "main" parsing mode. Ragel's existing idea of calling and returning from machines is important here too. As this currently stands this is not actually needed, but once heredocs are in play we will have two possible places to return to at the end of an interpolation sequence, so the state return stack maintained by Ragel will determine whether to return to string mode or heredoc mode.
899 lines
19 KiB
Go
899 lines
19 KiB
Go
package zclsyntax
|
|
|
|
import (
|
|
"reflect"
|
|
"testing"
|
|
|
|
"github.com/kylelemons/godebug/pretty"
|
|
"github.com/zclconf/go-zcl/zcl"
|
|
)
|
|
|
|
func TestScanTokens(t *testing.T) {
|
|
tests := []struct {
|
|
input string
|
|
want []Token
|
|
}{
|
|
// Empty input
|
|
{
|
|
``,
|
|
[]Token{
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
` `,
|
|
[]Token{
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
// TokenNumberLit
|
|
{
|
|
`1`,
|
|
[]Token{
|
|
{
|
|
Type: TokenNumberLit,
|
|
Bytes: []byte(`1`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`12`,
|
|
[]Token{
|
|
{
|
|
Type: TokenNumberLit,
|
|
Bytes: []byte(`12`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`12.3`,
|
|
[]Token{
|
|
{
|
|
Type: TokenNumberLit,
|
|
Bytes: []byte(`12.3`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`1e2`,
|
|
[]Token{
|
|
{
|
|
Type: TokenNumberLit,
|
|
Bytes: []byte(`1e2`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`1e+2`,
|
|
[]Token{
|
|
{
|
|
Type: TokenNumberLit,
|
|
Bytes: []byte(`1e+2`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
// TokenIdent
|
|
{
|
|
`hello`,
|
|
[]Token{
|
|
{
|
|
Type: TokenIdent,
|
|
Bytes: []byte(`hello`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`h3ll0`,
|
|
[]Token{
|
|
{
|
|
Type: TokenIdent,
|
|
Bytes: []byte(`h3ll0`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`héllo`, // combining acute accent
|
|
[]Token{
|
|
{
|
|
Type: TokenIdent,
|
|
Bytes: []byte(`héllo`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 7, Line: 1, Column: 6},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
// Literal-only Templates (string literals, effectively)
|
|
{
|
|
`""`,
|
|
[]Token{
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`"hello"`,
|
|
[]Token{
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenStringLit,
|
|
Bytes: []byte(`hello`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
// Templates with interpolations and control sequences
|
|
{
|
|
`"${1}"`,
|
|
[]Token{
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateInterp,
|
|
Bytes: []byte(`${`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenNumberLit,
|
|
Bytes: []byte(`1`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateSeqEnd,
|
|
Bytes: []byte(`}`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
End: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
End: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`"!{a}"`,
|
|
[]Token{
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateControl,
|
|
Bytes: []byte(`!{`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenIdent,
|
|
Bytes: []byte(`a`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateSeqEnd,
|
|
Bytes: []byte(`}`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
End: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
End: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`"${{}}"`,
|
|
[]Token{
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateInterp,
|
|
Bytes: []byte(`${`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenOBrace,
|
|
Bytes: []byte(`{`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCBrace,
|
|
Bytes: []byte(`}`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateSeqEnd,
|
|
Bytes: []byte(`}`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
End: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`"${""}"`,
|
|
[]Token{
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateInterp,
|
|
Bytes: []byte(`${`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateSeqEnd,
|
|
Bytes: []byte(`}`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
End: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`"${"${a}"}"`,
|
|
[]Token{
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateInterp,
|
|
Bytes: []byte(`${`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateInterp,
|
|
Bytes: []byte(`${`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
End: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenIdent,
|
|
Bytes: []byte(`a`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateSeqEnd,
|
|
Bytes: []byte(`}`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
End: zcl.Pos{Byte: 8, Line: 1, Column: 9},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 8, Line: 1, Column: 9},
|
|
End: zcl.Pos{Byte: 9, Line: 1, Column: 10},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateSeqEnd,
|
|
Bytes: []byte(`}`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 9, Line: 1, Column: 10},
|
|
End: zcl.Pos{Byte: 10, Line: 1, Column: 11},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 10, Line: 1, Column: 11},
|
|
End: zcl.Pos{Byte: 11, Line: 1, Column: 12},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 11, Line: 1, Column: 12},
|
|
End: zcl.Pos{Byte: 11, Line: 1, Column: 12},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`"${"${a} foo"}"`,
|
|
[]Token{
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateInterp,
|
|
Bytes: []byte(`${`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenOQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateInterp,
|
|
Bytes: []byte(`${`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
End: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenIdent,
|
|
Bytes: []byte(`a`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateSeqEnd,
|
|
Bytes: []byte(`}`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
End: zcl.Pos{Byte: 8, Line: 1, Column: 9},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenStringLit,
|
|
Bytes: []byte(` foo`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 8, Line: 1, Column: 9},
|
|
End: zcl.Pos{Byte: 12, Line: 1, Column: 13},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 12, Line: 1, Column: 13},
|
|
End: zcl.Pos{Byte: 13, Line: 1, Column: 14},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenTemplateSeqEnd,
|
|
Bytes: []byte(`}`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 13, Line: 1, Column: 14},
|
|
End: zcl.Pos{Byte: 14, Line: 1, Column: 15},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCQuote,
|
|
Bytes: []byte(`"`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 14, Line: 1, Column: 15},
|
|
End: zcl.Pos{Byte: 15, Line: 1, Column: 16},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 15, Line: 1, Column: 16},
|
|
End: zcl.Pos{Byte: 15, Line: 1, Column: 16},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
// Combinations
|
|
{
|
|
` (1 + 2) * 3 `,
|
|
[]Token{
|
|
{
|
|
Type: TokenOParen,
|
|
Bytes: []byte(`(`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenNumberLit,
|
|
Bytes: []byte(`1`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenPlus,
|
|
Bytes: []byte(`+`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
|
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenNumberLit,
|
|
Bytes: []byte(`2`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
|
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenCParen,
|
|
Bytes: []byte(`)`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
|
End: zcl.Pos{Byte: 8, Line: 1, Column: 9},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenStar,
|
|
Bytes: []byte(`*`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 9, Line: 1, Column: 10},
|
|
End: zcl.Pos{Byte: 10, Line: 1, Column: 11},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenNumberLit,
|
|
Bytes: []byte(`3`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 11, Line: 1, Column: 12},
|
|
End: zcl.Pos{Byte: 12, Line: 1, Column: 13},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 13, Line: 1, Column: 14},
|
|
End: zcl.Pos{Byte: 13, Line: 1, Column: 14},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
// Invalid things
|
|
{
|
|
`🌻`,
|
|
[]Token{
|
|
{
|
|
Type: TokenInvalid,
|
|
Bytes: []byte(`🌻`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 4, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 4, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
`|`,
|
|
[]Token{
|
|
{
|
|
Type: TokenBitwiseOr,
|
|
Bytes: []byte(`|`),
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
"\x80", // UTF-8 continuation without an introducer
|
|
[]Token{
|
|
{
|
|
Type: TokenBadUTF8,
|
|
Bytes: []byte{0x80},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
" \x80\x80", // UTF-8 continuation without an introducer
|
|
[]Token{
|
|
{
|
|
Type: TokenBadUTF8,
|
|
Bytes: []byte{0x80},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
|
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenBadUTF8,
|
|
Bytes: []byte{0x80},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
"\t\t",
|
|
[]Token{
|
|
{
|
|
Type: TokenTabs,
|
|
Bytes: []byte{0x09, 0x09},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
|
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
},
|
|
},
|
|
{
|
|
Type: TokenEOF,
|
|
Bytes: []byte{},
|
|
Range: zcl.Range{
|
|
Start: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
prettyConfig := &pretty.Config{
|
|
Diffable: true,
|
|
IncludeUnexported: true,
|
|
PrintStringers: true,
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.input, func(t *testing.T) {
|
|
got := scanTokens([]byte(test.input), "", zcl.Pos{Byte: 0, Line: 1, Column: 1})
|
|
|
|
if !reflect.DeepEqual(got, test.want) {
|
|
diff := prettyConfig.Compare(test.want, got)
|
|
t.Errorf(
|
|
"wrong result\ninput: %s\ndiff: %s",
|
|
test.input, diff,
|
|
)
|
|
}
|
|
})
|
|
}
|
|
}
|