hcl: split into meaningful packages
This commit is contained in:
Fatih Arslan
parent
72f3456c0f
commit
e93a8e97ca
94
ast/ast.go
Normal file
94
ast/ast.go
Normal file
@ -0,0 +1,94 @@
|
||||
package ast
|
||||
|
||||
import "github.com/fatih/hcl/token"
|
||||
|
||||
// Node is an element in the abstract syntax tree.
|
||||
type Node interface {
|
||||
node()
|
||||
Pos() token.Pos
|
||||
}
|
||||
|
||||
func (ObjectList) node() {}
|
||||
func (ObjectKey) node() {}
|
||||
func (ObjectItem) node() {}
|
||||
|
||||
func (ObjectType) node() {}
|
||||
func (LiteralType) node() {}
|
||||
func (ListType) node() {}
|
||||
|
||||
// ObjectList represents a list of ObjectItems. An HCL file itself is an
|
||||
// ObjectList.
|
||||
type ObjectList struct {
|
||||
Items []*ObjectItem
|
||||
}
|
||||
|
||||
func (o *ObjectList) Add(item *ObjectItem) {
|
||||
o.Items = append(o.Items, item)
|
||||
}
|
||||
|
||||
func (o *ObjectList) Pos() token.Pos {
|
||||
// always returns the uninitiliazed position
|
||||
return o.Items[0].Pos()
|
||||
}
|
||||
|
||||
// ObjectItem represents a HCL Object Item. An item is represented with a key
|
||||
// (or keys). It can be an assignment or an object (both normal and nested)
|
||||
type ObjectItem struct {
|
||||
// keys is only one length long if it's of type assignment. If it's a
|
||||
// nested object it can be larger than one. In that case "assign" is
|
||||
// invalid as there is no assignments for a nested object.
|
||||
Keys []*ObjectKey
|
||||
|
||||
// assign contains the position of "=", if any
|
||||
Assign token.Pos
|
||||
|
||||
// val is the item itself. It can be an object,list, number, bool or a
|
||||
// string. If key length is larger than one, val can be only of type
|
||||
// Object.
|
||||
Val Node
|
||||
}
|
||||
|
||||
func (o *ObjectItem) Pos() token.Pos {
|
||||
return o.Keys[0].Pos()
|
||||
}
|
||||
|
||||
// ObjectKeys are either an identifier or of type string.
|
||||
type ObjectKey struct {
|
||||
Token token.Token
|
||||
}
|
||||
|
||||
func (o *ObjectKey) Pos() token.Pos {
|
||||
return o.Token.Pos
|
||||
}
|
||||
|
||||
// LiteralType represents a literal of basic type. Valid types are:
|
||||
// token.NUMBER, token.FLOAT, token.BOOL and token.STRING
|
||||
type LiteralType struct {
|
||||
Token token.Token
|
||||
}
|
||||
|
||||
func (l *LiteralType) Pos() token.Pos {
|
||||
return l.Token.Pos
|
||||
}
|
||||
|
||||
// ListStatement represents a HCL List type
|
||||
type ListType struct {
|
||||
Lbrack token.Pos // position of "["
|
||||
Rbrack token.Pos // position of "]"
|
||||
List []Node // the elements in lexical order
|
||||
}
|
||||
|
||||
func (l *ListType) Pos() token.Pos {
|
||||
return l.Lbrack
|
||||
}
|
||||
|
||||
// ObjectType represents a HCL Object Type
|
||||
type ObjectType struct {
|
||||
Lbrace token.Pos // position of "{"
|
||||
Rbrace token.Pos // position of "}"
|
||||
List []Node // the nodes in lexical order
|
||||
}
|
||||
|
||||
func (b *ObjectType) Pos() token.Pos {
|
||||
return b.Lbrace
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package parser
|
||||
package ast
|
||||
|
||||
// Walk traverses an AST in depth-first order: It starts by calling fn(node);
|
||||
// node must not be nil. If f returns true, Walk invokes f recursively for
|
||||
@ -10,24 +10,24 @@ func Walk(node Node, fn func(Node) bool) {
|
||||
|
||||
switch n := node.(type) {
|
||||
case *ObjectList:
|
||||
for _, item := range n.items {
|
||||
for _, item := range n.Items {
|
||||
Walk(item, fn)
|
||||
}
|
||||
case *ObjectKey:
|
||||
// nothing to do
|
||||
case *ObjectItem:
|
||||
for _, k := range n.keys {
|
||||
for _, k := range n.Keys {
|
||||
Walk(k, fn)
|
||||
}
|
||||
Walk(n.val, fn)
|
||||
Walk(n.Val, fn)
|
||||
case *LiteralType:
|
||||
// nothing to do
|
||||
case *ListType:
|
||||
for _, l := range n.list {
|
||||
for _, l := range n.List {
|
||||
Walk(l, fn)
|
||||
}
|
||||
case *ObjectType:
|
||||
for _, l := range n.list {
|
||||
for _, l := range n.List {
|
||||
Walk(l, fn)
|
||||
}
|
||||
}
|
||||
116
parser/ast.go
116
parser/ast.go
@ -1,116 +0,0 @@
|
||||
package parser
|
||||
|
||||
import "github.com/fatih/hcl/scanner"
|
||||
|
||||
// Node is an element in the abstract syntax tree.
|
||||
type Node interface {
|
||||
node()
|
||||
Pos() scanner.Pos
|
||||
}
|
||||
|
||||
func (ObjectList) node() {}
|
||||
func (ObjectKey) node() {}
|
||||
func (ObjectItem) node() {}
|
||||
|
||||
func (ObjectType) node() {}
|
||||
func (LiteralType) node() {}
|
||||
func (ListType) node() {}
|
||||
|
||||
// ObjectList represents a list of ObjectItems. An HCL file itself is an
|
||||
// ObjectList.
|
||||
type ObjectList struct {
|
||||
items []*ObjectItem
|
||||
}
|
||||
|
||||
func (o *ObjectList) add(item *ObjectItem) {
|
||||
o.items = append(o.items, item)
|
||||
}
|
||||
|
||||
func (o *ObjectList) Pos() scanner.Pos {
|
||||
// always returns the uninitiliazed position
|
||||
return o.items[0].Pos()
|
||||
}
|
||||
|
||||
// ObjectItem represents a HCL Object Item. An item is represented with a key
|
||||
// (or keys). It can be an assignment or an object (both normal and nested)
|
||||
type ObjectItem struct {
|
||||
// keys is only one length long if it's of type assignment. If it's a
|
||||
// nested object it can be larger than one. In that case "assign" is
|
||||
// invalid as there is no assignments for a nested object.
|
||||
keys []*ObjectKey
|
||||
|
||||
// assign contains the position of "=", if any
|
||||
assign scanner.Pos
|
||||
|
||||
// val is the item itself. It can be an object,list, number, bool or a
|
||||
// string. If key length is larger than one, val can be only of type
|
||||
// Object.
|
||||
val Node
|
||||
}
|
||||
|
||||
func (o *ObjectItem) Pos() scanner.Pos {
|
||||
return o.keys[0].Pos()
|
||||
}
|
||||
|
||||
// ObjectKeys are either an identifier or of type string.
|
||||
type ObjectKey struct {
|
||||
token scanner.Token
|
||||
}
|
||||
|
||||
func (o *ObjectKey) Pos() scanner.Pos {
|
||||
return o.token.Pos
|
||||
}
|
||||
|
||||
// isValid() returns true if the underlying identifier satisfies one of the
|
||||
// valid types (IDENT or STRING)
|
||||
func (o *ObjectKey) isValid() bool {
|
||||
switch o.token.Type {
|
||||
case scanner.IDENT, scanner.STRING:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// LiteralType represents a literal of basic type. Valid types are:
|
||||
// scanner.NUMBER, scanner.FLOAT, scanner.BOOL and scanner.STRING
|
||||
type LiteralType struct {
|
||||
token scanner.Token
|
||||
}
|
||||
|
||||
// isValid() returns true if the underlying identifier satisfies one of the
|
||||
// valid types.
|
||||
func (l *LiteralType) isValid() bool {
|
||||
switch l.token.Type {
|
||||
case scanner.NUMBER, scanner.FLOAT, scanner.BOOL, scanner.STRING:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (l *LiteralType) Pos() scanner.Pos {
|
||||
return l.token.Pos
|
||||
}
|
||||
|
||||
// ListStatement represents a HCL List type
|
||||
type ListType struct {
|
||||
lbrack scanner.Pos // position of "["
|
||||
rbrack scanner.Pos // position of "]"
|
||||
list []Node // the elements in lexical order
|
||||
}
|
||||
|
||||
func (l *ListType) Pos() scanner.Pos {
|
||||
return l.lbrack
|
||||
}
|
||||
|
||||
// ObjectType represents a HCL Object Type
|
||||
type ObjectType struct {
|
||||
lbrace scanner.Pos // position of "{"
|
||||
rbrace scanner.Pos // position of "}"
|
||||
list []Node // the nodes in lexical order
|
||||
}
|
||||
|
||||
func (b *ObjectType) Pos() scanner.Pos {
|
||||
return b.lbrace
|
||||
}
|
||||
@ -4,14 +4,16 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/fatih/hcl/ast"
|
||||
"github.com/fatih/hcl/scanner"
|
||||
"github.com/fatih/hcl/token"
|
||||
)
|
||||
|
||||
type Parser struct {
|
||||
sc *scanner.Scanner
|
||||
|
||||
tok scanner.Token // last read token
|
||||
prevTok scanner.Token // previous read token
|
||||
tok token.Token // last read token
|
||||
prevTok token.Token // previous read token
|
||||
|
||||
enableTrace bool
|
||||
indent int
|
||||
@ -27,9 +29,9 @@ func New(src []byte) *Parser {
|
||||
var errEofToken = errors.New("EOF token found")
|
||||
|
||||
// Parse returns the fully parsed source and returns the abstract syntax tree.
|
||||
func (p *Parser) Parse() (Node, error) {
|
||||
func (p *Parser) Parse() (ast.Node, error) {
|
||||
defer un(trace(p, "ParseObjectList"))
|
||||
node := &ObjectList{}
|
||||
node := &ast.ObjectList{}
|
||||
|
||||
for {
|
||||
n, err := p.parseObjectItem()
|
||||
@ -41,14 +43,14 @@ func (p *Parser) Parse() (Node, error) {
|
||||
}
|
||||
|
||||
// we successfully parsed a node, add it to the final source node
|
||||
node.add(n)
|
||||
node.Add(n)
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// parseObjectItem parses a single object item
|
||||
func (p *Parser) parseObjectItem() (*ObjectItem, error) {
|
||||
func (p *Parser) parseObjectItem() (*ast.ObjectItem, error) {
|
||||
defer un(trace(p, "ParseObjectItem"))
|
||||
|
||||
keys, err := p.parseObjectKey()
|
||||
@ -58,19 +60,19 @@ func (p *Parser) parseObjectItem() (*ObjectItem, error) {
|
||||
|
||||
// either an assignment or object
|
||||
switch p.tok.Type {
|
||||
case scanner.ASSIGN:
|
||||
o := &ObjectItem{
|
||||
keys: keys,
|
||||
assign: p.tok.Pos,
|
||||
case token.ASSIGN:
|
||||
o := &ast.ObjectItem{
|
||||
Keys: keys,
|
||||
Assign: p.tok.Pos,
|
||||
}
|
||||
|
||||
o.val, err = p.parseType()
|
||||
o.Val, err = p.parseType()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return o, nil
|
||||
case scanner.LBRACE:
|
||||
case token.LBRACE:
|
||||
if len(keys) > 1 {
|
||||
// nested object
|
||||
fmt.Println("nested object")
|
||||
@ -85,20 +87,20 @@ func (p *Parser) parseObjectItem() (*ObjectItem, error) {
|
||||
|
||||
// parseType parses any type of Type, such as number, bool, string, object or
|
||||
// list.
|
||||
func (p *Parser) parseType() (Node, error) {
|
||||
func (p *Parser) parseType() (ast.Node, error) {
|
||||
defer un(trace(p, "ParseType"))
|
||||
tok := p.scan()
|
||||
|
||||
switch tok.Type {
|
||||
case scanner.NUMBER, scanner.FLOAT, scanner.BOOL, scanner.STRING:
|
||||
case token.NUMBER, token.FLOAT, token.BOOL, token.STRING:
|
||||
return p.parseLiteralType()
|
||||
case scanner.LBRACE:
|
||||
case token.LBRACE:
|
||||
return p.parseObjectType()
|
||||
case scanner.LBRACK:
|
||||
case token.LBRACK:
|
||||
return p.parseListType()
|
||||
case scanner.COMMENT:
|
||||
case token.COMMENT:
|
||||
// implement comment
|
||||
case scanner.EOF:
|
||||
case token.EOF:
|
||||
return nil, errEofToken
|
||||
}
|
||||
|
||||
@ -106,18 +108,18 @@ func (p *Parser) parseType() (Node, error) {
|
||||
}
|
||||
|
||||
// parseObjectKey parses an object key and returns a ObjectKey AST
|
||||
func (p *Parser) parseObjectKey() ([]*ObjectKey, error) {
|
||||
func (p *Parser) parseObjectKey() ([]*ast.ObjectKey, error) {
|
||||
tok := p.scan()
|
||||
if tok.Type == scanner.EOF {
|
||||
if tok.Type == token.EOF {
|
||||
return nil, errEofToken
|
||||
}
|
||||
|
||||
keys := make([]*ObjectKey, 0)
|
||||
keys := make([]*ast.ObjectKey, 0)
|
||||
|
||||
switch tok.Type {
|
||||
case scanner.IDENT, scanner.STRING:
|
||||
case token.IDENT, token.STRING:
|
||||
// add first found token
|
||||
keys = append(keys, &ObjectKey{token: tok})
|
||||
keys = append(keys, &ast.ObjectKey{Token: tok})
|
||||
default:
|
||||
return nil, fmt.Errorf("expected: IDENT | STRING got: %s", tok.Type)
|
||||
}
|
||||
@ -131,7 +133,7 @@ func (p *Parser) parseObjectKey() ([]*ObjectKey, error) {
|
||||
for {
|
||||
tok := p.scan()
|
||||
switch tok.Type {
|
||||
case scanner.ASSIGN:
|
||||
case token.ASSIGN:
|
||||
// assignment or object only, but not nested objects. this is not
|
||||
// allowed: `foo bar = {}`
|
||||
if nestedObj {
|
||||
@ -139,13 +141,13 @@ func (p *Parser) parseObjectKey() ([]*ObjectKey, error) {
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
case scanner.LBRACE:
|
||||
case token.LBRACE:
|
||||
// object
|
||||
return keys, nil
|
||||
case scanner.IDENT, scanner.STRING:
|
||||
case token.IDENT, token.STRING:
|
||||
// nested object
|
||||
nestedObj = true
|
||||
keys = append(keys, &ObjectKey{token: tok})
|
||||
keys = append(keys, &ast.ObjectKey{Token: tok})
|
||||
default:
|
||||
return nil, fmt.Errorf("expected: IDENT | STRING | ASSIGN | LBRACE got: %s", tok.Type)
|
||||
}
|
||||
@ -153,23 +155,23 @@ func (p *Parser) parseObjectKey() ([]*ObjectKey, error) {
|
||||
}
|
||||
|
||||
// parseLiteralType parses a literal type and returns a LiteralType AST
|
||||
func (p *Parser) parseLiteralType() (*LiteralType, error) {
|
||||
func (p *Parser) parseLiteralType() (*ast.LiteralType, error) {
|
||||
defer un(trace(p, "ParseLiteral"))
|
||||
|
||||
return &LiteralType{
|
||||
token: p.tok,
|
||||
return &ast.LiteralType{
|
||||
Token: p.tok,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// parseObjectType parses an object type and returns a ObjectType AST
|
||||
func (p *Parser) parseObjectType() (*ObjectType, error) {
|
||||
func (p *Parser) parseObjectType() (*ast.ObjectType, error) {
|
||||
defer un(trace(p, "ParseObjectYpe"))
|
||||
|
||||
return nil, errors.New("ObjectType is not implemented yet")
|
||||
}
|
||||
|
||||
// parseListType parses a list type and returns a ListType AST
|
||||
func (p *Parser) parseListType() (*ListType, error) {
|
||||
func (p *Parser) parseListType() (*ast.ListType, error) {
|
||||
defer un(trace(p, "ParseListType"))
|
||||
|
||||
return nil, errors.New("ListType is not implemented yet")
|
||||
@ -177,7 +179,7 @@ func (p *Parser) parseListType() (*ListType, error) {
|
||||
|
||||
// scan returns the next token from the underlying scanner.
|
||||
// If a token has been unscanned then read that instead.
|
||||
func (p *Parser) scan() scanner.Token {
|
||||
func (p *Parser) scan() token.Token {
|
||||
// If we have a token on the buffer, then return it.
|
||||
if p.n != 0 {
|
||||
p.n = 0
|
||||
|
||||
@ -7,7 +7,8 @@ import (
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/fatih/hcl/scanner"
|
||||
"github.com/fatih/hcl/ast"
|
||||
"github.com/fatih/hcl/token"
|
||||
)
|
||||
|
||||
func TestParseType(t *testing.T) {
|
||||
@ -22,7 +23,7 @@ func TestParseType(t *testing.T) {
|
||||
|
||||
fmt.Printf("n = %+v\n", n)
|
||||
|
||||
Walk(n, func(node Node) bool {
|
||||
ast.Walk(n, func(node ast.Node) bool {
|
||||
fmt.Printf("node = %+v\n", node)
|
||||
return true
|
||||
})
|
||||
@ -30,21 +31,21 @@ func TestParseType(t *testing.T) {
|
||||
|
||||
func TestObjectKey(t *testing.T) {
|
||||
keys := []struct {
|
||||
exp []scanner.TokenType
|
||||
exp []token.TokenType
|
||||
src string
|
||||
}{
|
||||
{[]scanner.TokenType{scanner.IDENT}, `foo {}`},
|
||||
{[]scanner.TokenType{scanner.IDENT}, `foo = {}`},
|
||||
{[]scanner.TokenType{scanner.IDENT}, `foo = bar`},
|
||||
{[]scanner.TokenType{scanner.IDENT}, `foo = 123`},
|
||||
{[]scanner.TokenType{scanner.IDENT}, `foo = "${var.bar}`},
|
||||
{[]scanner.TokenType{scanner.STRING}, `"foo" {}`},
|
||||
{[]scanner.TokenType{scanner.STRING}, `"foo" = {}`},
|
||||
{[]scanner.TokenType{scanner.STRING}, `"foo" = "${var.bar}`},
|
||||
{[]scanner.TokenType{scanner.IDENT, scanner.IDENT}, `foo bar {}`},
|
||||
{[]scanner.TokenType{scanner.IDENT, scanner.STRING}, `foo "bar" {}`},
|
||||
{[]scanner.TokenType{scanner.STRING, scanner.IDENT}, `"foo" bar {}`},
|
||||
{[]scanner.TokenType{scanner.IDENT, scanner.IDENT, scanner.IDENT}, `foo bar baz {}`},
|
||||
{[]token.TokenType{token.IDENT}, `foo {}`},
|
||||
{[]token.TokenType{token.IDENT}, `foo = {}`},
|
||||
{[]token.TokenType{token.IDENT}, `foo = bar`},
|
||||
{[]token.TokenType{token.IDENT}, `foo = 123`},
|
||||
{[]token.TokenType{token.IDENT}, `foo = "${var.bar}`},
|
||||
{[]token.TokenType{token.STRING}, `"foo" {}`},
|
||||
{[]token.TokenType{token.STRING}, `"foo" = {}`},
|
||||
{[]token.TokenType{token.STRING}, `"foo" = "${var.bar}`},
|
||||
{[]token.TokenType{token.IDENT, token.IDENT}, `foo bar {}`},
|
||||
{[]token.TokenType{token.IDENT, token.STRING}, `foo "bar" {}`},
|
||||
{[]token.TokenType{token.STRING, token.IDENT}, `"foo" bar {}`},
|
||||
{[]token.TokenType{token.IDENT, token.IDENT, token.IDENT}, `foo bar baz {}`},
|
||||
}
|
||||
|
||||
for _, k := range keys {
|
||||
@ -54,9 +55,9 @@ func TestObjectKey(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tokens := []scanner.TokenType{}
|
||||
tokens := []token.TokenType{}
|
||||
for _, o := range keys {
|
||||
tokens = append(tokens, o.token.Type)
|
||||
tokens = append(tokens, o.Token.Type)
|
||||
}
|
||||
|
||||
equals(t, k.exp, tokens)
|
||||
|
||||
@ -8,6 +8,8 @@ import (
|
||||
"os"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/fatih/hcl/token"
|
||||
)
|
||||
|
||||
// eof represents a marker rune for the end of the reader.
|
||||
@ -19,8 +21,8 @@ type Scanner struct {
|
||||
src []byte // Source buffer for immutable access
|
||||
|
||||
// Source Position
|
||||
srcPos Pos // current position
|
||||
prevPos Pos // previous position, used for peek() method
|
||||
srcPos token.Pos // current position
|
||||
prevPos token.Pos // previous position, used for peek() method
|
||||
|
||||
lastCharLen int // length of last character in bytes
|
||||
lastLineLen int // length of last line in characters (for correct column reporting)
|
||||
@ -30,7 +32,7 @@ type Scanner struct {
|
||||
|
||||
// Error is called for each error encountered. If no Error
|
||||
// function is set, the error is reported to os.Stderr.
|
||||
Error func(pos Pos, msg string)
|
||||
Error func(pos token.Pos, msg string)
|
||||
|
||||
// ErrorCount is incremented by one for each error encountered.
|
||||
ErrorCount int
|
||||
@ -39,7 +41,7 @@ type Scanner struct {
|
||||
// Scan. The Filename field is always left untouched by the Scanner. If
|
||||
// an error is reported (via Error) and Position is invalid, the scanner is
|
||||
// not inside a token.
|
||||
tokPos Pos
|
||||
tokPos token.Pos
|
||||
}
|
||||
|
||||
// New creates and initializes a new instance of Scanner using src as
|
||||
@ -117,7 +119,7 @@ func (s *Scanner) peek() rune {
|
||||
}
|
||||
|
||||
// Scan scans the next token and returns the token.
|
||||
func (s *Scanner) Scan() Token {
|
||||
func (s *Scanner) Scan() token.Token {
|
||||
ch := s.next()
|
||||
|
||||
// skip white space
|
||||
@ -125,7 +127,7 @@ func (s *Scanner) Scan() Token {
|
||||
ch = s.next()
|
||||
}
|
||||
|
||||
var tok TokenType
|
||||
var tok token.TokenType
|
||||
|
||||
// token text markings
|
||||
s.tokStart = s.srcPos.Offset - s.lastCharLen
|
||||
@ -147,47 +149,47 @@ func (s *Scanner) Scan() Token {
|
||||
|
||||
switch {
|
||||
case isLetter(ch):
|
||||
tok = IDENT
|
||||
tok = token.IDENT
|
||||
lit := s.scanIdentifier()
|
||||
if lit == "true" || lit == "false" {
|
||||
tok = BOOL
|
||||
tok = token.BOOL
|
||||
}
|
||||
case isDecimal(ch):
|
||||
tok = s.scanNumber(ch)
|
||||
default:
|
||||
switch ch {
|
||||
case eof:
|
||||
tok = EOF
|
||||
tok = token.EOF
|
||||
case '"':
|
||||
tok = STRING
|
||||
tok = token.STRING
|
||||
s.scanString()
|
||||
case '#', '/':
|
||||
tok = COMMENT
|
||||
tok = token.COMMENT
|
||||
s.scanComment(ch)
|
||||
case '.':
|
||||
tok = PERIOD
|
||||
tok = token.PERIOD
|
||||
ch = s.peek()
|
||||
if isDecimal(ch) {
|
||||
tok = FLOAT
|
||||
tok = token.FLOAT
|
||||
ch = s.scanMantissa(ch)
|
||||
ch = s.scanExponent(ch)
|
||||
}
|
||||
case '[':
|
||||
tok = LBRACK
|
||||
tok = token.LBRACK
|
||||
case ']':
|
||||
tok = RBRACK
|
||||
tok = token.RBRACK
|
||||
case '{':
|
||||
tok = LBRACE
|
||||
tok = token.LBRACE
|
||||
case '}':
|
||||
tok = RBRACE
|
||||
tok = token.RBRACE
|
||||
case ',':
|
||||
tok = COMMA
|
||||
tok = token.COMMA
|
||||
case '=':
|
||||
tok = ASSIGN
|
||||
tok = token.ASSIGN
|
||||
case '+':
|
||||
tok = ADD
|
||||
tok = token.ADD
|
||||
case '-':
|
||||
tok = SUB
|
||||
tok = token.SUB
|
||||
default:
|
||||
s.err("illegal char")
|
||||
}
|
||||
@ -203,7 +205,7 @@ func (s *Scanner) Scan() Token {
|
||||
}
|
||||
s.tokStart = s.tokEnd // ensure idempotency of tokenText() call
|
||||
|
||||
return Token{
|
||||
return token.Token{
|
||||
Type: tok,
|
||||
Pos: s.tokPos,
|
||||
Text: tokenText,
|
||||
@ -244,7 +246,7 @@ func (s *Scanner) scanComment(ch rune) {
|
||||
}
|
||||
|
||||
// scanNumber scans a HCL number definition starting with the given rune
|
||||
func (s *Scanner) scanNumber(ch rune) TokenType {
|
||||
func (s *Scanner) scanNumber(ch rune) token.TokenType {
|
||||
if ch == '0' {
|
||||
// check for hexadecimal, octal or float
|
||||
ch = s.next()
|
||||
@ -265,7 +267,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType {
|
||||
s.unread()
|
||||
}
|
||||
|
||||
return NUMBER
|
||||
return token.NUMBER
|
||||
}
|
||||
|
||||
// now it's either something like: 0421(octal) or 0.1231(float)
|
||||
@ -283,7 +285,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType {
|
||||
// literals of form 01e10 are treates as Numbers in HCL, which differs from Go.
|
||||
if ch == 'e' || ch == 'E' {
|
||||
ch = s.scanExponent(ch)
|
||||
return NUMBER
|
||||
return token.NUMBER
|
||||
}
|
||||
|
||||
if ch == '.' {
|
||||
@ -293,7 +295,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType {
|
||||
ch = s.next()
|
||||
ch = s.scanExponent(ch)
|
||||
}
|
||||
return FLOAT
|
||||
return token.FLOAT
|
||||
}
|
||||
|
||||
if illegalOctal {
|
||||
@ -303,7 +305,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType {
|
||||
if ch != eof {
|
||||
s.unread()
|
||||
}
|
||||
return NUMBER
|
||||
return token.NUMBER
|
||||
}
|
||||
|
||||
s.scanMantissa(ch)
|
||||
@ -311,7 +313,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType {
|
||||
// literals of form 1e10 are treates as Numbers in HCL, which differs from Go.
|
||||
if ch == 'e' || ch == 'E' {
|
||||
ch = s.scanExponent(ch)
|
||||
return NUMBER
|
||||
return token.NUMBER
|
||||
}
|
||||
|
||||
if ch == '.' {
|
||||
@ -320,11 +322,11 @@ func (s *Scanner) scanNumber(ch rune) TokenType {
|
||||
ch = s.next()
|
||||
ch = s.scanExponent(ch)
|
||||
}
|
||||
return FLOAT
|
||||
return token.FLOAT
|
||||
}
|
||||
|
||||
s.unread()
|
||||
return NUMBER
|
||||
return token.NUMBER
|
||||
}
|
||||
|
||||
// scanMantissa scans the mantissa begining from the rune. It returns the next
|
||||
@ -446,7 +448,7 @@ func (s *Scanner) scanIdentifier() string {
|
||||
|
||||
// recentPosition returns the position of the character immediately after the
|
||||
// character or token returned by the last call to Scan.
|
||||
func (s *Scanner) recentPosition() (pos Pos) {
|
||||
func (s *Scanner) recentPosition() (pos token.Pos) {
|
||||
pos.Offset = s.srcPos.Offset - s.lastCharLen
|
||||
switch {
|
||||
case s.srcPos.Column > 0:
|
||||
|
||||
@ -4,161 +4,163 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/fatih/hcl/token"
|
||||
)
|
||||
|
||||
var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
|
||||
|
||||
type tokenPair struct {
|
||||
tok TokenType
|
||||
tok token.TokenType
|
||||
text string
|
||||
}
|
||||
|
||||
var tokenLists = map[string][]tokenPair{
|
||||
"comment": []tokenPair{
|
||||
{COMMENT, "//"},
|
||||
{COMMENT, "////"},
|
||||
{COMMENT, "// comment"},
|
||||
{COMMENT, "// /* comment */"},
|
||||
{COMMENT, "// // comment //"},
|
||||
{COMMENT, "//" + f100},
|
||||
{COMMENT, "#"},
|
||||
{COMMENT, "##"},
|
||||
{COMMENT, "# comment"},
|
||||
{COMMENT, "# /* comment */"},
|
||||
{COMMENT, "# # comment #"},
|
||||
{COMMENT, "#" + f100},
|
||||
{COMMENT, "/**/"},
|
||||
{COMMENT, "/***/"},
|
||||
{COMMENT, "/* comment */"},
|
||||
{COMMENT, "/* // comment */"},
|
||||
{COMMENT, "/* /* comment */"},
|
||||
{COMMENT, "/*\n comment\n*/"},
|
||||
{COMMENT, "/*" + f100 + "*/"},
|
||||
{token.COMMENT, "//"},
|
||||
{token.COMMENT, "////"},
|
||||
{token.COMMENT, "// comment"},
|
||||
{token.COMMENT, "// /* comment */"},
|
||||
{token.COMMENT, "// // comment //"},
|
||||
{token.COMMENT, "//" + f100},
|
||||
{token.COMMENT, "#"},
|
||||
{token.COMMENT, "##"},
|
||||
{token.COMMENT, "# comment"},
|
||||
{token.COMMENT, "# /* comment */"},
|
||||
{token.COMMENT, "# # comment #"},
|
||||
{token.COMMENT, "#" + f100},
|
||||
{token.COMMENT, "/**/"},
|
||||
{token.COMMENT, "/***/"},
|
||||
{token.COMMENT, "/* comment */"},
|
||||
{token.COMMENT, "/* // comment */"},
|
||||
{token.COMMENT, "/* /* comment */"},
|
||||
{token.COMMENT, "/*\n comment\n*/"},
|
||||
{token.COMMENT, "/*" + f100 + "*/"},
|
||||
},
|
||||
"operator": []tokenPair{
|
||||
{LBRACK, "["},
|
||||
{LBRACE, "{"},
|
||||
{COMMA, ","},
|
||||
{PERIOD, "."},
|
||||
{RBRACK, "]"},
|
||||
{RBRACE, "}"},
|
||||
{ASSIGN, "="},
|
||||
{ADD, "+"},
|
||||
{SUB, "-"},
|
||||
{token.LBRACK, "["},
|
||||
{token.LBRACE, "{"},
|
||||
{token.COMMA, ","},
|
||||
{token.PERIOD, "."},
|
||||
{token.RBRACK, "]"},
|
||||
{token.RBRACE, "}"},
|
||||
{token.ASSIGN, "="},
|
||||
{token.ADD, "+"},
|
||||
{token.SUB, "-"},
|
||||
},
|
||||
"bool": []tokenPair{
|
||||
{BOOL, "true"},
|
||||
{BOOL, "false"},
|
||||
{token.BOOL, "true"},
|
||||
{token.BOOL, "false"},
|
||||
},
|
||||
"ident": []tokenPair{
|
||||
{IDENT, "a"},
|
||||
{IDENT, "a0"},
|
||||
{IDENT, "foobar"},
|
||||
{IDENT, "abc123"},
|
||||
{IDENT, "LGTM"},
|
||||
{IDENT, "_"},
|
||||
{IDENT, "_abc123"},
|
||||
{IDENT, "abc123_"},
|
||||
{IDENT, "_abc_123_"},
|
||||
{IDENT, "_äöü"},
|
||||
{IDENT, "_本"},
|
||||
{IDENT, "äöü"},
|
||||
{IDENT, "本"},
|
||||
{IDENT, "a۰۱۸"},
|
||||
{IDENT, "foo६४"},
|
||||
{IDENT, "bar９８７６"},
|
||||
"identoken.t": []tokenPair{
|
||||
{token.IDENT, "a"},
|
||||
{token.IDENT, "a0"},
|
||||
{token.IDENT, "foobar"},
|
||||
{token.IDENT, "abc123"},
|
||||
{token.IDENT, "LGTM"},
|
||||
{token.IDENT, "_"},
|
||||
{token.IDENT, "_abc123"},
|
||||
{token.IDENT, "abc123_"},
|
||||
{token.IDENT, "_abc_123_"},
|
||||
{token.IDENT, "_äöü"},
|
||||
{token.IDENT, "_本"},
|
||||
{token.IDENT, "äöü"},
|
||||
{token.IDENT, "本"},
|
||||
{token.IDENT, "a۰۱۸"},
|
||||
{token.IDENT, "foo६४"},
|
||||
{token.IDENT, "bar９８７６"},
|
||||
},
|
||||
"string": []tokenPair{
|
||||
{STRING, `" "`},
|
||||
{STRING, `"a"`},
|
||||
{STRING, `"本"`},
|
||||
{STRING, `"\a"`},
|
||||
{STRING, `"\b"`},
|
||||
{STRING, `"\f"`},
|
||||
{STRING, `"\n"`},
|
||||
{STRING, `"\r"`},
|
||||
{STRING, `"\t"`},
|
||||
{STRING, `"\v"`},
|
||||
{STRING, `"\""`},
|
||||
{STRING, `"\000"`},
|
||||
{STRING, `"\777"`},
|
||||
{STRING, `"\x00"`},
|
||||
{STRING, `"\xff"`},
|
||||
{STRING, `"\u0000"`},
|
||||
{STRING, `"\ufA16"`},
|
||||
{STRING, `"\U00000000"`},
|
||||
{STRING, `"\U0000ffAB"`},
|
||||
{STRING, `"` + f100 + `"`},
|
||||
"stritoken.ng": []tokenPair{
|
||||
{token.STRING, `" "`},
|
||||
{token.STRING, `"a"`},
|
||||
{token.STRING, `"本"`},
|
||||
{token.STRING, `"\a"`},
|
||||
{token.STRING, `"\b"`},
|
||||
{token.STRING, `"\f"`},
|
||||
{token.STRING, `"\n"`},
|
||||
{token.STRING, `"\r"`},
|
||||
{token.STRING, `"\t"`},
|
||||
{token.STRING, `"\v"`},
|
||||
{token.STRING, `"\""`},
|
||||
{token.STRING, `"\000"`},
|
||||
{token.STRING, `"\777"`},
|
||||
{token.STRING, `"\x00"`},
|
||||
{token.STRING, `"\xff"`},
|
||||
{token.STRING, `"\u0000"`},
|
||||
{token.STRING, `"\ufA16"`},
|
||||
{token.STRING, `"\U00000000"`},
|
||||
{token.STRING, `"\U0000ffAB"`},
|
||||
{token.STRING, `"` + f100 + `"`},
|
||||
},
|
||||
"number": []tokenPair{
|
||||
{NUMBER, "0"},
|
||||
{NUMBER, "1"},
|
||||
{NUMBER, "9"},
|
||||
{NUMBER, "42"},
|
||||
{NUMBER, "1234567890"},
|
||||
{NUMBER, "00"},
|
||||
{NUMBER, "01"},
|
||||
{NUMBER, "07"},
|
||||
{NUMBER, "042"},
|
||||
{NUMBER, "01234567"},
|
||||
{NUMBER, "0x0"},
|
||||
{NUMBER, "0x1"},
|
||||
{NUMBER, "0xf"},
|
||||
{NUMBER, "0x42"},
|
||||
{NUMBER, "0x123456789abcDEF"},
|
||||
{NUMBER, "0x" + f100},
|
||||
{NUMBER, "0X0"},
|
||||
{NUMBER, "0X1"},
|
||||
{NUMBER, "0XF"},
|
||||
{NUMBER, "0X42"},
|
||||
{NUMBER, "0X123456789abcDEF"},
|
||||
{NUMBER, "0X" + f100},
|
||||
{NUMBER, "0e0"},
|
||||
{NUMBER, "1e0"},
|
||||
{NUMBER, "42e0"},
|
||||
{NUMBER, "01234567890e0"},
|
||||
{NUMBER, "0E0"},
|
||||
{NUMBER, "1E0"},
|
||||
{NUMBER, "42E0"},
|
||||
{NUMBER, "01234567890E0"},
|
||||
{NUMBER, "0e+10"},
|
||||
{NUMBER, "1e-10"},
|
||||
{NUMBER, "42e+10"},
|
||||
{NUMBER, "01234567890e-10"},
|
||||
{NUMBER, "0E+10"},
|
||||
{NUMBER, "1E-10"},
|
||||
{NUMBER, "42E+10"},
|
||||
{NUMBER, "01234567890E-10"},
|
||||
"numbtoken.er": []tokenPair{
|
||||
{token.NUMBER, "0"},
|
||||
{token.NUMBER, "1"},
|
||||
{token.NUMBER, "9"},
|
||||
{token.NUMBER, "42"},
|
||||
{token.NUMBER, "1234567890"},
|
||||
{token.NUMBER, "00"},
|
||||
{token.NUMBER, "01"},
|
||||
{token.NUMBER, "07"},
|
||||
{token.NUMBER, "042"},
|
||||
{token.NUMBER, "01234567"},
|
||||
{token.NUMBER, "0x0"},
|
||||
{token.NUMBER, "0x1"},
|
||||
{token.NUMBER, "0xf"},
|
||||
{token.NUMBER, "0x42"},
|
||||
{token.NUMBER, "0x123456789abcDEF"},
|
||||
{token.NUMBER, "0x" + f100},
|
||||
{token.NUMBER, "0X0"},
|
||||
{token.NUMBER, "0X1"},
|
||||
{token.NUMBER, "0XF"},
|
||||
{token.NUMBER, "0X42"},
|
||||
{token.NUMBER, "0X123456789abcDEF"},
|
||||
{token.NUMBER, "0X" + f100},
|
||||
{token.NUMBER, "0e0"},
|
||||
{token.NUMBER, "1e0"},
|
||||
{token.NUMBER, "42e0"},
|
||||
{token.NUMBER, "01234567890e0"},
|
||||
{token.NUMBER, "0E0"},
|
||||
{token.NUMBER, "1E0"},
|
||||
{token.NUMBER, "42E0"},
|
||||
{token.NUMBER, "01234567890E0"},
|
||||
{token.NUMBER, "0e+10"},
|
||||
{token.NUMBER, "1e-10"},
|
||||
{token.NUMBER, "42e+10"},
|
||||
{token.NUMBER, "01234567890e-10"},
|
||||
{token.NUMBER, "0E+10"},
|
||||
{token.NUMBER, "1E-10"},
|
||||
{token.NUMBER, "42E+10"},
|
||||
{token.NUMBER, "01234567890E-10"},
|
||||
},
|
||||
"float": []tokenPair{
|
||||
{FLOAT, "0."},
|
||||
{FLOAT, "1."},
|
||||
{FLOAT, "42."},
|
||||
{FLOAT, "01234567890."},
|
||||
{FLOAT, ".0"},
|
||||
{FLOAT, ".1"},
|
||||
{FLOAT, ".42"},
|
||||
{FLOAT, ".0123456789"},
|
||||
{FLOAT, "0.0"},
|
||||
{FLOAT, "1.0"},
|
||||
{FLOAT, "42.0"},
|
||||
{FLOAT, "01234567890.0"},
|
||||
{FLOAT, "01.8e0"},
|
||||
{FLOAT, "1.4e0"},
|
||||
{FLOAT, "42.2e0"},
|
||||
{FLOAT, "01234567890.12e0"},
|
||||
{FLOAT, "0.E0"},
|
||||
{FLOAT, "1.12E0"},
|
||||
{FLOAT, "42.123E0"},
|
||||
{FLOAT, "01234567890.213E0"},
|
||||
{FLOAT, "0.2e+10"},
|
||||
{FLOAT, "1.2e-10"},
|
||||
{FLOAT, "42.54e+10"},
|
||||
{FLOAT, "01234567890.98e-10"},
|
||||
{FLOAT, "0.1E+10"},
|
||||
{FLOAT, "1.1E-10"},
|
||||
{FLOAT, "42.1E+10"},
|
||||
{FLOAT, "01234567890.1E-10"},
|
||||
"floatoken.t": []tokenPair{
|
||||
{token.FLOAT, "0."},
|
||||
{token.FLOAT, "1."},
|
||||
{token.FLOAT, "42."},
|
||||
{token.FLOAT, "01234567890."},
|
||||
{token.FLOAT, ".0"},
|
||||
{token.FLOAT, ".1"},
|
||||
{token.FLOAT, ".42"},
|
||||
{token.FLOAT, ".0123456789"},
|
||||
{token.FLOAT, "0.0"},
|
||||
{token.FLOAT, "1.0"},
|
||||
{token.FLOAT, "42.0"},
|
||||
{token.FLOAT, "01234567890.0"},
|
||||
{token.FLOAT, "01.8e0"},
|
||||
{token.FLOAT, "1.4e0"},
|
||||
{token.FLOAT, "42.2e0"},
|
||||
{token.FLOAT, "01234567890.12e0"},
|
||||
{token.FLOAT, "0.E0"},
|
||||
{token.FLOAT, "1.12E0"},
|
||||
{token.FLOAT, "42.123E0"},
|
||||
{token.FLOAT, "01234567890.213E0"},
|
||||
{token.FLOAT, "0.2e+10"},
|
||||
{token.FLOAT, "1.2e-10"},
|
||||
{token.FLOAT, "42.54e+10"},
|
||||
{token.FLOAT, "01234567890.98e-10"},
|
||||
{token.FLOAT, "0.1E+10"},
|
||||
{token.FLOAT, "1.1E-10"},
|
||||
{token.FLOAT, "42.1E+10"},
|
||||
{token.FLOAT, "01234567890.1E-10"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -184,7 +186,7 @@ func TestPosition(t *testing.T) {
|
||||
|
||||
s := New(buf.Bytes())
|
||||
|
||||
pos := Pos{"", 4, 1, 5}
|
||||
pos := token.Pos{"", 4, 1, 5}
|
||||
s.Scan()
|
||||
for _, listName := range orderedTokenLists {
|
||||
|
||||
@ -270,63 +272,63 @@ func TestRealExample(t *testing.T) {
|
||||
}`
|
||||
|
||||
literals := []struct {
|
||||
tokenType TokenType
|
||||
tokenType token.TokenType
|
||||
literal string
|
||||
}{
|
||||
{COMMENT, `// This comes from Terraform, as a test`},
|
||||
{IDENT, `variable`},
|
||||
{STRING, `"foo"`},
|
||||
{LBRACE, `{`},
|
||||
{IDENT, `default`},
|
||||
{ASSIGN, `=`},
|
||||
{STRING, `"bar"`},
|
||||
{IDENT, `description`},
|
||||
{ASSIGN, `=`},
|
||||
{STRING, `"bar"`},
|
||||
{RBRACE, `}`},
|
||||
{IDENT, `provider`},
|
||||
{STRING, `"aws"`},
|
||||
{LBRACE, `{`},
|
||||
{IDENT, `access_key`},
|
||||
{ASSIGN, `=`},
|
||||
{STRING, `"foo"`},
|
||||
{IDENT, `secret_key`},
|
||||
{ASSIGN, `=`},
|
||||
{STRING, `"bar"`},
|
||||
{RBRACE, `}`},
|
||||
{IDENT, `resource`},
|
||||
{STRING, `"aws_security_group"`},
|
||||
{STRING, `"firewall"`},
|
||||
{LBRACE, `{`},
|
||||
{IDENT, `count`},
|
||||
{ASSIGN, `=`},
|
||||
{NUMBER, `5`},
|
||||
{RBRACE, `}`},
|
||||
{IDENT, `resource`},
|
||||
{IDENT, `aws_instance`},
|
||||
{STRING, `"web"`},
|
||||
{LBRACE, `{`},
|
||||
{IDENT, `ami`},
|
||||
{ASSIGN, `=`},
|
||||
{STRING, `"${var.foo}"`},
|
||||
{IDENT, `security_groups`},
|
||||
{ASSIGN, `=`},
|
||||
{LBRACK, `[`},
|
||||
{STRING, `"foo"`},
|
||||
{COMMA, `,`},
|
||||
{STRING, `"${aws_security_group.firewall.foo}"`},
|
||||
{RBRACK, `]`},
|
||||
{IDENT, `network_interface`},
|
||||
{LBRACE, `{`},
|
||||
{IDENT, `device_index`},
|
||||
{ASSIGN, `=`},
|
||||
{NUMBER, `0`},
|
||||
{IDENT, `description`},
|
||||
{ASSIGN, `=`},
|
||||
{STRING, `"Main network interface"`},
|
||||
{RBRACE, `}`},
|
||||
{RBRACE, `}`},
|
||||
{EOF, ``},
|
||||
{token.COMMENT, `// This comes from Terraform, as a test`},
|
||||
{token.IDENT, `variable`},
|
||||
{token.STRING, `"foo"`},
|
||||
{token.LBRACE, `{`},
|
||||
{token.IDENT, `default`},
|
||||
{token.ASSIGN, `=`},
|
||||
{token.STRING, `"bar"`},
|
||||
{token.IDENT, `description`},
|
||||
{token.ASSIGN, `=`},
|
||||
{token.STRING, `"bar"`},
|
||||
{token.RBRACE, `}`},
|
||||
{token.IDENT, `provider`},
|
||||
{token.STRING, `"aws"`},
|
||||
{token.LBRACE, `{`},
|
||||
{token.IDENT, `access_key`},
|
||||
{token.ASSIGN, `=`},
|
||||
{token.STRING, `"foo"`},
|
||||
{token.IDENT, `secret_key`},
|
||||
{token.ASSIGN, `=`},
|
||||
{token.STRING, `"bar"`},
|
||||
{token.RBRACE, `}`},
|
||||
{token.IDENT, `resource`},
|
||||
{token.STRING, `"aws_security_group"`},
|
||||
{token.STRING, `"firewall"`},
|
||||
{token.LBRACE, `{`},
|
||||
{token.IDENT, `count`},
|
||||
{token.ASSIGN, `=`},
|
||||
{token.NUMBER, `5`},
|
||||
{token.RBRACE, `}`},
|
||||
{token.IDENT, `resource`},
|
||||
{token.IDENT, `aws_instance`},
|
||||
{token.STRING, `"web"`},
|
||||
{token.LBRACE, `{`},
|
||||
{token.IDENT, `ami`},
|
||||
{token.ASSIGN, `=`},
|
||||
{token.STRING, `"${var.foo}"`},
|
||||
{token.IDENT, `security_groups`},
|
||||
{token.ASSIGN, `=`},
|
||||
{token.LBRACK, `[`},
|
||||
{token.STRING, `"foo"`},
|
||||
{token.COMMA, `,`},
|
||||
{token.STRING, `"${aws_security_group.firewall.foo}"`},
|
||||
{token.RBRACK, `]`},
|
||||
{token.IDENT, `network_interface`},
|
||||
{token.LBRACE, `{`},
|
||||
{token.IDENT, `device_index`},
|
||||
{token.ASSIGN, `=`},
|
||||
{token.NUMBER, `0`},
|
||||
{token.IDENT, `description`},
|
||||
{token.ASSIGN, `=`},
|
||||
{token.STRING, `"Main network interface"`},
|
||||
{token.RBRACE, `}`},
|
||||
{token.RBRACE, `}`},
|
||||
{token.EOF, ``},
|
||||
}
|
||||
|
||||
s := New([]byte(complexHCL))
|
||||
@ -344,32 +346,32 @@ func TestRealExample(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestError(t *testing.T) {
|
||||
testError(t, "\x80", "1:1", "illegal UTF-8 encoding", ILLEGAL)
|
||||
testError(t, "\xff", "1:1", "illegal UTF-8 encoding", ILLEGAL)
|
||||
testError(t, "\x80", "1:1", "illegal UTF-8 encoding", token.ILLEGAL)
|
||||
testError(t, "\xff", "1:1", "illegal UTF-8 encoding", token.ILLEGAL)
|
||||
|
||||
testError(t, "ab\x80", "1:3", "illegal UTF-8 encoding", IDENT)
|
||||
testError(t, "abc\xff", "1:4", "illegal UTF-8 encoding", IDENT)
|
||||
testError(t, "ab\x80", "1:3", "illegal UTF-8 encoding", token.IDENT)
|
||||
testError(t, "abc\xff", "1:4", "illegal UTF-8 encoding", token.IDENT)
|
||||
|
||||
testError(t, `"ab`+"\x80", "1:4", "illegal UTF-8 encoding", STRING)
|
||||
testError(t, `"abc`+"\xff", "1:5", "illegal UTF-8 encoding", STRING)
|
||||
testError(t, `"ab`+"\x80", "1:4", "illegal UTF-8 encoding", token.STRING)
|
||||
testError(t, `"abc`+"\xff", "1:5", "illegal UTF-8 encoding", token.STRING)
|
||||
|
||||
testError(t, `01238`, "1:6", "illegal octal number", NUMBER)
|
||||
testError(t, `01238123`, "1:9", "illegal octal number", NUMBER)
|
||||
testError(t, `0x`, "1:3", "illegal hexadecimal number", NUMBER)
|
||||
testError(t, `0xg`, "1:3", "illegal hexadecimal number", NUMBER)
|
||||
testError(t, `'aa'`, "1:1", "illegal char", ILLEGAL)
|
||||
testError(t, `01238`, "1:6", "illegal octal number", token.NUMBER)
|
||||
testError(t, `01238123`, "1:9", "illegal octal number", token.NUMBER)
|
||||
testError(t, `0x`, "1:3", "illegal hexadecimal number", token.NUMBER)
|
||||
testError(t, `0xg`, "1:3", "illegal hexadecimal number", token.NUMBER)
|
||||
testError(t, `'aa'`, "1:1", "illegal char", token.ILLEGAL)
|
||||
|
||||
testError(t, `"`, "1:2", "literal not terminated", STRING)
|
||||
testError(t, `"abc`, "1:5", "literal not terminated", STRING)
|
||||
testError(t, `"abc`+"\n", "1:5", "literal not terminated", STRING)
|
||||
testError(t, `/*/`, "1:4", "comment not terminated", COMMENT)
|
||||
testError(t, `"`, "1:2", "literal not terminated", token.STRING)
|
||||
testError(t, `"abc`, "1:5", "literal not terminated", token.STRING)
|
||||
testError(t, `"abc`+"\n", "1:5", "literal not terminated", token.STRING)
|
||||
testError(t, `/*/`, "1:4", "comment not terminated", token.COMMENT)
|
||||
}
|
||||
|
||||
func testError(t *testing.T, src, pos, msg string, tok TokenType) {
|
||||
func testError(t *testing.T, src, pos, msg string, tok token.TokenType) {
|
||||
s := New([]byte(src))
|
||||
|
||||
errorCalled := false
|
||||
s.Error = func(p Pos, m string) {
|
||||
s.Error = func(p token.Pos, m string) {
|
||||
if !errorCalled {
|
||||
if pos != p.String() {
|
||||
t.Errorf("pos = %q, want %q for %q", p, pos, src)
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package scanner
|
||||
package token
|
||||
|
||||
import "fmt"
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package scanner
|
||||
package token
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@ -1,4 +1,4 @@
|
||||
package scanner
|
||||
package token
|
||||
|
||||
import "testing"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user