From e93a8e97ca78bc183efe2da89114cbd29a63a1e8 Mon Sep 17 00:00:00 2001 From: Fatih Arslan Date: Fri, 16 Oct 2015 23:12:26 +0300 Subject: [PATCH] hcl: split into meaningful packages --- ast/ast.go | 94 +++++++ {parser => ast}/walk.go | 12 +- parser/ast.go | 116 --------- parser/parser.go | 68 ++--- parser/parser_test.go | 35 +-- scanner/scanner.go | 64 ++--- scanner/scanner_test.go | 422 ++++++++++++++++--------------- {scanner => token}/position.go | 2 +- {scanner => token}/token.go | 2 +- {scanner => token}/token_test.go | 2 +- 10 files changed, 401 insertions(+), 416 deletions(-) create mode 100644 ast/ast.go rename {parser => ast}/walk.go (78%) delete mode 100644 parser/ast.go rename {scanner => token}/position.go (98%) rename {scanner => token}/token.go (99%) rename {scanner => token}/token_test.go (97%) diff --git a/ast/ast.go b/ast/ast.go new file mode 100644 index 0000000..9c8a01e --- /dev/null +++ b/ast/ast.go @@ -0,0 +1,94 @@ +package ast + +import "github.com/fatih/hcl/token" + +// Node is an element in the abstract syntax tree. +type Node interface { + node() + Pos() token.Pos +} + +func (ObjectList) node() {} +func (ObjectKey) node() {} +func (ObjectItem) node() {} + +func (ObjectType) node() {} +func (LiteralType) node() {} +func (ListType) node() {} + +// ObjectList represents a list of ObjectItems. An HCL file itself is an +// ObjectList. +type ObjectList struct { + Items []*ObjectItem +} + +func (o *ObjectList) Add(item *ObjectItem) { + o.Items = append(o.Items, item) +} + +func (o *ObjectList) Pos() token.Pos { + // always returns the uninitiliazed position + return o.Items[0].Pos() +} + +// ObjectItem represents a HCL Object Item. An item is represented with a key +// (or keys). It can be an assignment or an object (both normal and nested) +type ObjectItem struct { + // keys is only one length long if it's of type assignment. If it's a + // nested object it can be larger than one. In that case "assign" is + // invalid as there is no assignments for a nested object. + Keys []*ObjectKey + + // assign contains the position of "=", if any + Assign token.Pos + + // val is the item itself. It can be an object,list, number, bool or a + // string. If key length is larger than one, val can be only of type + // Object. + Val Node +} + +func (o *ObjectItem) Pos() token.Pos { + return o.Keys[0].Pos() +} + +// ObjectKeys are either an identifier or of type string. +type ObjectKey struct { + Token token.Token +} + +func (o *ObjectKey) Pos() token.Pos { + return o.Token.Pos +} + +// LiteralType represents a literal of basic type. Valid types are: +// token.NUMBER, token.FLOAT, token.BOOL and token.STRING +type LiteralType struct { + Token token.Token +} + +func (l *LiteralType) Pos() token.Pos { + return l.Token.Pos +} + +// ListStatement represents a HCL List type +type ListType struct { + Lbrack token.Pos // position of "[" + Rbrack token.Pos // position of "]" + List []Node // the elements in lexical order +} + +func (l *ListType) Pos() token.Pos { + return l.Lbrack +} + +// ObjectType represents a HCL Object Type +type ObjectType struct { + Lbrace token.Pos // position of "{" + Rbrace token.Pos // position of "}" + List []Node // the nodes in lexical order +} + +func (b *ObjectType) Pos() token.Pos { + return b.Lbrace +} diff --git a/parser/walk.go b/ast/walk.go similarity index 78% rename from parser/walk.go rename to ast/walk.go index 2bb9ed5..c015a67 100644 --- a/parser/walk.go +++ b/ast/walk.go @@ -1,4 +1,4 @@ -package parser +package ast // Walk traverses an AST in depth-first order: It starts by calling fn(node); // node must not be nil. If f returns true, Walk invokes f recursively for @@ -10,24 +10,24 @@ func Walk(node Node, fn func(Node) bool) { switch n := node.(type) { case *ObjectList: - for _, item := range n.items { + for _, item := range n.Items { Walk(item, fn) } case *ObjectKey: // nothing to do case *ObjectItem: - for _, k := range n.keys { + for _, k := range n.Keys { Walk(k, fn) } - Walk(n.val, fn) + Walk(n.Val, fn) case *LiteralType: // nothing to do case *ListType: - for _, l := range n.list { + for _, l := range n.List { Walk(l, fn) } case *ObjectType: - for _, l := range n.list { + for _, l := range n.List { Walk(l, fn) } } diff --git a/parser/ast.go b/parser/ast.go deleted file mode 100644 index 1ff158c..0000000 --- a/parser/ast.go +++ /dev/null @@ -1,116 +0,0 @@ -package parser - -import "github.com/fatih/hcl/scanner" - -// Node is an element in the abstract syntax tree. -type Node interface { - node() - Pos() scanner.Pos -} - -func (ObjectList) node() {} -func (ObjectKey) node() {} -func (ObjectItem) node() {} - -func (ObjectType) node() {} -func (LiteralType) node() {} -func (ListType) node() {} - -// ObjectList represents a list of ObjectItems. An HCL file itself is an -// ObjectList. -type ObjectList struct { - items []*ObjectItem -} - -func (o *ObjectList) add(item *ObjectItem) { - o.items = append(o.items, item) -} - -func (o *ObjectList) Pos() scanner.Pos { - // always returns the uninitiliazed position - return o.items[0].Pos() -} - -// ObjectItem represents a HCL Object Item. An item is represented with a key -// (or keys). It can be an assignment or an object (both normal and nested) -type ObjectItem struct { - // keys is only one length long if it's of type assignment. If it's a - // nested object it can be larger than one. In that case "assign" is - // invalid as there is no assignments for a nested object. - keys []*ObjectKey - - // assign contains the position of "=", if any - assign scanner.Pos - - // val is the item itself. It can be an object,list, number, bool or a - // string. If key length is larger than one, val can be only of type - // Object. - val Node -} - -func (o *ObjectItem) Pos() scanner.Pos { - return o.keys[0].Pos() -} - -// ObjectKeys are either an identifier or of type string. -type ObjectKey struct { - token scanner.Token -} - -func (o *ObjectKey) Pos() scanner.Pos { - return o.token.Pos -} - -// isValid() returns true if the underlying identifier satisfies one of the -// valid types (IDENT or STRING) -func (o *ObjectKey) isValid() bool { - switch o.token.Type { - case scanner.IDENT, scanner.STRING: - return true - default: - return false - } -} - -// LiteralType represents a literal of basic type. Valid types are: -// scanner.NUMBER, scanner.FLOAT, scanner.BOOL and scanner.STRING -type LiteralType struct { - token scanner.Token -} - -// isValid() returns true if the underlying identifier satisfies one of the -// valid types. -func (l *LiteralType) isValid() bool { - switch l.token.Type { - case scanner.NUMBER, scanner.FLOAT, scanner.BOOL, scanner.STRING: - return true - default: - return false - } -} - -func (l *LiteralType) Pos() scanner.Pos { - return l.token.Pos -} - -// ListStatement represents a HCL List type -type ListType struct { - lbrack scanner.Pos // position of "[" - rbrack scanner.Pos // position of "]" - list []Node // the elements in lexical order -} - -func (l *ListType) Pos() scanner.Pos { - return l.lbrack -} - -// ObjectType represents a HCL Object Type -type ObjectType struct { - lbrace scanner.Pos // position of "{" - rbrace scanner.Pos // position of "}" - list []Node // the nodes in lexical order -} - -func (b *ObjectType) Pos() scanner.Pos { - return b.lbrace -} diff --git a/parser/parser.go b/parser/parser.go index 8aafa1e..3fea2a9 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -4,14 +4,16 @@ import ( "errors" "fmt" + "github.com/fatih/hcl/ast" "github.com/fatih/hcl/scanner" + "github.com/fatih/hcl/token" ) type Parser struct { sc *scanner.Scanner - tok scanner.Token // last read token - prevTok scanner.Token // previous read token + tok token.Token // last read token + prevTok token.Token // previous read token enableTrace bool indent int @@ -27,9 +29,9 @@ func New(src []byte) *Parser { var errEofToken = errors.New("EOF token found") // Parse returns the fully parsed source and returns the abstract syntax tree. -func (p *Parser) Parse() (Node, error) { +func (p *Parser) Parse() (ast.Node, error) { defer un(trace(p, "ParseObjectList")) - node := &ObjectList{} + node := &ast.ObjectList{} for { n, err := p.parseObjectItem() @@ -41,14 +43,14 @@ func (p *Parser) Parse() (Node, error) { } // we successfully parsed a node, add it to the final source node - node.add(n) + node.Add(n) } return node, nil } // parseObjectItem parses a single object item -func (p *Parser) parseObjectItem() (*ObjectItem, error) { +func (p *Parser) parseObjectItem() (*ast.ObjectItem, error) { defer un(trace(p, "ParseObjectItem")) keys, err := p.parseObjectKey() @@ -58,19 +60,19 @@ func (p *Parser) parseObjectItem() (*ObjectItem, error) { // either an assignment or object switch p.tok.Type { - case scanner.ASSIGN: - o := &ObjectItem{ - keys: keys, - assign: p.tok.Pos, + case token.ASSIGN: + o := &ast.ObjectItem{ + Keys: keys, + Assign: p.tok.Pos, } - o.val, err = p.parseType() + o.Val, err = p.parseType() if err != nil { return nil, err } return o, nil - case scanner.LBRACE: + case token.LBRACE: if len(keys) > 1 { // nested object fmt.Println("nested object") @@ -85,20 +87,20 @@ func (p *Parser) parseObjectItem() (*ObjectItem, error) { // parseType parses any type of Type, such as number, bool, string, object or // list. -func (p *Parser) parseType() (Node, error) { +func (p *Parser) parseType() (ast.Node, error) { defer un(trace(p, "ParseType")) tok := p.scan() switch tok.Type { - case scanner.NUMBER, scanner.FLOAT, scanner.BOOL, scanner.STRING: + case token.NUMBER, token.FLOAT, token.BOOL, token.STRING: return p.parseLiteralType() - case scanner.LBRACE: + case token.LBRACE: return p.parseObjectType() - case scanner.LBRACK: + case token.LBRACK: return p.parseListType() - case scanner.COMMENT: + case token.COMMENT: // implement comment - case scanner.EOF: + case token.EOF: return nil, errEofToken } @@ -106,18 +108,18 @@ func (p *Parser) parseType() (Node, error) { } // parseObjectKey parses an object key and returns a ObjectKey AST -func (p *Parser) parseObjectKey() ([]*ObjectKey, error) { +func (p *Parser) parseObjectKey() ([]*ast.ObjectKey, error) { tok := p.scan() - if tok.Type == scanner.EOF { + if tok.Type == token.EOF { return nil, errEofToken } - keys := make([]*ObjectKey, 0) + keys := make([]*ast.ObjectKey, 0) switch tok.Type { - case scanner.IDENT, scanner.STRING: + case token.IDENT, token.STRING: // add first found token - keys = append(keys, &ObjectKey{token: tok}) + keys = append(keys, &ast.ObjectKey{Token: tok}) default: return nil, fmt.Errorf("expected: IDENT | STRING got: %s", tok.Type) } @@ -131,7 +133,7 @@ func (p *Parser) parseObjectKey() ([]*ObjectKey, error) { for { tok := p.scan() switch tok.Type { - case scanner.ASSIGN: + case token.ASSIGN: // assignment or object only, but not nested objects. this is not // allowed: `foo bar = {}` if nestedObj { @@ -139,13 +141,13 @@ func (p *Parser) parseObjectKey() ([]*ObjectKey, error) { } return keys, nil - case scanner.LBRACE: + case token.LBRACE: // object return keys, nil - case scanner.IDENT, scanner.STRING: + case token.IDENT, token.STRING: // nested object nestedObj = true - keys = append(keys, &ObjectKey{token: tok}) + keys = append(keys, &ast.ObjectKey{Token: tok}) default: return nil, fmt.Errorf("expected: IDENT | STRING | ASSIGN | LBRACE got: %s", tok.Type) } @@ -153,23 +155,23 @@ func (p *Parser) parseObjectKey() ([]*ObjectKey, error) { } // parseLiteralType parses a literal type and returns a LiteralType AST -func (p *Parser) parseLiteralType() (*LiteralType, error) { +func (p *Parser) parseLiteralType() (*ast.LiteralType, error) { defer un(trace(p, "ParseLiteral")) - return &LiteralType{ - token: p.tok, + return &ast.LiteralType{ + Token: p.tok, }, nil } // parseObjectType parses an object type and returns a ObjectType AST -func (p *Parser) parseObjectType() (*ObjectType, error) { +func (p *Parser) parseObjectType() (*ast.ObjectType, error) { defer un(trace(p, "ParseObjectYpe")) return nil, errors.New("ObjectType is not implemented yet") } // parseListType parses a list type and returns a ListType AST -func (p *Parser) parseListType() (*ListType, error) { +func (p *Parser) parseListType() (*ast.ListType, error) { defer un(trace(p, "ParseListType")) return nil, errors.New("ListType is not implemented yet") @@ -177,7 +179,7 @@ func (p *Parser) parseListType() (*ListType, error) { // scan returns the next token from the underlying scanner. // If a token has been unscanned then read that instead. -func (p *Parser) scan() scanner.Token { +func (p *Parser) scan() token.Token { // If we have a token on the buffer, then return it. if p.n != 0 { p.n = 0 diff --git a/parser/parser_test.go b/parser/parser_test.go index efbb8d2..4af18ab 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -7,7 +7,8 @@ import ( "runtime" "testing" - "github.com/fatih/hcl/scanner" + "github.com/fatih/hcl/ast" + "github.com/fatih/hcl/token" ) func TestParseType(t *testing.T) { @@ -22,7 +23,7 @@ func TestParseType(t *testing.T) { fmt.Printf("n = %+v\n", n) - Walk(n, func(node Node) bool { + ast.Walk(n, func(node ast.Node) bool { fmt.Printf("node = %+v\n", node) return true }) @@ -30,21 +31,21 @@ func TestParseType(t *testing.T) { func TestObjectKey(t *testing.T) { keys := []struct { - exp []scanner.TokenType + exp []token.TokenType src string }{ - {[]scanner.TokenType{scanner.IDENT}, `foo {}`}, - {[]scanner.TokenType{scanner.IDENT}, `foo = {}`}, - {[]scanner.TokenType{scanner.IDENT}, `foo = bar`}, - {[]scanner.TokenType{scanner.IDENT}, `foo = 123`}, - {[]scanner.TokenType{scanner.IDENT}, `foo = "${var.bar}`}, - {[]scanner.TokenType{scanner.STRING}, `"foo" {}`}, - {[]scanner.TokenType{scanner.STRING}, `"foo" = {}`}, - {[]scanner.TokenType{scanner.STRING}, `"foo" = "${var.bar}`}, - {[]scanner.TokenType{scanner.IDENT, scanner.IDENT}, `foo bar {}`}, - {[]scanner.TokenType{scanner.IDENT, scanner.STRING}, `foo "bar" {}`}, - {[]scanner.TokenType{scanner.STRING, scanner.IDENT}, `"foo" bar {}`}, - {[]scanner.TokenType{scanner.IDENT, scanner.IDENT, scanner.IDENT}, `foo bar baz {}`}, + {[]token.TokenType{token.IDENT}, `foo {}`}, + {[]token.TokenType{token.IDENT}, `foo = {}`}, + {[]token.TokenType{token.IDENT}, `foo = bar`}, + {[]token.TokenType{token.IDENT}, `foo = 123`}, + {[]token.TokenType{token.IDENT}, `foo = "${var.bar}`}, + {[]token.TokenType{token.STRING}, `"foo" {}`}, + {[]token.TokenType{token.STRING}, `"foo" = {}`}, + {[]token.TokenType{token.STRING}, `"foo" = "${var.bar}`}, + {[]token.TokenType{token.IDENT, token.IDENT}, `foo bar {}`}, + {[]token.TokenType{token.IDENT, token.STRING}, `foo "bar" {}`}, + {[]token.TokenType{token.STRING, token.IDENT}, `"foo" bar {}`}, + {[]token.TokenType{token.IDENT, token.IDENT, token.IDENT}, `foo bar baz {}`}, } for _, k := range keys { @@ -54,9 +55,9 @@ func TestObjectKey(t *testing.T) { t.Fatal(err) } - tokens := []scanner.TokenType{} + tokens := []token.TokenType{} for _, o := range keys { - tokens = append(tokens, o.token.Type) + tokens = append(tokens, o.Token.Type) } equals(t, k.exp, tokens) diff --git a/scanner/scanner.go b/scanner/scanner.go index 1d6696f..f622b18 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -8,6 +8,8 @@ import ( "os" "unicode" "unicode/utf8" + + "github.com/fatih/hcl/token" ) // eof represents a marker rune for the end of the reader. @@ -19,8 +21,8 @@ type Scanner struct { src []byte // Source buffer for immutable access // Source Position - srcPos Pos // current position - prevPos Pos // previous position, used for peek() method + srcPos token.Pos // current position + prevPos token.Pos // previous position, used for peek() method lastCharLen int // length of last character in bytes lastLineLen int // length of last line in characters (for correct column reporting) @@ -30,7 +32,7 @@ type Scanner struct { // Error is called for each error encountered. If no Error // function is set, the error is reported to os.Stderr. - Error func(pos Pos, msg string) + Error func(pos token.Pos, msg string) // ErrorCount is incremented by one for each error encountered. ErrorCount int @@ -39,7 +41,7 @@ type Scanner struct { // Scan. The Filename field is always left untouched by the Scanner. If // an error is reported (via Error) and Position is invalid, the scanner is // not inside a token. - tokPos Pos + tokPos token.Pos } // New creates and initializes a new instance of Scanner using src as @@ -117,7 +119,7 @@ func (s *Scanner) peek() rune { } // Scan scans the next token and returns the token. -func (s *Scanner) Scan() Token { +func (s *Scanner) Scan() token.Token { ch := s.next() // skip white space @@ -125,7 +127,7 @@ func (s *Scanner) Scan() Token { ch = s.next() } - var tok TokenType + var tok token.TokenType // token text markings s.tokStart = s.srcPos.Offset - s.lastCharLen @@ -147,47 +149,47 @@ func (s *Scanner) Scan() Token { switch { case isLetter(ch): - tok = IDENT + tok = token.IDENT lit := s.scanIdentifier() if lit == "true" || lit == "false" { - tok = BOOL + tok = token.BOOL } case isDecimal(ch): tok = s.scanNumber(ch) default: switch ch { case eof: - tok = EOF + tok = token.EOF case '"': - tok = STRING + tok = token.STRING s.scanString() case '#', '/': - tok = COMMENT + tok = token.COMMENT s.scanComment(ch) case '.': - tok = PERIOD + tok = token.PERIOD ch = s.peek() if isDecimal(ch) { - tok = FLOAT + tok = token.FLOAT ch = s.scanMantissa(ch) ch = s.scanExponent(ch) } case '[': - tok = LBRACK + tok = token.LBRACK case ']': - tok = RBRACK + tok = token.RBRACK case '{': - tok = LBRACE + tok = token.LBRACE case '}': - tok = RBRACE + tok = token.RBRACE case ',': - tok = COMMA + tok = token.COMMA case '=': - tok = ASSIGN + tok = token.ASSIGN case '+': - tok = ADD + tok = token.ADD case '-': - tok = SUB + tok = token.SUB default: s.err("illegal char") } @@ -203,7 +205,7 @@ func (s *Scanner) Scan() Token { } s.tokStart = s.tokEnd // ensure idempotency of tokenText() call - return Token{ + return token.Token{ Type: tok, Pos: s.tokPos, Text: tokenText, @@ -244,7 +246,7 @@ func (s *Scanner) scanComment(ch rune) { } // scanNumber scans a HCL number definition starting with the given rune -func (s *Scanner) scanNumber(ch rune) TokenType { +func (s *Scanner) scanNumber(ch rune) token.TokenType { if ch == '0' { // check for hexadecimal, octal or float ch = s.next() @@ -265,7 +267,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType { s.unread() } - return NUMBER + return token.NUMBER } // now it's either something like: 0421(octal) or 0.1231(float) @@ -283,7 +285,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType { // literals of form 01e10 are treates as Numbers in HCL, which differs from Go. if ch == 'e' || ch == 'E' { ch = s.scanExponent(ch) - return NUMBER + return token.NUMBER } if ch == '.' { @@ -293,7 +295,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType { ch = s.next() ch = s.scanExponent(ch) } - return FLOAT + return token.FLOAT } if illegalOctal { @@ -303,7 +305,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType { if ch != eof { s.unread() } - return NUMBER + return token.NUMBER } s.scanMantissa(ch) @@ -311,7 +313,7 @@ func (s *Scanner) scanNumber(ch rune) TokenType { // literals of form 1e10 are treates as Numbers in HCL, which differs from Go. if ch == 'e' || ch == 'E' { ch = s.scanExponent(ch) - return NUMBER + return token.NUMBER } if ch == '.' { @@ -320,11 +322,11 @@ func (s *Scanner) scanNumber(ch rune) TokenType { ch = s.next() ch = s.scanExponent(ch) } - return FLOAT + return token.FLOAT } s.unread() - return NUMBER + return token.NUMBER } // scanMantissa scans the mantissa begining from the rune. It returns the next @@ -446,7 +448,7 @@ func (s *Scanner) scanIdentifier() string { // recentPosition returns the position of the character immediately after the // character or token returned by the last call to Scan. -func (s *Scanner) recentPosition() (pos Pos) { +func (s *Scanner) recentPosition() (pos token.Pos) { pos.Offset = s.srcPos.Offset - s.lastCharLen switch { case s.srcPos.Column > 0: diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 62590d6..f9b2f5e 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -4,161 +4,163 @@ import ( "bytes" "fmt" "testing" + + "github.com/fatih/hcl/token" ) var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" type tokenPair struct { - tok TokenType + tok token.TokenType text string } var tokenLists = map[string][]tokenPair{ "comment": []tokenPair{ - {COMMENT, "//"}, - {COMMENT, "////"}, - {COMMENT, "// comment"}, - {COMMENT, "// /* comment */"}, - {COMMENT, "// // comment //"}, - {COMMENT, "//" + f100}, - {COMMENT, "#"}, - {COMMENT, "##"}, - {COMMENT, "# comment"}, - {COMMENT, "# /* comment */"}, - {COMMENT, "# # comment #"}, - {COMMENT, "#" + f100}, - {COMMENT, "/**/"}, - {COMMENT, "/***/"}, - {COMMENT, "/* comment */"}, - {COMMENT, "/* // comment */"}, - {COMMENT, "/* /* comment */"}, - {COMMENT, "/*\n comment\n*/"}, - {COMMENT, "/*" + f100 + "*/"}, + {token.COMMENT, "//"}, + {token.COMMENT, "////"}, + {token.COMMENT, "// comment"}, + {token.COMMENT, "// /* comment */"}, + {token.COMMENT, "// // comment //"}, + {token.COMMENT, "//" + f100}, + {token.COMMENT, "#"}, + {token.COMMENT, "##"}, + {token.COMMENT, "# comment"}, + {token.COMMENT, "# /* comment */"}, + {token.COMMENT, "# # comment #"}, + {token.COMMENT, "#" + f100}, + {token.COMMENT, "/**/"}, + {token.COMMENT, "/***/"}, + {token.COMMENT, "/* comment */"}, + {token.COMMENT, "/* // comment */"}, + {token.COMMENT, "/* /* comment */"}, + {token.COMMENT, "/*\n comment\n*/"}, + {token.COMMENT, "/*" + f100 + "*/"}, }, "operator": []tokenPair{ - {LBRACK, "["}, - {LBRACE, "{"}, - {COMMA, ","}, - {PERIOD, "."}, - {RBRACK, "]"}, - {RBRACE, "}"}, - {ASSIGN, "="}, - {ADD, "+"}, - {SUB, "-"}, + {token.LBRACK, "["}, + {token.LBRACE, "{"}, + {token.COMMA, ","}, + {token.PERIOD, "."}, + {token.RBRACK, "]"}, + {token.RBRACE, "}"}, + {token.ASSIGN, "="}, + {token.ADD, "+"}, + {token.SUB, "-"}, }, "bool": []tokenPair{ - {BOOL, "true"}, - {BOOL, "false"}, + {token.BOOL, "true"}, + {token.BOOL, "false"}, }, - "ident": []tokenPair{ - {IDENT, "a"}, - {IDENT, "a0"}, - {IDENT, "foobar"}, - {IDENT, "abc123"}, - {IDENT, "LGTM"}, - {IDENT, "_"}, - {IDENT, "_abc123"}, - {IDENT, "abc123_"}, - {IDENT, "_abc_123_"}, - {IDENT, "_äöü"}, - {IDENT, "_本"}, - {IDENT, "äöü"}, - {IDENT, "本"}, - {IDENT, "a۰۱۸"}, - {IDENT, "foo६४"}, - {IDENT, "bar９８７６"}, + "identoken.t": []tokenPair{ + {token.IDENT, "a"}, + {token.IDENT, "a0"}, + {token.IDENT, "foobar"}, + {token.IDENT, "abc123"}, + {token.IDENT, "LGTM"}, + {token.IDENT, "_"}, + {token.IDENT, "_abc123"}, + {token.IDENT, "abc123_"}, + {token.IDENT, "_abc_123_"}, + {token.IDENT, "_äöü"}, + {token.IDENT, "_本"}, + {token.IDENT, "äöü"}, + {token.IDENT, "本"}, + {token.IDENT, "a۰۱۸"}, + {token.IDENT, "foo६४"}, + {token.IDENT, "bar９８７６"}, }, - "string": []tokenPair{ - {STRING, `" "`}, - {STRING, `"a"`}, - {STRING, `"本"`}, - {STRING, `"\a"`}, - {STRING, `"\b"`}, - {STRING, `"\f"`}, - {STRING, `"\n"`}, - {STRING, `"\r"`}, - {STRING, `"\t"`}, - {STRING, `"\v"`}, - {STRING, `"\""`}, - {STRING, `"\000"`}, - {STRING, `"\777"`}, - {STRING, `"\x00"`}, - {STRING, `"\xff"`}, - {STRING, `"\u0000"`}, - {STRING, `"\ufA16"`}, - {STRING, `"\U00000000"`}, - {STRING, `"\U0000ffAB"`}, - {STRING, `"` + f100 + `"`}, + "stritoken.ng": []tokenPair{ + {token.STRING, `" "`}, + {token.STRING, `"a"`}, + {token.STRING, `"本"`}, + {token.STRING, `"\a"`}, + {token.STRING, `"\b"`}, + {token.STRING, `"\f"`}, + {token.STRING, `"\n"`}, + {token.STRING, `"\r"`}, + {token.STRING, `"\t"`}, + {token.STRING, `"\v"`}, + {token.STRING, `"\""`}, + {token.STRING, `"\000"`}, + {token.STRING, `"\777"`}, + {token.STRING, `"\x00"`}, + {token.STRING, `"\xff"`}, + {token.STRING, `"\u0000"`}, + {token.STRING, `"\ufA16"`}, + {token.STRING, `"\U00000000"`}, + {token.STRING, `"\U0000ffAB"`}, + {token.STRING, `"` + f100 + `"`}, }, - "number": []tokenPair{ - {NUMBER, "0"}, - {NUMBER, "1"}, - {NUMBER, "9"}, - {NUMBER, "42"}, - {NUMBER, "1234567890"}, - {NUMBER, "00"}, - {NUMBER, "01"}, - {NUMBER, "07"}, - {NUMBER, "042"}, - {NUMBER, "01234567"}, - {NUMBER, "0x0"}, - {NUMBER, "0x1"}, - {NUMBER, "0xf"}, - {NUMBER, "0x42"}, - {NUMBER, "0x123456789abcDEF"}, - {NUMBER, "0x" + f100}, - {NUMBER, "0X0"}, - {NUMBER, "0X1"}, - {NUMBER, "0XF"}, - {NUMBER, "0X42"}, - {NUMBER, "0X123456789abcDEF"}, - {NUMBER, "0X" + f100}, - {NUMBER, "0e0"}, - {NUMBER, "1e0"}, - {NUMBER, "42e0"}, - {NUMBER, "01234567890e0"}, - {NUMBER, "0E0"}, - {NUMBER, "1E0"}, - {NUMBER, "42E0"}, - {NUMBER, "01234567890E0"}, - {NUMBER, "0e+10"}, - {NUMBER, "1e-10"}, - {NUMBER, "42e+10"}, - {NUMBER, "01234567890e-10"}, - {NUMBER, "0E+10"}, - {NUMBER, "1E-10"}, - {NUMBER, "42E+10"}, - {NUMBER, "01234567890E-10"}, + "numbtoken.er": []tokenPair{ + {token.NUMBER, "0"}, + {token.NUMBER, "1"}, + {token.NUMBER, "9"}, + {token.NUMBER, "42"}, + {token.NUMBER, "1234567890"}, + {token.NUMBER, "00"}, + {token.NUMBER, "01"}, + {token.NUMBER, "07"}, + {token.NUMBER, "042"}, + {token.NUMBER, "01234567"}, + {token.NUMBER, "0x0"}, + {token.NUMBER, "0x1"}, + {token.NUMBER, "0xf"}, + {token.NUMBER, "0x42"}, + {token.NUMBER, "0x123456789abcDEF"}, + {token.NUMBER, "0x" + f100}, + {token.NUMBER, "0X0"}, + {token.NUMBER, "0X1"}, + {token.NUMBER, "0XF"}, + {token.NUMBER, "0X42"}, + {token.NUMBER, "0X123456789abcDEF"}, + {token.NUMBER, "0X" + f100}, + {token.NUMBER, "0e0"}, + {token.NUMBER, "1e0"}, + {token.NUMBER, "42e0"}, + {token.NUMBER, "01234567890e0"}, + {token.NUMBER, "0E0"}, + {token.NUMBER, "1E0"}, + {token.NUMBER, "42E0"}, + {token.NUMBER, "01234567890E0"}, + {token.NUMBER, "0e+10"}, + {token.NUMBER, "1e-10"}, + {token.NUMBER, "42e+10"}, + {token.NUMBER, "01234567890e-10"}, + {token.NUMBER, "0E+10"}, + {token.NUMBER, "1E-10"}, + {token.NUMBER, "42E+10"}, + {token.NUMBER, "01234567890E-10"}, }, - "float": []tokenPair{ - {FLOAT, "0."}, - {FLOAT, "1."}, - {FLOAT, "42."}, - {FLOAT, "01234567890."}, - {FLOAT, ".0"}, - {FLOAT, ".1"}, - {FLOAT, ".42"}, - {FLOAT, ".0123456789"}, - {FLOAT, "0.0"}, - {FLOAT, "1.0"}, - {FLOAT, "42.0"}, - {FLOAT, "01234567890.0"}, - {FLOAT, "01.8e0"}, - {FLOAT, "1.4e0"}, - {FLOAT, "42.2e0"}, - {FLOAT, "01234567890.12e0"}, - {FLOAT, "0.E0"}, - {FLOAT, "1.12E0"}, - {FLOAT, "42.123E0"}, - {FLOAT, "01234567890.213E0"}, - {FLOAT, "0.2e+10"}, - {FLOAT, "1.2e-10"}, - {FLOAT, "42.54e+10"}, - {FLOAT, "01234567890.98e-10"}, - {FLOAT, "0.1E+10"}, - {FLOAT, "1.1E-10"}, - {FLOAT, "42.1E+10"}, - {FLOAT, "01234567890.1E-10"}, + "floatoken.t": []tokenPair{ + {token.FLOAT, "0."}, + {token.FLOAT, "1."}, + {token.FLOAT, "42."}, + {token.FLOAT, "01234567890."}, + {token.FLOAT, ".0"}, + {token.FLOAT, ".1"}, + {token.FLOAT, ".42"}, + {token.FLOAT, ".0123456789"}, + {token.FLOAT, "0.0"}, + {token.FLOAT, "1.0"}, + {token.FLOAT, "42.0"}, + {token.FLOAT, "01234567890.0"}, + {token.FLOAT, "01.8e0"}, + {token.FLOAT, "1.4e0"}, + {token.FLOAT, "42.2e0"}, + {token.FLOAT, "01234567890.12e0"}, + {token.FLOAT, "0.E0"}, + {token.FLOAT, "1.12E0"}, + {token.FLOAT, "42.123E0"}, + {token.FLOAT, "01234567890.213E0"}, + {token.FLOAT, "0.2e+10"}, + {token.FLOAT, "1.2e-10"}, + {token.FLOAT, "42.54e+10"}, + {token.FLOAT, "01234567890.98e-10"}, + {token.FLOAT, "0.1E+10"}, + {token.FLOAT, "1.1E-10"}, + {token.FLOAT, "42.1E+10"}, + {token.FLOAT, "01234567890.1E-10"}, }, } @@ -184,7 +186,7 @@ func TestPosition(t *testing.T) { s := New(buf.Bytes()) - pos := Pos{"", 4, 1, 5} + pos := token.Pos{"", 4, 1, 5} s.Scan() for _, listName := range orderedTokenLists { @@ -270,63 +272,63 @@ func TestRealExample(t *testing.T) { }` literals := []struct { - tokenType TokenType + tokenType token.TokenType literal string }{ - {COMMENT, `// This comes from Terraform, as a test`}, - {IDENT, `variable`}, - {STRING, `"foo"`}, - {LBRACE, `{`}, - {IDENT, `default`}, - {ASSIGN, `=`}, - {STRING, `"bar"`}, - {IDENT, `description`}, - {ASSIGN, `=`}, - {STRING, `"bar"`}, - {RBRACE, `}`}, - {IDENT, `provider`}, - {STRING, `"aws"`}, - {LBRACE, `{`}, - {IDENT, `access_key`}, - {ASSIGN, `=`}, - {STRING, `"foo"`}, - {IDENT, `secret_key`}, - {ASSIGN, `=`}, - {STRING, `"bar"`}, - {RBRACE, `}`}, - {IDENT, `resource`}, - {STRING, `"aws_security_group"`}, - {STRING, `"firewall"`}, - {LBRACE, `{`}, - {IDENT, `count`}, - {ASSIGN, `=`}, - {NUMBER, `5`}, - {RBRACE, `}`}, - {IDENT, `resource`}, - {IDENT, `aws_instance`}, - {STRING, `"web"`}, - {LBRACE, `{`}, - {IDENT, `ami`}, - {ASSIGN, `=`}, - {STRING, `"${var.foo}"`}, - {IDENT, `security_groups`}, - {ASSIGN, `=`}, - {LBRACK, `[`}, - {STRING, `"foo"`}, - {COMMA, `,`}, - {STRING, `"${aws_security_group.firewall.foo}"`}, - {RBRACK, `]`}, - {IDENT, `network_interface`}, - {LBRACE, `{`}, - {IDENT, `device_index`}, - {ASSIGN, `=`}, - {NUMBER, `0`}, - {IDENT, `description`}, - {ASSIGN, `=`}, - {STRING, `"Main network interface"`}, - {RBRACE, `}`}, - {RBRACE, `}`}, - {EOF, ``}, + {token.COMMENT, `// This comes from Terraform, as a test`}, + {token.IDENT, `variable`}, + {token.STRING, `"foo"`}, + {token.LBRACE, `{`}, + {token.IDENT, `default`}, + {token.ASSIGN, `=`}, + {token.STRING, `"bar"`}, + {token.IDENT, `description`}, + {token.ASSIGN, `=`}, + {token.STRING, `"bar"`}, + {token.RBRACE, `}`}, + {token.IDENT, `provider`}, + {token.STRING, `"aws"`}, + {token.LBRACE, `{`}, + {token.IDENT, `access_key`}, + {token.ASSIGN, `=`}, + {token.STRING, `"foo"`}, + {token.IDENT, `secret_key`}, + {token.ASSIGN, `=`}, + {token.STRING, `"bar"`}, + {token.RBRACE, `}`}, + {token.IDENT, `resource`}, + {token.STRING, `"aws_security_group"`}, + {token.STRING, `"firewall"`}, + {token.LBRACE, `{`}, + {token.IDENT, `count`}, + {token.ASSIGN, `=`}, + {token.NUMBER, `5`}, + {token.RBRACE, `}`}, + {token.IDENT, `resource`}, + {token.IDENT, `aws_instance`}, + {token.STRING, `"web"`}, + {token.LBRACE, `{`}, + {token.IDENT, `ami`}, + {token.ASSIGN, `=`}, + {token.STRING, `"${var.foo}"`}, + {token.IDENT, `security_groups`}, + {token.ASSIGN, `=`}, + {token.LBRACK, `[`}, + {token.STRING, `"foo"`}, + {token.COMMA, `,`}, + {token.STRING, `"${aws_security_group.firewall.foo}"`}, + {token.RBRACK, `]`}, + {token.IDENT, `network_interface`}, + {token.LBRACE, `{`}, + {token.IDENT, `device_index`}, + {token.ASSIGN, `=`}, + {token.NUMBER, `0`}, + {token.IDENT, `description`}, + {token.ASSIGN, `=`}, + {token.STRING, `"Main network interface"`}, + {token.RBRACE, `}`}, + {token.RBRACE, `}`}, + {token.EOF, ``}, } s := New([]byte(complexHCL)) @@ -344,32 +346,32 @@ func TestRealExample(t *testing.T) { } func TestError(t *testing.T) { - testError(t, "\x80", "1:1", "illegal UTF-8 encoding", ILLEGAL) - testError(t, "\xff", "1:1", "illegal UTF-8 encoding", ILLEGAL) + testError(t, "\x80", "1:1", "illegal UTF-8 encoding", token.ILLEGAL) + testError(t, "\xff", "1:1", "illegal UTF-8 encoding", token.ILLEGAL) - testError(t, "ab\x80", "1:3", "illegal UTF-8 encoding", IDENT) - testError(t, "abc\xff", "1:4", "illegal UTF-8 encoding", IDENT) + testError(t, "ab\x80", "1:3", "illegal UTF-8 encoding", token.IDENT) + testError(t, "abc\xff", "1:4", "illegal UTF-8 encoding", token.IDENT) - testError(t, `"ab`+"\x80", "1:4", "illegal UTF-8 encoding", STRING) - testError(t, `"abc`+"\xff", "1:5", "illegal UTF-8 encoding", STRING) + testError(t, `"ab`+"\x80", "1:4", "illegal UTF-8 encoding", token.STRING) + testError(t, `"abc`+"\xff", "1:5", "illegal UTF-8 encoding", token.STRING) - testError(t, `01238`, "1:6", "illegal octal number", NUMBER) - testError(t, `01238123`, "1:9", "illegal octal number", NUMBER) - testError(t, `0x`, "1:3", "illegal hexadecimal number", NUMBER) - testError(t, `0xg`, "1:3", "illegal hexadecimal number", NUMBER) - testError(t, `'aa'`, "1:1", "illegal char", ILLEGAL) + testError(t, `01238`, "1:6", "illegal octal number", token.NUMBER) + testError(t, `01238123`, "1:9", "illegal octal number", token.NUMBER) + testError(t, `0x`, "1:3", "illegal hexadecimal number", token.NUMBER) + testError(t, `0xg`, "1:3", "illegal hexadecimal number", token.NUMBER) + testError(t, `'aa'`, "1:1", "illegal char", token.ILLEGAL) - testError(t, `"`, "1:2", "literal not terminated", STRING) - testError(t, `"abc`, "1:5", "literal not terminated", STRING) - testError(t, `"abc`+"\n", "1:5", "literal not terminated", STRING) - testError(t, `/*/`, "1:4", "comment not terminated", COMMENT) + testError(t, `"`, "1:2", "literal not terminated", token.STRING) + testError(t, `"abc`, "1:5", "literal not terminated", token.STRING) + testError(t, `"abc`+"\n", "1:5", "literal not terminated", token.STRING) + testError(t, `/*/`, "1:4", "comment not terminated", token.COMMENT) } -func testError(t *testing.T, src, pos, msg string, tok TokenType) { +func testError(t *testing.T, src, pos, msg string, tok token.TokenType) { s := New([]byte(src)) errorCalled := false - s.Error = func(p Pos, m string) { + s.Error = func(p token.Pos, m string) { if !errorCalled { if pos != p.String() { t.Errorf("pos = %q, want %q for %q", p, pos, src) diff --git a/scanner/position.go b/token/position.go similarity index 98% rename from scanner/position.go rename to token/position.go index aef546c..c151e50 100644 --- a/scanner/position.go +++ b/token/position.go @@ -1,4 +1,4 @@ -package scanner +package token import "fmt" diff --git a/scanner/token.go b/token/token.go similarity index 99% rename from scanner/token.go rename to token/token.go index deb6f9a..ada0d86 100644 --- a/scanner/token.go +++ b/token/token.go @@ -1,4 +1,4 @@ -package scanner +package token import ( "fmt" diff --git a/scanner/token_test.go b/token/token_test.go similarity index 97% rename from scanner/token_test.go rename to token/token_test.go index 0e05576..534c1d0 100644 --- a/scanner/token_test.go +++ b/token/token_test.go @@ -1,4 +1,4 @@ -package scanner +package token import "testing"