go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

remove hclnew

Browse Source
This commit is contained in:
Mitchell Hashimoto 2015-11-06 16:18:55 -08:00
parent c1090aad4a
commit 5c5e6206ce
39 changed files with 0 additions and 3173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hclnew/.travis.yml
View File

@ -1,3 +0,0 @@
language: go
go: 1.5

141
hclnew/ast/ast.go
View File

@ -1,141 +0,0 @@
// Package ast declares the types used to represent syntax trees for HCL
// (HashiCorp Configuration Language)
package ast
import (
"github.com/hashicorp/hcl/hcl/token"
)
// Node is an element in the abstract syntax tree.
type Node interface {
node()
Pos() token.Pos
}
func (File) node() {}
func (ObjectList) node() {}
func (ObjectKey) node() {}
func (ObjectItem) node() {}
func (Comment) node() {}
func (CommentGroup) node() {}
func (ObjectType) node() {}
func (LiteralType) node() {}
func (ListType) node() {}
// File represents a single HCL file
type File struct {
Node Node // usually a *ObjectList
Comments []*CommentGroup // list of all comments in the source
}
func (f *File) Pos() token.Pos {
return f.Node.Pos()
}
// ObjectList represents a list of ObjectItems. An HCL file itself is an
// ObjectList.
type ObjectList struct {
Items []*ObjectItem
}
func (o *ObjectList) Add(item *ObjectItem) {
o.Items = append(o.Items, item)
}
func (o *ObjectList) Pos() token.Pos {
// always returns the uninitiliazed position
return o.Items[0].Pos()
}
// ObjectItem represents a HCL Object Item. An item is represented with a key
// (or keys). It can be an assignment or an object (both normal and nested)
type ObjectItem struct {
// keys is only one length long if it's of type assignment. If it's a
// nested object it can be larger than one. In that case "assign" is
// invalid as there is no assignments for a nested object.
Keys []*ObjectKey
// assign contains the position of "=", if any
Assign token.Pos
// val is the item itself. It can be an object,list, number, bool or a
// string. If key length is larger than one, val can be only of type
// Object.
Val Node
LeadComment *CommentGroup // associated lead comment
LineComment *CommentGroup // associated line comment
}
func (o *ObjectItem) Pos() token.Pos {
return o.Keys[0].Pos()
}
// ObjectKeys are either an identifier or of type string.
type ObjectKey struct {
Token token.Token
}
func (o *ObjectKey) Pos() token.Pos {
return o.Token.Pos
}
// LiteralType represents a literal of basic type. Valid types are:
// token.NUMBER, token.FLOAT, token.BOOL and token.STRING
type LiteralType struct {
Token token.Token
// associated line comment, only when used in a list
LineComment *CommentGroup
}
func (l *LiteralType) Pos() token.Pos {
return l.Token.Pos
}
// ListStatement represents a HCL List type
type ListType struct {
Lbrack token.Pos // position of "["
Rbrack token.Pos // position of "]"
List []Node // the elements in lexical order
}
func (l *ListType) Pos() token.Pos {
return l.Lbrack
}
func (l *ListType) Add(node Node) {
l.List = append(l.List, node)
}
// ObjectType represents a HCL Object Type
type ObjectType struct {
Lbrace token.Pos // position of "{"
Rbrace token.Pos // position of "}"
List *ObjectList // the nodes in lexical order
}
func (o *ObjectType) Pos() token.Pos {
return o.Lbrace
}
// Comment node represents a single //, # style or /*- style commment
type Comment struct {
Start token.Pos // position of / or #
Text string
}
func (c *Comment) Pos() token.Pos {
return c.Start
}
// CommentGroup node represents a sequence of comments with no other tokens and
// no empty lines between.
type CommentGroup struct {
List []*Comment // len(List) > 0
}
func (c *CommentGroup) Pos() token.Pos {
return c.List[0].Pos()
}

42
hclnew/ast/walk.go
View File

@ -1,42 +0,0 @@
package ast
import "fmt"
// Walk traverses an AST in depth-first order: It starts by calling fn(node);
// node must not be nil. If f returns true, Walk invokes f recursively for
// each of the non-nil children of node, followed by a call of f(nil).
func Walk(node Node, fn func(Node) bool) {
if !fn(node) {
return
}
switch n := node.(type) {
case *File:
Walk(n.Node, fn)
case *ObjectList:
for _, item := range n.Items {
Walk(item, fn)
}
case *ObjectKey:
// nothing to do
case *ObjectItem:
for _, k := range n.Keys {
Walk(k, fn)
}
Walk(n.Val, fn)
case *LiteralType:
// nothing to do
case *ListType:
for _, l := range n.List {
Walk(l, fn)
}
case *ObjectType:
for _, l := range n.List.Items {
Walk(l, fn)
}
default:
fmt.Printf(" unknown type: %T\n", n)
}
fn(nil)
}

385
hclnew/parser/parser.go
View File

@ -1,385 +0,0 @@
// Package parser implements a parser for HCL (HashiCorp Configuration
// Language)
package parser
import (
"errors"
"fmt"
"github.com/hashicorp/hcl/hcl/ast"
"github.com/hashicorp/hcl/hcl/scanner"
"github.com/hashicorp/hcl/hcl/token"
)
type Parser struct {
sc *scanner.Scanner
// Last read token
tok token.Token
commaPrev token.Token
comments []*ast.CommentGroup
leadComment *ast.CommentGroup // last lead comment
lineComment *ast.CommentGroup // last line comment
enableTrace bool
indent int
n int // buffer size (max = 1)
}
func newParser(src []byte) *Parser {
return &Parser{
sc: scanner.New(src),
}
}
// Parse returns the fully parsed source and returns the abstract syntax tree.
func Parse(src []byte) (*ast.File, error) {
p := newParser(src)
return p.Parse()
}
var errEofToken = errors.New("EOF token found")
// Parse returns the fully parsed source and returns the abstract syntax tree.
func (p *Parser) Parse() (*ast.File, error) {
f := &ast.File{}
var err error
f.Node, err = p.objectList()
if err != nil {
return nil, err
}
f.Comments = p.comments
return f, nil
}
func (p *Parser) objectList() (*ast.ObjectList, error) {
defer un(trace(p, "ParseObjectList"))
node := &ast.ObjectList{}
for {
n, err := p.objectItem()
if err == errEofToken {
break // we are finished
}
// we don't return a nil node, because might want to use already
// collected items.
if err != nil {
return node, err
}
node.Add(n)
}
return node, nil
}
func (p *Parser) consumeComment() (comment *ast.Comment, endline int) {
endline = p.tok.Pos.Line
// count the endline if it's multiline comment, ie starting with /*
if len(p.tok.Text) > 1 && p.tok.Text[1] == '*' {
// don't use range here - no need to decode Unicode code points
for i := 0; i < len(p.tok.Text); i++ {
if p.tok.Text[i] == '\n' {
endline++
}
}
}
comment = &ast.Comment{Start: p.tok.Pos, Text: p.tok.Text}
p.tok = p.sc.Scan()
return
}
func (p *Parser) consumeCommentGroup(n int) (comments *ast.CommentGroup, endline int) {
var list []*ast.Comment
endline = p.tok.Pos.Line
for p.tok.Type == token.COMMENT && p.tok.Pos.Line <= endline+n {
var comment *ast.Comment
comment, endline = p.consumeComment()
list = append(list, comment)
}
// add comment group to the comments list
comments = &ast.CommentGroup{List: list}
p.comments = append(p.comments, comments)
return
}
// objectItem parses a single object item
func (p *Parser) objectItem() (*ast.ObjectItem, error) {
defer un(trace(p, "ParseObjectItem"))
keys, err := p.objectKey()
if err != nil {
return nil, err
}
o := &ast.ObjectItem{
Keys: keys,
}
if p.leadComment != nil {
o.LeadComment = p.leadComment
p.leadComment = nil
}
switch p.tok.Type {
case token.ASSIGN:
o.Assign = p.tok.Pos
o.Val, err = p.object()
if err != nil {
return nil, err
}
case token.LBRACE:
o.Val, err = p.objectType()
if err != nil {
return nil, err
}
}
// do a look-ahead for line comment
p.scan()
if o.Val.Pos().Line == keys[0].Pos().Line && p.lineComment != nil {
o.LineComment = p.lineComment
p.lineComment = nil
}
p.unscan()
return o, nil
}
// objectKey parses an object key and returns a ObjectKey AST
func (p *Parser) objectKey() ([]*ast.ObjectKey, error) {
keyCount := 0
keys := make([]*ast.ObjectKey, 0)
for {
tok := p.scan()
switch tok.Type {
case token.EOF:
return nil, errEofToken
case token.ASSIGN:
// assignment or object only, but not nested objects. this is not
// allowed: `foo bar = {}`
if keyCount > 1 {
return nil, fmt.Errorf("nested object expected: LBRACE got: %s", p.tok.Type)
}
if keyCount == 0 {
return nil, errors.New("no keys found!!!")
}
return keys, nil
case token.LBRACE:
// object
return keys, nil
case token.IDENT, token.STRING:
keyCount++
keys = append(keys, &ast.ObjectKey{Token: p.tok})
case token.ILLEGAL:
fmt.Println("illegal")
default:
return nil, fmt.Errorf("expected: IDENT | STRING | ASSIGN | LBRACE got: %s", p.tok.Type)
}
}
}
// object parses any type of object, such as number, bool, string, object or
// list.
func (p *Parser) object() (ast.Node, error) {
defer un(trace(p, "ParseType"))
tok := p.scan()
switch tok.Type {
case token.NUMBER, token.FLOAT, token.BOOL, token.STRING:
return p.literalType()
case token.LBRACE:
return p.objectType()
case token.LBRACK:
return p.listType()
case token.COMMENT:
// implement comment
case token.EOF:
return nil, errEofToken
}
return nil, fmt.Errorf("Unknown token: %+v", tok)
}
// objectType parses an object type and returns a ObjectType AST
func (p *Parser) objectType() (*ast.ObjectType, error) {
defer un(trace(p, "ParseObjectType"))
// we assume that the currently scanned token is a LBRACE
o := &ast.ObjectType{
Lbrace: p.tok.Pos,
}
l, err := p.objectList()
// if we hit RBRACE, we are good to go (means we parsed all Items), if it's
// not a RBRACE, it's an syntax error and we just return it.
if err != nil && p.tok.Type != token.RBRACE {
return nil, err
}
o.List = l
o.Rbrace = p.tok.Pos // advanced via parseObjectList
return o, nil
}
// listType parses a list type and returns a ListType AST
func (p *Parser) listType() (*ast.ListType, error) {
defer un(trace(p, "ParseListType"))
// we assume that the currently scanned token is a LBRACK
l := &ast.ListType{
Lbrack: p.tok.Pos,
}
for {
tok := p.scan()
switch tok.Type {
case token.NUMBER, token.FLOAT, token.STRING:
node, err := p.literalType()
if err != nil {
return nil, err
}
l.Add(node)
case token.COMMA:
// get next list item or we are at the end
// do a look-ahead for line comment
p.scan()
if p.lineComment != nil {
lit, ok := l.List[len(l.List)-1].(*ast.LiteralType)
if ok {
lit.LineComment = p.lineComment
l.List[len(l.List)-1] = lit
p.lineComment = nil
}
}
p.unscan()
continue
case token.BOOL:
// TODO(arslan) should we support? not supported by HCL yet
case token.LBRACK:
// TODO(arslan) should we support nested lists? Even though it's
// written in README of HCL, it's not a part of the grammar
// (not defined in parse.y)
case token.RBRACK:
// finished
l.Rbrack = p.tok.Pos
return l, nil
default:
return nil, fmt.Errorf("unexpected token while parsing list: %s", tok.Type)
}
}
}
// literalType parses a literal type and returns a LiteralType AST
func (p *Parser) literalType() (*ast.LiteralType, error) {
defer un(trace(p, "ParseLiteral"))
return &ast.LiteralType{
Token: p.tok,
}, nil
}
// scan returns the next token from the underlying scanner. If a token has
// been unscanned then read that instead. In the process, it collects any
// comment groups encountered, and remembers the last lead and line comments.
func (p *Parser) scan() token.Token {
// If we have a token on the buffer, then return it.
if p.n != 0 {
p.n = 0
return p.tok
}
// Otherwise read the next token from the scanner and Save it to the buffer
// in case we unscan later.
prev := p.tok
p.tok = p.sc.Scan()
if p.tok.Type == token.COMMENT {
var comment *ast.CommentGroup
var endline int
// fmt.Printf("p.tok.Pos.Line = %+v prev: %d endline %d \n",
// p.tok.Pos.Line, prev.Pos.Line, endline)
if p.tok.Pos.Line == prev.Pos.Line {
// The comment is on same line as the previous token; it
// cannot be a lead comment but may be a line comment.
comment, endline = p.consumeCommentGroup(0)
if p.tok.Pos.Line != endline {
// The next token is on a different line, thus
// the last comment group is a line comment.
p.lineComment = comment
}
}
// consume successor comments, if any
endline = -1
for p.tok.Type == token.COMMENT {
comment, endline = p.consumeCommentGroup(1)
}
if endline+1 == p.tok.Pos.Line && p.tok.Type != token.RBRACE {
switch p.tok.Type {
case token.RBRACE, token.RBRACK:
// Do not count for these cases
default:
// The next token is following on the line immediately after the
// comment group, thus the last comment group is a lead comment.
p.leadComment = comment
}
}
}
return p.tok
}
// unscan pushes the previously read token back onto the buffer.
func (p *Parser) unscan() {
p.n = 1
}
// ----------------------------------------------------------------------------
// Parsing support
func (p *Parser) printTrace(a ...interface{}) {
if !p.enableTrace {
return
}
const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
const n = len(dots)
fmt.Printf("%5d:%3d: ", p.tok.Pos.Line, p.tok.Pos.Column)
i := 2 * p.indent
for i > n {
fmt.Print(dots)
i -= n
}
// i <= n
fmt.Print(dots[0:i])
fmt.Println(a...)
}
func trace(p *Parser, msg string) *Parser {
p.printTrace(msg, "(")
p.indent++
return p
}
// Usage pattern: defer un(trace(p, "..."))
func un(p *Parser) {
p.indent--
p.printTrace(")")
}

301
hclnew/parser/parser_test.go
View File

@ -1,301 +0,0 @@
package parser
import (
"fmt"
"io/ioutil"
"path/filepath"
"reflect"
"runtime"
"testing"
"github.com/fatih/hcl/ast"
"github.com/fatih/hcl/token"
)
func TestType(t *testing.T) {
var literals = []struct {
typ token.Type
src string
}{
{token.STRING, `foo = "foo"`},
{token.NUMBER, `foo = 123`},
{token.FLOAT, `foo = 123.12`},
{token.FLOAT, `foo = -123.12`},
{token.BOOL, `foo = true`},
}
for _, l := range literals {
p := newParser([]byte(l.src))
item, err := p.objectItem()
if err != nil {
t.Error(err)
}
lit, ok := item.Val.(*ast.LiteralType)
if !ok {
t.Errorf("node should be of type LiteralType, got: %T", item.Val)
}
if lit.Token.Type != l.typ {
t.Errorf("want: %s, got: %s", l.typ, lit.Token.Type)
}
}
}
func TestListType(t *testing.T) {
var literals = []struct {
src string
tokens []token.Type
}{
{
`foo = ["123", 123]`,
[]token.Type{token.STRING, token.NUMBER},
},
{
`foo = [123, "123",]`,
[]token.Type{token.NUMBER, token.STRING},
},
{
`foo = []`,
[]token.Type{},
},
{
`foo = ["123", 123]`,
[]token.Type{token.STRING, token.NUMBER},
},
}
for _, l := range literals {
p := newParser([]byte(l.src))
item, err := p.objectItem()
if err != nil {
t.Error(err)
}
list, ok := item.Val.(*ast.ListType)
if !ok {
t.Errorf("node should be of type LiteralType, got: %T", item.Val)
}
tokens := []token.Type{}
for _, li := range list.List {
if tp, ok := li.(*ast.LiteralType); ok {
tokens = append(tokens, tp.Token.Type)
}
}
equals(t, l.tokens, tokens)
}
}
func TestObjectType(t *testing.T) {
var literals = []struct {
src string
nodeType []ast.Node
itemLen int
}{
{
`foo = {}`,
nil,
0,
},
{
`foo = {
bar = "fatih"
}`,
[]ast.Node{&ast.LiteralType{}},
1,
},
{
`foo = {
bar = "fatih"
baz = ["arslan"]
}`,
[]ast.Node{
&ast.LiteralType{},
&ast.ListType{},
},
2,
},
{
`foo = {
bar {}
}`,
[]ast.Node{
&ast.ObjectType{},
},
1,
},
{
`foo {
bar {}
foo = true
}`,
[]ast.Node{
&ast.ObjectType{},
&ast.LiteralType{},
},
2,
},
}
for _, l := range literals {
p := newParser([]byte(l.src))
// p.enableTrace = true
item, err := p.objectItem()
if err != nil {
t.Error(err)
}
// we know that the ObjectKey name is foo for all cases, what matters
// is the object
obj, ok := item.Val.(*ast.ObjectType)
if !ok {
t.Errorf("node should be of type LiteralType, got: %T", item.Val)
}
// check if the total length of items are correct
equals(t, l.itemLen, len(obj.List.Items))
// check if the types are correct
for i, item := range obj.List.Items {
equals(t, reflect.TypeOf(l.nodeType[i]), reflect.TypeOf(item.Val))
}
}
}
func TestObjectKey(t *testing.T) {
keys := []struct {
exp []token.Type
src string
}{
{[]token.Type{token.IDENT}, `foo {}`},
{[]token.Type{token.IDENT}, `foo = {}`},
{[]token.Type{token.IDENT}, `foo = bar`},
{[]token.Type{token.IDENT}, `foo = 123`},
{[]token.Type{token.IDENT}, `foo = "${var.bar}`},
{[]token.Type{token.STRING}, `"foo" {}`},
{[]token.Type{token.STRING}, `"foo" = {}`},
{[]token.Type{token.STRING}, `"foo" = "${var.bar}`},
{[]token.Type{token.IDENT, token.IDENT}, `foo bar {}`},
{[]token.Type{token.IDENT, token.STRING}, `foo "bar" {}`},
{[]token.Type{token.STRING, token.IDENT}, `"foo" bar {}`},
{[]token.Type{token.IDENT, token.IDENT, token.IDENT}, `foo bar baz {}`},
}
for _, k := range keys {
p := newParser([]byte(k.src))
keys, err := p.objectKey()
if err != nil {
t.Fatal(err)
}
tokens := []token.Type{}
for _, o := range keys {
tokens = append(tokens, o.Token.Type)
}
equals(t, k.exp, tokens)
}
errKeys := []struct {
src string
}{
{`foo 12 {}`},
{`foo bar = {}`},
{`foo []`},
{`12 {}`},
}
for _, k := range errKeys {
p := newParser([]byte(k.src))
_, err := p.objectKey()
if err == nil {
t.Errorf("case '%s' should give an error", k.src)
}
}
}
// Official HCL tests
func TestParse(t *testing.T) {
cases := []struct {
Name string
Err bool
}{
{
"assign_colon.hcl",
true,
},
{
"comment.hcl",
false,
},
{
"comment_single.hcl",
false,
},
{
"empty.hcl",
false,
},
{
"list_comma.hcl",
false,
},
{
"multiple.hcl",
false,
},
{
"structure.hcl",
false,
},
{
"structure_basic.hcl",
false,
},
{
"structure_empty.hcl",
false,
},
{
"complex.hcl",
false,
},
{
"assign_deep.hcl",
true,
},
{
"types.hcl",
false,
},
{
"array_comment.hcl",
false,
},
}
const fixtureDir = "./test-fixtures"
for _, tc := range cases {
d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.Name))
if err != nil {
t.Fatalf("err: %s", err)
}
_, err = Parse(d)
if (err != nil) != tc.Err {
t.Fatalf("Input: %s\n\nError: %s", tc.Name, err)
}
}
}
// equals fails the test if exp is not equal to act.
func equals(tb testing.TB, exp, act interface{}) {
if !reflect.DeepEqual(exp, act) {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
tb.FailNow()
}
}

4
hclnew/parser/test-fixtures/array_comment.hcl
View File

@ -1,4 +0,0 @@
foo = [
"1",
"2", # comment
]

6
hclnew/parser/test-fixtures/assign_colon.hcl
View File

@ -1,6 +0,0 @@
resource = [{
"foo": {
"bar": {},
"baz": [1, 2, "foo"],
}
}]

5
hclnew/parser/test-fixtures/assign_deep.hcl
View File

@ -1,5 +0,0 @@
resource = [{
foo = [{
bar = {}
}]
}]

15
hclnew/parser/test-fixtures/comment.hcl
View File

@ -1,15 +0,0 @@
// Foo
/* Bar */
/*
/*
Baz
*/
# Another
# Multiple
# Lines
foo = "bar"

1
hclnew/parser/test-fixtures/comment_single.hcl
View File

@ -1 +0,0 @@
# Hello

39
hclnew/parser/test-fixtures/complex.hcl
View File

@ -1,39 +0,0 @@
variable "foo" {
default = "bar"
description = "bar"
}
provider "aws" {
access_key = "foo"
secret_key = "bar"
}
provider "do" {
api_key = "${var.foo}"
}
resource "aws_security_group" "firewall" {
count = 5
}
resource aws_instance "web" {
ami = "${var.foo}"
security_groups = [
"foo",
"${aws_security_group.firewall.foo}",
]
network_interface = {
device_index = 0
description = "Main network interface"
}
}
resource "aws_instance" "db" {
security_groups = "${aws_security_group.firewall.*.id}"
VPC = "foo"
depends_on = ["aws_instance.web"]
}
output "web_ip" {
value = "${aws_instance.web.private_ip}"
}

1
hclnew/parser/test-fixtures/complex_key.hcl
View File

@ -1 +0,0 @@
foo.bar = "baz"

0
hclnew/parser/test-fixtures/empty.hcl
View File

1
hclnew/parser/test-fixtures/list.hcl
View File

@ -1 +0,0 @@
foo = [1, 2, "foo"]

1
hclnew/parser/test-fixtures/list_comma.hcl
View File

@ -1 +0,0 @@
foo = [1, 2, "foo",]

2
hclnew/parser/test-fixtures/multiple.hcl
View File

@ -1,2 +0,0 @@
foo = "bar"
key = 7

3
hclnew/parser/test-fixtures/old.hcl
View File

@ -1,3 +0,0 @@
default = {
"eu-west-1": "ami-b1cf19c6",
}

5
hclnew/parser/test-fixtures/structure.hcl
View File

@ -1,5 +0,0 @@
// This is a test structure for the lexer
foo bar "baz" {
key = 7
foo = "bar"
}

5
hclnew/parser/test-fixtures/structure_basic.hcl
View File

@ -1,5 +0,0 @@
foo {
value = 7
"value" = 8
"complex::value" = 9
}

1
hclnew/parser/test-fixtures/structure_empty.hcl
View File

@ -1 +0,0 @@
resource "foo" "bar" {}

7
hclnew/parser/test-fixtures/types.hcl
View File

@ -1,7 +0,0 @@
foo = "bar"
bar = 7
baz = [1,2,3]
foo = -12
bar = 3.14159
foo = true
bar = false

509
hclnew/printer/nodes.go
View File

@ -1,509 +0,0 @@
package printer
import (
"bytes"
"fmt"
"sort"
"github.com/fatih/hcl/ast"
"github.com/fatih/hcl/token"
)
const (
blank = byte(' ')
newline = byte('\n')
tab = byte('\t')
infinity = 1 << 30 // offset or line
)
type printer struct {
cfg Config
prev token.Pos
comments []*ast.CommentGroup // may be nil, contains all comments
standaloneComments []*ast.CommentGroup // contains all standalone comments (not assigned to any node)
enableTrace bool
indentTrace int
}
type ByPosition []*ast.CommentGroup
func (b ByPosition) Len() int { return len(b) }
func (b ByPosition) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b ByPosition) Less(i, j int) bool { return b[i].Pos().Before(b[j].Pos()) }
// collectComments comments all standalone comments which are not lead or line
// comment
func (p *printer) collectComments(node ast.Node) {
// first collect all comments. This is already stored in
// ast.File.(comments)
ast.Walk(node, func(nn ast.Node) bool {
switch t := nn.(type) {
case *ast.File:
p.comments = t.Comments
return false
}
return true
})
standaloneComments := make(map[token.Pos]*ast.CommentGroup, 0)
for _, c := range p.comments {
standaloneComments[c.Pos()] = c
}
// next remove all lead and line comments from the overall comment map.
// This will give us comments which are standalone, comments which are not
// assigned to any kind of node.
ast.Walk(node, func(nn ast.Node) bool {
switch t := nn.(type) {
case *ast.LiteralType:
if t.LineComment != nil {
for _, comment := range t.LineComment.List {
if _, ok := standaloneComments[comment.Pos()]; ok {
delete(standaloneComments, comment.Pos())
}
}
}
case *ast.ObjectItem:
if t.LeadComment != nil {
for _, comment := range t.LeadComment.List {
if _, ok := standaloneComments[comment.Pos()]; ok {
delete(standaloneComments, comment.Pos())
}
}
}
if t.LineComment != nil {
for _, comment := range t.LineComment.List {
if _, ok := standaloneComments[comment.Pos()]; ok {
delete(standaloneComments, comment.Pos())
}
}
}
}
return true
})
for _, c := range standaloneComments {
p.standaloneComments = append(p.standaloneComments, c)
}
sort.Sort(ByPosition(p.standaloneComments))
}
// output prints creates b printable HCL output and returns it.
func (p *printer) output(n interface{}) []byte {
var buf bytes.Buffer
switch t := n.(type) {
case *ast.File:
return p.output(t.Node)
case *ast.ObjectList:
var index int
var nextItem token.Pos
var commented bool
for {
// TODO(arslan): refactor below comment printing, we have the same in objectType
for _, c := range p.standaloneComments {
for _, comment := range c.List {
if index != len(t.Items) {
nextItem = t.Items[index].Pos()
} else {
nextItem = token.Pos{Offset: infinity, Line: infinity}
}
if comment.Pos().After(p.prev) && comment.Pos().Before(nextItem) {
// if we hit the end add newlines so we can print the comment
if index == len(t.Items) {
buf.Write([]byte{newline, newline})
}
buf.WriteString(comment.Text)
buf.WriteByte(newline)
if index != len(t.Items) {
buf.WriteByte(newline)
}
}
}
}
if index == len(t.Items) {
break
}
buf.Write(p.output(t.Items[index]))
if !commented && index != len(t.Items)-1 {
buf.Write([]byte{newline, newline})
}
index++
}
case *ast.ObjectKey:
buf.WriteString(t.Token.Text)
case *ast.ObjectItem:
p.prev = t.Pos()
buf.Write(p.objectItem(t))
case *ast.LiteralType:
buf.WriteString(t.Token.Text)
case *ast.ListType:
buf.Write(p.list(t))
case *ast.ObjectType:
buf.Write(p.objectType(t))
default:
fmt.Printf(" unknown type: %T\n", n)
}
return buf.Bytes()
}
// objectItem returns the printable HCL form of an object item. An object type
// starts with one/multiple keys and has a value. The value might be of any
// type.
func (p *printer) objectItem(o *ast.ObjectItem) []byte {
defer un(trace(p, fmt.Sprintf("ObjectItem: %s", o.Keys[0].Token.Text)))
var buf bytes.Buffer
if o.LeadComment != nil {
for _, comment := range o.LeadComment.List {
buf.WriteString(comment.Text)
buf.WriteByte(newline)
}
}
for i, k := range o.Keys {
buf.WriteString(k.Token.Text)
buf.WriteByte(blank)
// reach end of key
if i == len(o.Keys)-1 && len(o.Keys) == 1 {
buf.WriteString("=")
buf.WriteByte(blank)
}
}
buf.Write(p.output(o.Val))
if o.Val.Pos().Line == o.Keys[0].Pos().Line && o.LineComment != nil {
buf.WriteByte(blank)
for _, comment := range o.LineComment.List {
buf.WriteString(comment.Text)
}
}
return buf.Bytes()
}
// objectType returns the printable HCL form of an object type. An object type
// begins with a brace and ends with a brace.
func (p *printer) objectType(o *ast.ObjectType) []byte {
defer un(trace(p, "ObjectType"))
var buf bytes.Buffer
buf.WriteString("{")
buf.WriteByte(newline)
var index int
var nextItem token.Pos
var commented bool
for {
// Print stand alone comments
for _, c := range p.standaloneComments {
for _, comment := range c.List {
// if we hit the end, last item should be the brace
if index != len(o.List.Items) {
nextItem = o.List.Items[index].Pos()
} else {
nextItem = o.Rbrace
}
if comment.Pos().After(p.prev) && comment.Pos().Before(nextItem) {
// add newline if it's between other printed nodes
if index > 0 {
commented = true
buf.WriteByte(newline)
}
buf.Write(p.indent([]byte(comment.Text)))
buf.WriteByte(newline)
if index != len(o.List.Items) {
buf.WriteByte(newline) // do not print on the end
}
}
}
}
if index == len(o.List.Items) {
p.prev = o.Rbrace
break
}
// check if we have adjacent one liner items. If yes we'll going to align
// the comments.
var aligned []*ast.ObjectItem
for _, item := range o.List.Items[index:] {
// we don't group one line lists
if len(o.List.Items) == 1 {
break
}
// one means a oneliner with out any lead comment
// two means a oneliner with lead comment
// anything else might be something else
cur := lines(string(p.objectItem(item)))
if cur > 2 {
break
}
curPos := item.Pos()
nextPos := token.Pos{}
if index != len(o.List.Items)-1 {
nextPos = o.List.Items[index+1].Pos()
}
prevPos := token.Pos{}
if index != 0 {
prevPos = o.List.Items[index-1].Pos()
}
// fmt.Println("DEBUG ----------------")
// fmt.Printf("prev = %+v prevPos: %s\n", prev, prevPos)
// fmt.Printf("cur = %+v curPos: %s\n", cur, curPos)
// fmt.Printf("next = %+v nextPos: %s\n", next, nextPos)
if curPos.Line+1 == nextPos.Line {
aligned = append(aligned, item)
index++
continue
}
if curPos.Line-1 == prevPos.Line {
aligned = append(aligned, item)
index++
// finish if we have a new line or comment next. This happens
// if the next item is not adjacent
if curPos.Line+1 != nextPos.Line {
break
}
continue
}
break
}
// put newlines if the items are between other non aligned items.
// newlines are also added if there is a standalone comment already, so
// check it too
if !commented && index != len(aligned) {
buf.WriteByte(newline)
}
if len(aligned) >= 1 {
p.prev = aligned[len(aligned)-1].Pos()
items := p.alignedItems(aligned)
buf.Write(p.indent(items))
} else {
p.prev = o.List.Items[index].Pos()
buf.Write(p.indent(p.objectItem(o.List.Items[index])))
index++
}
buf.WriteByte(newline)
}
buf.WriteString("}")
return buf.Bytes()
}
func (p *printer) alignedItems(items []*ast.ObjectItem) []byte {
var buf bytes.Buffer
// find the longest key and value length, needed for alignment
var longestKeyLen int // longest key length
var longestValLen int // longest value length
for _, item := range items {
key := len(item.Keys[0].Token.Text)
val := len(p.output(item.Val))
if key > longestKeyLen {
longestKeyLen = key
}
if val > longestValLen {
longestValLen = val
}
}
for i, item := range items {
if item.LeadComment != nil {
for _, comment := range item.LeadComment.List {
buf.WriteString(comment.Text)
buf.WriteByte(newline)
}
}
for i, k := range item.Keys {
keyLen := len(k.Token.Text)
buf.WriteString(k.Token.Text)
for i := 0; i < longestKeyLen-keyLen+1; i++ {
buf.WriteByte(blank)
}
// reach end of key
if i == len(item.Keys)-1 && len(item.Keys) == 1 {
buf.WriteString("=")
buf.WriteByte(blank)
}
}
val := p.output(item.Val)
valLen := len(val)
buf.Write(val)
if item.Val.Pos().Line == item.Keys[0].Pos().Line && item.LineComment != nil {
for i := 0; i < longestValLen-valLen+1; i++ {
buf.WriteByte(blank)
}
for _, comment := range item.LineComment.List {
buf.WriteString(comment.Text)
}
}
// do not print for the last item
if i != len(items)-1 {
buf.WriteByte(newline)
}
}
return buf.Bytes()
}
// list returns the printable HCL form of an list type.
func (p *printer) list(l *ast.ListType) []byte {
var buf bytes.Buffer
buf.WriteString("[")
var longestLine int
for _, item := range l.List {
// for now we assume that the list only contains literal types
if lit, ok := item.(*ast.LiteralType); ok {
lineLen := len(lit.Token.Text)
if lineLen > longestLine {
longestLine = lineLen
}
}
}
for i, item := range l.List {
if item.Pos().Line != l.Lbrack.Line {
// multiline list, add newline before we add each item
buf.WriteByte(newline)
// also indent each line
val := p.output(item)
curLen := len(val)
buf.Write(p.indent(val))
buf.WriteString(",")
if lit, ok := item.(*ast.LiteralType); ok && lit.LineComment != nil {
// if the next item doesn't have any comments, do not align
buf.WriteByte(blank) // align one space
if i != len(l.List)-1 {
if lit, ok := l.List[i+1].(*ast.LiteralType); ok && lit.LineComment != nil {
for i := 0; i < longestLine-curLen; i++ {
buf.WriteByte(blank)
}
}
}
for _, comment := range lit.LineComment.List {
buf.WriteString(comment.Text)
}
}
if i == len(l.List)-1 {
buf.WriteByte(newline)
}
} else {
buf.Write(p.output(item))
if i != len(l.List)-1 {
buf.WriteString(",")
buf.WriteByte(blank)
}
}
}
buf.WriteString("]")
return buf.Bytes()
}
// indent indents the lines of the given buffer for each non-empty line
func (p *printer) indent(buf []byte) []byte {
var prefix []byte
if p.cfg.SpacesWidth != 0 {
for i := 0; i < p.cfg.SpacesWidth; i++ {
prefix = append(prefix, blank)
}
} else {
prefix = []byte{tab}
}
var res []byte
bol := true
for _, c := range buf {
if bol && c != '\n' {
res = append(res, prefix...)
}
res = append(res, c)
bol = c == '\n'
}
return res
}
func lines(txt string) int {
endline := 1
for i := 0; i < len(txt); i++ {
if txt[i] == '\n' {
endline++
}
}
return endline
}
// ----------------------------------------------------------------------------
// Tracing support
func (p *printer) printTrace(a ...interface{}) {
if !p.enableTrace {
return
}
const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
const n = len(dots)
i := 2 * p.indentTrace
for i > n {
fmt.Print(dots)
i -= n
}
// i <= n
fmt.Print(dots[0:i])
fmt.Println(a...)
}
func trace(p *printer, msg string) *printer {
p.printTrace(msg, "(")
p.indentTrace++
return p
}
// Usage pattern: defer un(trace(p, "..."))
func un(p *printer) {
p.indentTrace--
p.printTrace(")")
}

64
hclnew/printer/printer.go
View File

@ -1,64 +0,0 @@
// Package printer implements printing of AST nodes to HCL format.
package printer
import (
"bytes"
"io"
"text/tabwriter"
"github.com/fatih/hcl/ast"
"github.com/fatih/hcl/parser"
)
var DefaultConfig = Config{
SpacesWidth: 2,
}
// A Config node controls the output of Fprint.
type Config struct {
SpacesWidth int // if set, it will use spaces instead of tabs for alignment
}
func (c *Config) Fprint(output io.Writer, node ast.Node) error {
p := &printer{
cfg: *c,
comments: make([]*ast.CommentGroup, 0),
standaloneComments: make([]*ast.CommentGroup, 0),
// enableTrace: true,
}
p.collectComments(node)
if _, err := output.Write(p.output(node)); err != nil {
return err
}
// flush tabwriter, if any
var err error
if tw, _ := output.(*tabwriter.Writer); tw != nil {
err = tw.Flush()
}
return err
}
// Fprint "pretty-prints" an HCL node to output
// It calls Config.Fprint with default settings.
func Fprint(output io.Writer, node ast.Node) error {
return DefaultConfig.Fprint(output, node)
}
// Format formats src HCL and returns the result.
func Format(src []byte) ([]byte, error) {
node, err := parser.Parse(src)
if err != nil {
return nil, err
}
var buf bytes.Buffer
if err := DefaultConfig.Fprint(&buf, node); err != nil {
return nil, err
}
return buf.Bytes(), nil
}

143
hclnew/printer/printer_test.go
View File

@ -1,143 +0,0 @@
package printer
import (
"bytes"
"errors"
"flag"
"fmt"
"io/ioutil"
"path/filepath"
"testing"
"github.com/fatih/hcl/parser"
)
var update = flag.Bool("update", false, "update golden files")
const (
dataDir = "testdata"
)
type entry struct {
source, golden string
}
// Use go test -update to create/update the respective golden files.
var data = []entry{
{"complexhcl.input", "complexhcl.golden"},
{"list.input", "list.golden"},
{"comment.input", "comment.golden"},
{"comment_aligned.input", "comment_aligned.golden"},
{"comment_standalone.input", "comment_standalone.golden"},
}
func TestFiles(t *testing.T) {
for _, e := range data {
source := filepath.Join(dataDir, e.source)
golden := filepath.Join(dataDir, e.golden)
check(t, source, golden)
}
}
func check(t *testing.T, source, golden string) {
src, err := ioutil.ReadFile(source)
if err != nil {
t.Error(err)
return
}
res, err := format(src)
if err != nil {
t.Error(err)
return
}
// update golden files if necessary
if *update {
if err := ioutil.WriteFile(golden, res, 0644); err != nil {
t.Error(err)
}
return
}
// get golden
gld, err := ioutil.ReadFile(golden)
if err != nil {
t.Error(err)
return
}
// formatted source and golden must be the same
if err := diff(source, golden, res, gld); err != nil {
t.Error(err)
return
}
}
// diff compares a and b.
func diff(aname, bname string, a, b []byte) error {
var buf bytes.Buffer // holding long error message
// compare lengths
if len(a) != len(b) {
fmt.Fprintf(&buf, "\nlength changed: len(%s) = %d, len(%s) = %d", aname, len(a), bname, len(b))
}
// compare contents
line := 1
offs := 1
for i := 0; i < len(a) && i < len(b); i++ {
ch := a[i]
if ch != b[i] {
fmt.Fprintf(&buf, "\n%s:%d:%d: %s", aname, line, i-offs+1, lineAt(a, offs))
fmt.Fprintf(&buf, "\n%s:%d:%d: %s", bname, line, i-offs+1, lineAt(b, offs))
fmt.Fprintf(&buf, "\n\n")
break
}
if ch == '\n' {
line++
offs = i + 1
}
}
if buf.Len() > 0 {
return errors.New(buf.String())
}
return nil
}
// format parses src, prints the corresponding AST, verifies the resulting
// src is syntactically correct, and returns the resulting src or an error
// if any.
func format(src []byte) ([]byte, error) {
// parse src
node, err := parser.Parse(src)
if err != nil {
return nil, fmt.Errorf("parse: %s\n%s", err, src)
}
var buf bytes.Buffer
cfg := &Config{}
if err := cfg.Fprint(&buf, node); err != nil {
return nil, fmt.Errorf("print: %s", err)
}
// make sure formatted output is syntactically correct
res := buf.Bytes()
if _, err := parser.Parse(src); err != nil {
return nil, fmt.Errorf("parse: %s\n%s", err, src)
}
return res, nil
}
// lineAt returns the line in text starting at offset offs.
func lineAt(text []byte, offs int) []byte {
i := offs
for i < len(text) && text[i] != '\n' {
i++
}
return text[offs:i]
}

36
hclnew/printer/testdata/comment.golden vendored
View File

@ -1,36 +0,0 @@
// A standalone comment is a comment which is not attached to any kind of node
// This comes from Terraform, as a test
variable "foo" {
# Standalone comment should be still here
default = "bar"
description = "bar" # yooo
}
/* This is a multi line standalone
comment*/
// fatih arslan
/* This is a developer test
account and a multine comment */
developer = ["fatih", "arslan"] // fatih arslan
# One line here
numbers = [1, 2] // another line here
# Another comment
variable = {
description = "bar" # another yooo
foo = {
# Nested standalone
bar = "fatih"
}
}
// lead comment
foo = {
bar = "fatih" // line comment 2
} // line comment 3

37
hclnew/printer/testdata/comment.input vendored
View File

@ -1,37 +0,0 @@
// A standalone comment is a comment which is not attached to any kind of node
// This comes from Terraform, as a test
variable "foo" {
# Standalone comment should be still here
default = "bar"
description = "bar" # yooo
}
/* This is a multi line standalone
comment*/
// fatih arslan
/* This is a developer test
account and a multine comment */
developer = [ "fatih", "arslan"] // fatih arslan
# One line here
numbers = [1,2] // another line here
# Another comment
variable = {
description = "bar" # another yooo
foo {
# Nested standalone
bar = "fatih"
}
}
// lead comment
foo {
bar = "fatih" // line comment 2
} // line comment 3

25
hclnew/printer/testdata/comment_aligned.golden vendored
View File

@ -1,25 +0,0 @@
aligned = {
# We have some aligned items below
foo = "fatih" # yoo1
default = "bar" # yoo2
bar = "bar and foo" # yoo3
default = {
bar = "example"
}
#deneme arslan
fatih = ["fatih"] # yoo4
#fatih arslan
fatiharslan = ["arslan"] // yoo5
default = {
bar = "example"
}
security_groups = [
"foo", # kenya 1
"${aws_security_group.firewall.foo}", # kenya 2
]
}

21
hclnew/printer/testdata/comment_aligned.input vendored
View File

@ -1,21 +0,0 @@
aligned {
# We have some aligned items below
foo = "fatih" # yoo1
default = "bar" # yoo2
bar = "bar and foo" # yoo3
default = {
bar = "example"
}
#deneme arslan
fatih = ["fatih"] # yoo4
#fatih arslan
fatiharslan = ["arslan"] // yoo5
default = {
bar = "example"
}
security_groups = [
"foo", # kenya 1
"${aws_security_group.firewall.foo}", # kenya 2
]
}

16
hclnew/printer/testdata/comment_standalone.golden vendored
View File

@ -1,16 +0,0 @@
// A standalone comment
aligned = {
# Standalone 1
a = "bar" # yoo1
default = "bar" # yoo2
# Standalone 2
}
# Standalone 3
numbers = [1, 2] // another line here
# Standalone 4

16
hclnew/printer/testdata/comment_standalone.input vendored
View File

@ -1,16 +0,0 @@
// A standalone comment
aligned {
# Standalone 1
a = "bar" # yoo1
default = "bar" # yoo2
# Standalone 2
}
# Standalone 3
numbers = [1,2] // another line here
# Standalone 4

44
hclnew/printer/testdata/complexhcl.golden vendored
View File

@ -1,44 +0,0 @@
variable "foo" {
default = "bar"
description = "bar"
}
developer = ["fatih", "arslan"]
provider "aws" {
access_key = "foo"
secret_key = "bar"
}
provider "do" {
api_key = "${var.foo}"
}
resource "aws_security_group" "firewall" {
count = 5
}
resource aws_instance "web" {
ami = "${var.foo}"
security_groups = [
"foo",
"${aws_security_group.firewall.foo}",
]
network_interface = {
device_index = 0
description = "Main network interface"
}
}
resource "aws_instance" "db" {
security_groups = "${aws_security_group.firewall.*.id}"
VPC = "foo"
depends_on = ["aws_instance.web"]
}
output "web_ip" {
value = "${aws_instance.web.private_ip}"
}

44
hclnew/printer/testdata/complexhcl.input vendored
View File

@ -1,44 +0,0 @@
variable "foo" {
default = "bar"
description = "bar"
}
developer = [ "fatih", "arslan"]
provider "aws" {
access_key ="foo"
secret_key = "bar"
}
provider "do" {
api_key = "${var.foo}"
}
resource "aws_security_group" "firewall" {
count = 5
}
resource aws_instance "web" {
ami = "${var.foo}"
security_groups = [
"foo",
"${aws_security_group.firewall.foo}"
]
network_interface {
device_index = 0
description = "Main network interface"
}
}
resource "aws_instance" "db" {
security_groups = "${aws_security_group.firewall.*.id}"
VPC = "foo"
depends_on = ["aws_instance.web"]
}
output "web_ip" {
value="${aws_instance.web.private_ip}"
}

27
hclnew/printer/testdata/list.golden vendored
View File

@ -1,27 +0,0 @@
foo = ["fatih", "arslan"]
foo = ["bar", "qaz"]
foo = ["zeynep",
"arslan",
]
foo = ["fatih", "zeynep",
"arslan",
]
foo = [
"vim-go",
"golang",
"hcl",
]
foo = []
foo = [1, 2, 3, 4]
foo = [
"kenya",
"ethiopia",
"columbia",
]

21
hclnew/printer/testdata/list.input vendored
View File

@ -1,21 +0,0 @@
foo = ["fatih", "arslan" ]
foo = [ "bar", "qaz", ]
foo = [ "zeynep",
"arslan", ]
foo = ["fatih", "zeynep",
"arslan", ]
foo = [
"vim-go",
"golang", "hcl"]
foo = []
foo = [1, 2,3, 4]
foo = [
"kenya", "ethiopia",
"columbia"]

529
hclnew/scanner/scanner.go
View File

@ -1,529 +0,0 @@
// Package scanner implements a scanner for HCL (HashiCorp Configuration
// Language) source text.
package scanner
import (
"bytes"
"fmt"
"os"
"unicode"
"unicode/utf8"
"github.com/fatih/hcl/token"
)
// eof represents a marker rune for the end of the reader.
const eof = rune(0)
// Scanner defines a lexical scanner
type Scanner struct {
buf *bytes.Buffer // Source buffer for advancing and scanning
src []byte // Source buffer for immutable access
// Source Position
srcPos token.Pos // current position
prevPos token.Pos // previous position, used for peek() method
lastCharLen int // length of last character in bytes
lastLineLen int // length of last line in characters (for correct column reporting)
tokStart int // token text start position
tokEnd int // token text end position
// Error is called for each error encountered. If no Error
// function is set, the error is reported to os.Stderr.
Error func(pos token.Pos, msg string)
// ErrorCount is incremented by one for each error encountered.
ErrorCount int
// tokPos is the start position of most recently scanned token; set by
// Scan. The Filename field is always left untouched by the Scanner. If
// an error is reported (via Error) and Position is invalid, the scanner is
// not inside a token.
tokPos token.Pos
}
// New creates and initializes a new instance of Scanner using src as
// its source content.
func New(src []byte) *Scanner {
// even though we accept a src, we read from a io.Reader compatible type
// (*bytes.Buffer). So in the future we might easily change it to streaming
// read.
b := bytes.NewBuffer(src)
s := &Scanner{
buf: b,
src: src,
}
// srcPosition always starts with 1
s.srcPos.Line = 1
return s
}
// next reads the next rune from the bufferred reader. Returns the rune(0) if
// an error occurs (or io.EOF is returned).
func (s *Scanner) next() rune {
ch, size, err := s.buf.ReadRune()
if err != nil {
// advance for error reporting
s.srcPos.Column++
s.srcPos.Offset += size
s.lastCharLen = size
return eof
}
if ch == utf8.RuneError && size == 1 {
s.srcPos.Column++
s.srcPos.Offset += size
s.lastCharLen = size
s.err("illegal UTF-8 encoding")
return ch
}
// remember last position
s.prevPos = s.srcPos
s.srcPos.Column++
s.lastCharLen = size
s.srcPos.Offset += size
if ch == '\n' {
s.srcPos.Line++
s.lastLineLen = s.srcPos.Column
s.srcPos.Column = 0
}
// debug
// fmt.Printf("ch: %q, offset:column: %d:%d\n", ch, s.srcPos.Offset, s.srcPos.Column)
return ch
}
// unread unreads the previous read Rune and updates the source position
func (s *Scanner) unread() {
if err := s.buf.UnreadRune(); err != nil {
panic(err) // this is user fault, we should catch it
}
s.srcPos = s.prevPos // put back last position
}
// peek returns the next rune without advancing the reader.
func (s *Scanner) peek() rune {
peek, _, err := s.buf.ReadRune()
if err != nil {
return eof
}
s.buf.UnreadRune()
return peek
}
// Scan scans the next token and returns the token.
func (s *Scanner) Scan() token.Token {
ch := s.next()
// skip white space
for isWhitespace(ch) {
ch = s.next()
}
var tok token.Type
// token text markings
s.tokStart = s.srcPos.Offset - s.lastCharLen
// token position, initial next() is moving the offset by one(size of rune
// actually), though we are interested with the starting point
s.tokPos.Offset = s.srcPos.Offset - s.lastCharLen
if s.srcPos.Column > 0 {
// common case: last character was not a '\n'
s.tokPos.Line = s.srcPos.Line
s.tokPos.Column = s.srcPos.Column
} else {
// last character was a '\n'
// (we cannot be at the beginning of the source
// since we have called next() at least once)
s.tokPos.Line = s.srcPos.Line - 1
s.tokPos.Column = s.lastLineLen
}
switch {
case isLetter(ch):
tok = token.IDENT
lit := s.scanIdentifier()
if lit == "true" || lit == "false" {
tok = token.BOOL
}
case isDecimal(ch):
tok = s.scanNumber(ch)
default:
switch ch {
case eof:
tok = token.EOF
case '"':
tok = token.STRING
s.scanString()
case '#', '/':
tok = token.COMMENT
s.scanComment(ch)
case '.':
tok = token.PERIOD
ch = s.peek()
if isDecimal(ch) {
tok = token.FLOAT
ch = s.scanMantissa(ch)
ch = s.scanExponent(ch)
}
case '[':
tok = token.LBRACK
case ']':
tok = token.RBRACK
case '{':
tok = token.LBRACE
case '}':
tok = token.RBRACE
case ',':
tok = token.COMMA
case '=':
tok = token.ASSIGN
case '+':
tok = token.ADD
case '-':
if isDecimal(s.peek()) {
ch := s.next()
tok = s.scanNumber(ch)
} else {
tok = token.SUB
}
default:
s.err("illegal char")
}
}
// finish token ending
s.tokEnd = s.srcPos.Offset
// create token literal
var tokenText string
if s.tokStart >= 0 {
tokenText = string(s.src[s.tokStart:s.tokEnd])
}
s.tokStart = s.tokEnd // ensure idempotency of tokenText() call
return token.Token{
Type: tok,
Pos: s.tokPos,
Text: tokenText,
}
}
func (s *Scanner) scanComment(ch rune) {
// single line comments
if ch == '#' || (ch == '/' && s.peek() != '*') {
ch = s.next()
for ch != '\n' && ch >= 0 {
ch = s.next()
}
s.unread()
return
}
// be sure we get the character after /* This allows us to find comment's
// that are not erminated
if ch == '/' {
s.next()
ch = s.next() // read character after "/*"
}
// look for /* - style comments
for {
if ch < 0 || ch == eof {
s.err("comment not terminated")
break
}
ch0 := ch
ch = s.next()
if ch0 == '*' && ch == '/' {
break
}
}
}
// scanNumber scans a HCL number definition starting with the given rune
func (s *Scanner) scanNumber(ch rune) token.Type {
if ch == '0' {
// check for hexadecimal, octal or float
ch = s.next()
if ch == 'x' || ch == 'X' {
// hexadecimal
ch = s.next()
found := false
for isHexadecimal(ch) {
ch = s.next()
found = true
}
if !found {
s.err("illegal hexadecimal number")
}
if ch != eof {
s.unread()
}
return token.NUMBER
}
// now it's either something like: 0421(octal) or 0.1231(float)
illegalOctal := false
for isDecimal(ch) {
ch = s.next()
if ch == '8' || ch == '9' {
// this is just a possibility. For example 0159 is illegal, but
// 0159.23 is valid. So we mark a possible illegal octal. If
// the next character is not a period, we'll print the error.
illegalOctal = true
}
}
// literals of form 01e10 are treates as Numbers in HCL, which differs from Go.
if ch == 'e' || ch == 'E' {
ch = s.scanExponent(ch)
return token.NUMBER
}
if ch == '.' {
ch = s.scanFraction(ch)
if ch == 'e' || ch == 'E' {
ch = s.next()
ch = s.scanExponent(ch)
}
return token.FLOAT
}
if illegalOctal {
s.err("illegal octal number")
}
if ch != eof {
s.unread()
}
return token.NUMBER
}
s.scanMantissa(ch)
ch = s.next() // seek forward
// literals of form 1e10 are treates as Numbers in HCL, which differs from Go.
if ch == 'e' || ch == 'E' {
ch = s.scanExponent(ch)
return token.NUMBER
}
if ch == '.' {
ch = s.scanFraction(ch)
if ch == 'e' || ch == 'E' {
ch = s.next()
ch = s.scanExponent(ch)
}
return token.FLOAT
}
if ch != eof {
s.unread()
}
return token.NUMBER
}
// scanMantissa scans the mantissa begining from the rune. It returns the next
// non decimal rune. It's used to determine wheter it's a fraction or exponent.
func (s *Scanner) scanMantissa(ch rune) rune {
scanned := false
for isDecimal(ch) {
ch = s.next()
scanned = true
}
if scanned && ch != eof {
s.unread()
}
return ch
}
// scanFraction scans the fraction after the '.' rune
func (s *Scanner) scanFraction(ch rune) rune {
if ch == '.' {
ch = s.peek() // we peek just to see if we can move forward
ch = s.scanMantissa(ch)
}
return ch
}
// scanExponent scans the remaining parts of an exponent after the 'e' or 'E'
// rune.
func (s *Scanner) scanExponent(ch rune) rune {
if ch == 'e' || ch == 'E' {
ch = s.next()
if ch == '-' || ch == '+' {
ch = s.next()
}
ch = s.scanMantissa(ch)
}
return ch
}
// scanString scans a quoted string
func (s *Scanner) scanString() {
for {
// '"' opening already consumed
// read character after quote
ch := s.next()
if ch == '\n' || ch < 0 || ch == eof {
s.err("literal not terminated")
return
}
if ch == '"' {
break
}
if ch == '\\' {
s.scanEscape()
}
}
return
}
// scanEscape scans an escape sequence
func (s *Scanner) scanEscape() rune {
// http://en.cppreference.com/w/cpp/language/escape
ch := s.next() // read character after '/'
switch ch {
case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', '"':
// nothing to do
case '0', '1', '2', '3', '4', '5', '6', '7':
// octal notation
ch = s.scanDigits(ch, 8, 3)
case 'x':
// hexademical notation
ch = s.scanDigits(s.next(), 16, 2)
case 'u':
// universal character name
ch = s.scanDigits(s.next(), 16, 4)
case 'U':
// universal character name
ch = s.scanDigits(s.next(), 16, 8)
default:
s.err("illegal char escape")
}
return ch
}
// scanDigits scans a rune with the given base for n times. For example an
// octal notation \184 would yield in scanDigits(ch, 8, 3)
func (s *Scanner) scanDigits(ch rune, base, n int) rune {
for n > 0 && digitVal(ch) < base {
ch = s.next()
n--
}
if n > 0 {
s.err("illegal char escape")
}
// we scanned all digits, put the last non digit char back
s.unread()
return ch
}
// scanIdentifier scans an identifier and returns the literal string
func (s *Scanner) scanIdentifier() string {
offs := s.srcPos.Offset - s.lastCharLen
ch := s.next()
for isLetter(ch) || isDigit(ch) {
ch = s.next()
}
if ch != eof {
s.unread() // we got identifier, put back latest char
}
return string(s.src[offs:s.srcPos.Offset])
}
// recentPosition returns the position of the character immediately after the
// character or token returned by the last call to Scan.
func (s *Scanner) recentPosition() (pos token.Pos) {
pos.Offset = s.srcPos.Offset - s.lastCharLen
switch {
case s.srcPos.Column > 0:
// common case: last character was not a '\n'
pos.Line = s.srcPos.Line
pos.Column = s.srcPos.Column
case s.lastLineLen > 0:
// last character was a '\n'
// (we cannot be at the beginning of the source
// since we have called next() at least once)
pos.Line = s.srcPos.Line - 1
pos.Column = s.lastLineLen
default:
// at the beginning of the source
pos.Line = 1
pos.Column = 1
}
return
}
// err prints the error of any scanning to s.Error function. If the function is
// not defined, by default it prints them to os.Stderr
func (s *Scanner) err(msg string) {
s.ErrorCount++
pos := s.recentPosition()
if s.Error != nil {
s.Error(pos, msg)
return
}
fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg)
}
// isHexadecimal returns true if the given rune is a letter
func isLetter(ch rune) bool {
return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch)
}
// isHexadecimal returns true if the given rune is a decimal digit
func isDigit(ch rune) bool {
return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
}
// isHexadecimal returns true if the given rune is a decimal number
func isDecimal(ch rune) bool {
return '0' <= ch && ch <= '9'
}
// isHexadecimal returns true if the given rune is an hexadecimal number
func isHexadecimal(ch rune) bool {
return '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F'
}
// isWhitespace returns true if the rune is a space, tab, newline or carriage return
func isWhitespace(ch rune) bool {
return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r'
}
// digitVal returns the integer value of a given octal,decimal or hexadecimal rune
func digitVal(ch rune) int {
switch {
case '0' <= ch && ch <= '9':
return int(ch - '0')
case 'a' <= ch && ch <= 'f':
return int(ch - 'a' + 10)
case 'A' <= ch && ch <= 'F':
return int(ch - 'A' + 10)
}
return 16 // larger than any legal digit val
}

486
hclnew/scanner/scanner_test.go
View File

@ -1,486 +0,0 @@
package scanner
import (
"bytes"
"fmt"
"testing"
"github.com/fatih/hcl/token"
)
var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
type tokenPair struct {
tok token.Type
text string
}
var tokenLists = map[string][]tokenPair{
"comment": []tokenPair{
{token.COMMENT, "//"},
{token.COMMENT, "////"},
{token.COMMENT, "// comment"},
{token.COMMENT, "// /* comment */"},
{token.COMMENT, "// // comment //"},
{token.COMMENT, "//" + f100},
{token.COMMENT, "#"},
{token.COMMENT, "##"},
{token.COMMENT, "# comment"},
{token.COMMENT, "# /* comment */"},
{token.COMMENT, "# # comment #"},
{token.COMMENT, "#" + f100},
{token.COMMENT, "/**/"},
{token.COMMENT, "/***/"},
{token.COMMENT, "/* comment */"},
{token.COMMENT, "/* // comment */"},
{token.COMMENT, "/* /* comment */"},
{token.COMMENT, "/*\n comment\n*/"},
{token.COMMENT, "/*" + f100 + "*/"},
},
"operator": []tokenPair{
{token.LBRACK, "["},
{token.LBRACE, "{"},
{token.COMMA, ","},
{token.PERIOD, "."},
{token.RBRACK, "]"},
{token.RBRACE, "}"},
{token.ASSIGN, "="},
{token.ADD, "+"},
{token.SUB, "-"},
},
"bool": []tokenPair{
{token.BOOL, "true"},
{token.BOOL, "false"},
},
"ident": []tokenPair{
{token.IDENT, "a"},
{token.IDENT, "a0"},
{token.IDENT, "foobar"},
{token.IDENT, "abc123"},
{token.IDENT, "LGTM"},
{token.IDENT, "_"},
{token.IDENT, "_abc123"},
{token.IDENT, "abc123_"},
{token.IDENT, "_abc_123_"},
{token.IDENT, "_äöü"},
{token.IDENT, "_本"},
{token.IDENT, "äöü"},
{token.IDENT, "本"},
{token.IDENT, "a۰۱۸"},
{token.IDENT, "foo६४"},
{token.IDENT, "bar９８７６"},
},
"string": []tokenPair{
{token.STRING, `" "`},
{token.STRING, `"a"`},
{token.STRING, `"本"`},
{token.STRING, `"\a"`},
{token.STRING, `"\b"`},
{token.STRING, `"\f"`},
{token.STRING, `"\n"`},
{token.STRING, `"\r"`},
{token.STRING, `"\t"`},
{token.STRING, `"\v"`},
{token.STRING, `"\""`},
{token.STRING, `"\000"`},
{token.STRING, `"\777"`},
{token.STRING, `"\x00"`},
{token.STRING, `"\xff"`},
{token.STRING, `"\u0000"`},
{token.STRING, `"\ufA16"`},
{token.STRING, `"\U00000000"`},
{token.STRING, `"\U0000ffAB"`},
{token.STRING, `"` + f100 + `"`},
},
"number": []tokenPair{
{token.NUMBER, "0"},
{token.NUMBER, "1"},
{token.NUMBER, "9"},
{token.NUMBER, "42"},
{token.NUMBER, "1234567890"},
{token.NUMBER, "00"},
{token.NUMBER, "01"},
{token.NUMBER, "07"},
{token.NUMBER, "042"},
{token.NUMBER, "01234567"},
{token.NUMBER, "0x0"},
{token.NUMBER, "0x1"},
{token.NUMBER, "0xf"},
{token.NUMBER, "0x42"},
{token.NUMBER, "0x123456789abcDEF"},
{token.NUMBER, "0x" + f100},
{token.NUMBER, "0X0"},
{token.NUMBER, "0X1"},
{token.NUMBER, "0XF"},
{token.NUMBER, "0X42"},
{token.NUMBER, "0X123456789abcDEF"},
{token.NUMBER, "0X" + f100},
{token.NUMBER, "0e0"},
{token.NUMBER, "1e0"},
{token.NUMBER, "42e0"},
{token.NUMBER, "01234567890e0"},
{token.NUMBER, "0E0"},
{token.NUMBER, "1E0"},
{token.NUMBER, "42E0"},
{token.NUMBER, "01234567890E0"},
{token.NUMBER, "0e+10"},
{token.NUMBER, "1e-10"},
{token.NUMBER, "42e+10"},
{token.NUMBER, "01234567890e-10"},
{token.NUMBER, "0E+10"},
{token.NUMBER, "1E-10"},
{token.NUMBER, "42E+10"},
{token.NUMBER, "01234567890E-10"},
{token.NUMBER, "-0"},
{token.NUMBER, "-1"},
{token.NUMBER, "-9"},
{token.NUMBER, "-42"},
{token.NUMBER, "-1234567890"},
{token.NUMBER, "-00"},
{token.NUMBER, "-01"},
{token.NUMBER, "-07"},
{token.NUMBER, "-042"},
{token.NUMBER, "-01234567"},
{token.NUMBER, "-0x0"},
{token.NUMBER, "-0x1"},
{token.NUMBER, "-0xf"},
{token.NUMBER, "-0x42"},
{token.NUMBER, "-0x123456789abcDEF"},
{token.NUMBER, "-0x" + f100},
{token.NUMBER, "-0X0"},
{token.NUMBER, "-0X1"},
{token.NUMBER, "-0XF"},
{token.NUMBER, "-0X42"},
{token.NUMBER, "-0X123456789abcDEF"},
{token.NUMBER, "-0X" + f100},
{token.NUMBER, "-0e0"},
{token.NUMBER, "-1e0"},
{token.NUMBER, "-42e0"},
{token.NUMBER, "-01234567890e0"},
{token.NUMBER, "-0E0"},
{token.NUMBER, "-1E0"},
{token.NUMBER, "-42E0"},
{token.NUMBER, "-01234567890E0"},
{token.NUMBER, "-0e+10"},
{token.NUMBER, "-1e-10"},
{token.NUMBER, "-42e+10"},
{token.NUMBER, "-01234567890e-10"},
{token.NUMBER, "-0E+10"},
{token.NUMBER, "-1E-10"},
{token.NUMBER, "-42E+10"},
{token.NUMBER, "-01234567890E-10"},
},
"float": []tokenPair{
{token.FLOAT, "0."},
{token.FLOAT, "1."},
{token.FLOAT, "42."},
{token.FLOAT, "01234567890."},
{token.FLOAT, ".0"},
{token.FLOAT, ".1"},
{token.FLOAT, ".42"},
{token.FLOAT, ".0123456789"},
{token.FLOAT, "0.0"},
{token.FLOAT, "1.0"},
{token.FLOAT, "42.0"},
{token.FLOAT, "01234567890.0"},
{token.FLOAT, "01.8e0"},
{token.FLOAT, "1.4e0"},
{token.FLOAT, "42.2e0"},
{token.FLOAT, "01234567890.12e0"},
{token.FLOAT, "0.E0"},
{token.FLOAT, "1.12E0"},
{token.FLOAT, "42.123E0"},
{token.FLOAT, "01234567890.213E0"},
{token.FLOAT, "0.2e+10"},
{token.FLOAT, "1.2e-10"},
{token.FLOAT, "42.54e+10"},
{token.FLOAT, "01234567890.98e-10"},
{token.FLOAT, "0.1E+10"},
{token.FLOAT, "1.1E-10"},
{token.FLOAT, "42.1E+10"},
{token.FLOAT, "01234567890.1E-10"},
{token.FLOAT, "-0.0"},
{token.FLOAT, "-1.0"},
{token.FLOAT, "-42.0"},
{token.FLOAT, "-01234567890.0"},
{token.FLOAT, "-01.8e0"},
{token.FLOAT, "-1.4e0"},
{token.FLOAT, "-42.2e0"},
{token.FLOAT, "-01234567890.12e0"},
{token.FLOAT, "-0.E0"},
{token.FLOAT, "-1.12E0"},
{token.FLOAT, "-42.123E0"},
{token.FLOAT, "-01234567890.213E0"},
{token.FLOAT, "-0.2e+10"},
{token.FLOAT, "-1.2e-10"},
{token.FLOAT, "-42.54e+10"},
{token.FLOAT, "-01234567890.98e-10"},
{token.FLOAT, "-0.1E+10"},
{token.FLOAT, "-1.1E-10"},
{token.FLOAT, "-42.1E+10"},
{token.FLOAT, "-01234567890.1E-10"},
},
}
var orderedTokenLists = []string{
"comment",
"operator",
"bool",
"ident",
"string",
"number",
"float",
}
func TestPosition(t *testing.T) {
// create artifical source code
buf := new(bytes.Buffer)
for _, listName := range orderedTokenLists {
for _, ident := range tokenLists[listName] {
fmt.Fprintf(buf, "\t\t\t\t%s\n", ident.text)
}
}
s := New(buf.Bytes())
pos := token.Pos{"", 4, 1, 5}
s.Scan()
for _, listName := range orderedTokenLists {
for _, k := range tokenLists[listName] {
curPos := s.tokPos
// fmt.Printf("[%q] s = %+v:%+v\n", k.text, curPos.Offset, curPos.Column)
if curPos.Offset != pos.Offset {
t.Fatalf("offset = %d, want %d for %q", curPos.Offset, pos.Offset, k.text)
}
if curPos.Line != pos.Line {
t.Fatalf("line = %d, want %d for %q", curPos.Line, pos.Line, k.text)
}
if curPos.Column != pos.Column {
t.Fatalf("column = %d, want %d for %q", curPos.Column, pos.Column, k.text)
}
pos.Offset += 4 + len(k.text) + 1 // 4 tabs + token bytes + newline
pos.Line += countNewlines(k.text) + 1 // each token is on a new line
s.Scan()
}
}
// make sure there were no token-internal errors reported by scanner
if s.ErrorCount != 0 {
t.Errorf("%d errors", s.ErrorCount)
}
}
func TestComment(t *testing.T) {
testTokenList(t, tokenLists["comment"])
}
func TestOperator(t *testing.T) {
testTokenList(t, tokenLists["operator"])
}
func TestBool(t *testing.T) {
testTokenList(t, tokenLists["bool"])
}
func TestIdent(t *testing.T) {
testTokenList(t, tokenLists["ident"])
}
func TestString(t *testing.T) {
testTokenList(t, tokenLists["string"])
}
func TestNumber(t *testing.T) {
testTokenList(t, tokenLists["number"])
}
func TestFloat(t *testing.T) {
testTokenList(t, tokenLists["float"])
}
func TestRealExample(t *testing.T) {
complexHCL := `// This comes from Terraform, as a test
variable "foo" {
default = "bar"
description = "bar"
}
provider "aws" {
access_key = "foo"
secret_key = "bar"
}
resource "aws_security_group" "firewall" {
count = 5
}
resource aws_instance "web" {
ami = "${var.foo}"
security_groups = [
"foo",
"${aws_security_group.firewall.foo}"
]
network_interface {
device_index = 0
description = "Main network interface"
}
}`
literals := []struct {
tokenType token.Type
literal string
}{
{token.COMMENT, `// This comes from Terraform, as a test`},
{token.IDENT, `variable`},
{token.STRING, `"foo"`},
{token.LBRACE, `{`},
{token.IDENT, `default`},
{token.ASSIGN, `=`},
{token.STRING, `"bar"`},
{token.IDENT, `description`},
{token.ASSIGN, `=`},
{token.STRING, `"bar"`},
{token.RBRACE, `}`},
{token.IDENT, `provider`},
{token.STRING, `"aws"`},
{token.LBRACE, `{`},
{token.IDENT, `access_key`},
{token.ASSIGN, `=`},
{token.STRING, `"foo"`},
{token.IDENT, `secret_key`},
{token.ASSIGN, `=`},
{token.STRING, `"bar"`},
{token.RBRACE, `}`},
{token.IDENT, `resource`},
{token.STRING, `"aws_security_group"`},
{token.STRING, `"firewall"`},
{token.LBRACE, `{`},
{token.IDENT, `count`},
{token.ASSIGN, `=`},
{token.NUMBER, `5`},
{token.RBRACE, `}`},
{token.IDENT, `resource`},
{token.IDENT, `aws_instance`},
{token.STRING, `"web"`},
{token.LBRACE, `{`},
{token.IDENT, `ami`},
{token.ASSIGN, `=`},
{token.STRING, `"${var.foo}"`},
{token.IDENT, `security_groups`},
{token.ASSIGN, `=`},
{token.LBRACK, `[`},
{token.STRING, `"foo"`},
{token.COMMA, `,`},
{token.STRING, `"${aws_security_group.firewall.foo}"`},
{token.RBRACK, `]`},
{token.IDENT, `network_interface`},
{token.LBRACE, `{`},
{token.IDENT, `device_index`},
{token.ASSIGN, `=`},
{token.NUMBER, `0`},
{token.IDENT, `description`},
{token.ASSIGN, `=`},
{token.STRING, `"Main network interface"`},
{token.RBRACE, `}`},
{token.RBRACE, `}`},
{token.EOF, ``},
}
s := New([]byte(complexHCL))
for _, l := range literals {
tok := s.Scan()
if l.tokenType != tok.Type {
t.Errorf("got: %s want %s for %s\n", tok, l.tokenType, tok.String())
}
if l.literal != tok.Text {
t.Errorf("got: %s want %s\n", tok, l.literal)
}
}
}
func TestError(t *testing.T) {
testError(t, "\x80", "1:1", "illegal UTF-8 encoding", token.ILLEGAL)
testError(t, "\xff", "1:1", "illegal UTF-8 encoding", token.ILLEGAL)
testError(t, "ab\x80", "1:3", "illegal UTF-8 encoding", token.IDENT)
testError(t, "abc\xff", "1:4", "illegal UTF-8 encoding", token.IDENT)
testError(t, `"ab`+"\x80", "1:4", "illegal UTF-8 encoding", token.STRING)
testError(t, `"abc`+"\xff", "1:5", "illegal UTF-8 encoding", token.STRING)
testError(t, `01238`, "1:6", "illegal octal number", token.NUMBER)
testError(t, `01238123`, "1:9", "illegal octal number", token.NUMBER)
testError(t, `0x`, "1:3", "illegal hexadecimal number", token.NUMBER)
testError(t, `0xg`, "1:3", "illegal hexadecimal number", token.NUMBER)
testError(t, `'aa'`, "1:1", "illegal char", token.ILLEGAL)
testError(t, `"`, "1:2", "literal not terminated", token.STRING)
testError(t, `"abc`, "1:5", "literal not terminated", token.STRING)
testError(t, `"abc`+"\n", "1:5", "literal not terminated", token.STRING)
testError(t, `/*/`, "1:4", "comment not terminated", token.COMMENT)
}
func testError(t *testing.T, src, pos, msg string, tok token.Type) {
s := New([]byte(src))
errorCalled := false
s.Error = func(p token.Pos, m string) {
if !errorCalled {
if pos != p.String() {
t.Errorf("pos = %q, want %q for %q", p, pos, src)
}
if m != msg {
t.Errorf("msg = %q, want %q for %q", m, msg, src)
}
errorCalled = true
}
}
tk := s.Scan()
if tk.Type != tok {
t.Errorf("tok = %s, want %s for %q", tk, tok, src)
}
if !errorCalled {
t.Errorf("error handler not called for %q", src)
}
if s.ErrorCount == 0 {
t.Errorf("count = %d, want > 0 for %q", s.ErrorCount, src)
}
}
func testTokenList(t *testing.T, tokenList []tokenPair) {
// create artifical source code
buf := new(bytes.Buffer)
for _, ident := range tokenList {
fmt.Fprintf(buf, "%s\n", ident.text)
}
s := New(buf.Bytes())
for _, ident := range tokenList {
tok := s.Scan()
if tok.Type != ident.tok {
t.Errorf("tok = %q want %q for %q\n", tok, ident.tok, ident.text)
}
if tok.Text != ident.text {
t.Errorf("text = %q want %q", tok.String(), ident.text)
}
}
}
func countNewlines(s string) int {
n := 0
for _, ch := range s {
if ch == '\n' {
n++
}
}
return n
}

46
hclnew/token/position.go
View File

@ -1,46 +0,0 @@
package token
import "fmt"
// Pos describes an arbitrary source position
// including the file, line, and column location.
// A Position is valid if the line number is > 0.
type Pos struct {
Filename string // filename, if any
Offset int // offset, starting at 0
Line int // line number, starting at 1
Column int // column number, starting at 1 (character count)
}
// IsValid returns true if the position is valid.
func (p *Pos) IsValid() bool { return p.Line > 0 }
// String returns a string in one of several forms:
//
// file:line:column valid position with file name
// line:column valid position without file name
// file invalid position with file name
// - invalid position without file name
func (p Pos) String() string {
s := p.Filename
if p.IsValid() {
if s != "" {
s += ":"
}
s += fmt.Sprintf("%d:%d", p.Line, p.Column)
}
if s == "" {
s = "-"
}
return s
}
// Before reports whether the position p is before u.
func (p Pos) Before(u Pos) bool {
return u.Offset > p.Offset || u.Line > p.Line
}
// After reports whether the position p is after u.
func (p Pos) After(u Pos) bool {
return u.Offset < p.Offset || u.Line < p.Line
}

105
hclnew/token/token.go
View File

@ -1,105 +0,0 @@
// Package token defines constants representing the lexical tokens for HCL
// (HashiCorp Configuration Language)
package token
import (
"fmt"
"strconv"
)
// Token defines a single HCL token which can be obtained via the Scanner
type Token struct {
Type Type
Pos Pos
Text string
}
// Type is the set of lexical tokens of the HCL (HashiCorp Configuration Language)
type Type int
const (
// Special tokens
ILLEGAL Type = iota
EOF
COMMENT
identifier_beg
IDENT // literals
literal_beg
NUMBER // 12345
FLOAT // 123.45
BOOL // true,false
STRING // "abc"
literal_end
identifier_end
operator_beg
LBRACK // [
LBRACE // {
COMMA // ,
PERIOD // .
RBRACK // ]
RBRACE // }
ASSIGN // =
ADD // +
SUB // -
operator_end
)
var tokens = [...]string{
ILLEGAL: "ILLEGAL",
EOF: "EOF",
COMMENT: "COMMENT",
IDENT: "IDENT",
NUMBER: "NUMBER",
FLOAT: "FLOAT",
BOOL: "BOOL",
STRING: "STRING",
LBRACK: "LBRACK",
LBRACE: "LBRACE",
COMMA: "COMMA",
PERIOD: "PERIOD",
RBRACK: "RBRACK",
RBRACE: "RBRACE",
ASSIGN: "ASSIGN",
ADD: "ADD",
SUB: "SUB",
}
// String returns the string corresponding to the token tok.
func (t Type) String() string {
s := ""
if 0 <= t && t < Type(len(tokens)) {
s = tokens[t]
}
if s == "" {
s = "token(" + strconv.Itoa(int(t)) + ")"
}
return s
}
// IsIdentifier returns true for tokens corresponding to identifiers and basic
// type literals; it returns false otherwise.
func (t Type) IsIdentifier() bool { return identifier_beg < t && t < identifier_end }
// IsLiteral returns true for tokens corresponding to basic type literals; it
// returns false otherwise.
func (t Type) IsLiteral() bool { return literal_beg < t && t < literal_end }
// IsOperator returns true for tokens corresponding to operators and
// delimiters; it returns false otherwise.
func (t Type) IsOperator() bool { return operator_beg < t && t < operator_end }
// String returns the token's literal text. Note that this is only
// applicable for certain token types, such as token.IDENT,
// token.STRING, etc..
func (t Token) String() string {
return fmt.Sprintf("%s %s %s", t.Pos.String(), t.Type.String(), t.Text)
}

36
hclnew/token/token_test.go
View File

@ -1,36 +0,0 @@
package token
import "testing"
func TestTypeString(t *testing.T) {
var tokens = []struct {
tt Type
str string
}{
{ILLEGAL, "ILLEGAL"},
{EOF, "EOF"},
{COMMENT, "COMMENT"},
{IDENT, "IDENT"},
{NUMBER, "NUMBER"},
{FLOAT, "FLOAT"},
{BOOL, "BOOL"},
{STRING, "STRING"},
{LBRACK, "LBRACK"},
{LBRACE, "LBRACE"},
{COMMA, "COMMA"},
{PERIOD, "PERIOD"},
{RBRACK, "RBRACK"},
{RBRACE, "RBRACE"},
{ASSIGN, "ASSIGN"},
{ADD, "ADD"},
{SUB, "SUB"},
}
for _, token := range tokens {
if token.tt.String() != token.str {
t.Errorf("want: %q got:%q\n", token.str, token.tt)
}
}
}