go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

scanner: use new hcl/token package

Browse Source
This commit is contained in:
Fatih Arslan 2015-10-04 20:19:39 +03:00
parent df82bd3e9c
commit 52befe2093
2 changed files with 33 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scanner/scanner.go
View File

@ -6,6 +6,8 @@ import (
"io/ioutil"
"log"
"unicode"
"github.com/fatih/hcl/token"
)
// eof represents a marker rune for the end of the reader.
@ -83,7 +85,7 @@ func (s *Scanner) peek() rune {
}
// Scan scans the next token and returns the token.
func (s *Scanner) Scan() (tok Token) {
func (s *Scanner) Scan() (tok token.Token) {
ch := s.next()
// skip white space
@ -96,10 +98,10 @@ func (s *Scanner) Scan() (tok Token) {
s.tokPos = s.currPos.Offset - s.lastCharLen
if isLetter(ch) {
tok = IDENT
tok = token.IDENT
lit := s.scanIdentifier()
if lit == "true" || lit == "false" {
tok = BOOL
tok = token.BOOL
}
}
@ -110,9 +112,9 @@ func (s *Scanner) Scan() (tok Token) {
switch ch {
case eof:
tok = EOF
tok = token.EOF
case '"':
tok = STRING
tok = token.STRING
s.scanString()
}

50
scanner/scanner_text.go → scanner/scanner_test.go
View File

@ -4,17 +4,19 @@ import (
"bytes"
"fmt"
"testing"
"github.com/fatih/hcl/token"
)
var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
type token struct {
tok Token
type tokenPair struct {
tok token.Token
text string
}
func TestBool(t *testing.T) {
var tokenList = []token{
var tokenList = []tokenPair{
{token.BOOL, "true"},
{token.BOOL, "false"},
}
@ -44,23 +46,23 @@ func TestBool(t *testing.T) {
}
func TestIdent(t *testing.T) {
var tokenList = []token{
{IDENT, "a"},
{IDENT, "a0"},
{IDENT, "foobar"},
{IDENT, "abc123"},
{IDENT, "LGTM"},
{IDENT, "_"},
{IDENT, "_abc123"},
{IDENT, "abc123_"},
{IDENT, "_abc_123_"},
{IDENT, "_äöü"},
{IDENT, "_本"},
{IDENT, "äöü"},
{IDENT, "本"},
{IDENT, "a۰۱۸"},
{IDENT, "foo६४"},
{IDENT, "bar９８７６"},
var tokenList = []tokenPair{
{token.IDENT, "a"},
{token.IDENT, "a0"},
{token.IDENT, "foobar"},
{token.IDENT, "abc123"},
{token.IDENT, "LGTM"},
{token.IDENT, "_"},
{token.IDENT, "_abc123"},
{token.IDENT, "abc123_"},
{token.IDENT, "_abc_123_"},
{token.IDENT, "_äöü"},
{token.IDENT, "_本"},
{token.IDENT, "äöü"},
{token.IDENT, "本"},
{token.IDENT, "a۰۱۸"},
{token.IDENT, "foo६४"},
{token.IDENT, "bar９８７６"},
}
// create artifical source code
@ -88,10 +90,10 @@ func TestIdent(t *testing.T) {
}
func TestString(t *testing.T) {
var tokenList = []token{
{STRING, `" "`},
{STRING, `"a"`},
{STRING, `"本"`},
var tokenList = []tokenPair{
{token.STRING, `" "`},
{token.STRING, `"a"`},
{token.STRING, `"本"`},
// {STRING, `"\a"`},
// {STRING, `"\b"`},
// {STRING, `"\f"`},