go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

scanner: use new hcl/token package

Browse Source
This commit is contained in:
Fatih Arslan 2015-10-04 20:19:39 +03:00
parent df82bd3e9c
commit 52befe2093
2 changed files with 33 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scanner/scanner.go
View File

@ -6,6 +6,8 @@ import (
"io/ioutil" "io/ioutil"
"log" "log"
"unicode" "unicode"
"github.com/fatih/hcl/token"
) )
// eof represents a marker rune for the end of the reader. // eof represents a marker rune for the end of the reader.
@ -83,7 +85,7 @@ func (s *Scanner) peek() rune {
} }
// Scan scans the next token and returns the token. // Scan scans the next token and returns the token.
func (s *Scanner) Scan() (tok Token) { func (s *Scanner) Scan() (tok token.Token) {
ch := s.next() ch := s.next()
// skip white space // skip white space
@ -96,10 +98,10 @@ func (s *Scanner) Scan() (tok Token) {
s.tokPos = s.currPos.Offset - s.lastCharLen s.tokPos = s.currPos.Offset - s.lastCharLen
if isLetter(ch) { if isLetter(ch) {
tok = IDENT tok = token.IDENT
lit := s.scanIdentifier() lit := s.scanIdentifier()
if lit == "true" || lit == "false" { if lit == "true" || lit == "false" {
tok = BOOL tok = token.BOOL
} }
} }
@ -110,9 +112,9 @@ func (s *Scanner) Scan() (tok Token) {
switch ch { switch ch {
case eof: case eof:
tok = EOF tok = token.EOF
case '"': case '"':
tok = STRING tok = token.STRING
s.scanString() s.scanString()
} }

50
scanner/scanner_text.go → scanner/scanner_test.go
View File

@ -4,17 +4,19 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"testing" "testing"
"github.com/fatih/hcl/token"
) )
var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
type token struct { type tokenPair struct {
tok Token tok token.Token
text string text string
} }
func TestBool(t *testing.T) { func TestBool(t *testing.T) {
var tokenList = []token{ var tokenList = []tokenPair{
{token.BOOL, "true"}, {token.BOOL, "true"},
{token.BOOL, "false"}, {token.BOOL, "false"},
} }
@ -44,23 +46,23 @@ func TestBool(t *testing.T) {
} }
func TestIdent(t *testing.T) { func TestIdent(t *testing.T) {
var tokenList = []token{ var tokenList = []tokenPair{
{IDENT, "a"}, {token.IDENT, "a"},
{IDENT, "a0"}, {token.IDENT, "a0"},
{IDENT, "foobar"}, {token.IDENT, "foobar"},
{IDENT, "abc123"}, {token.IDENT, "abc123"},
{IDENT, "LGTM"}, {token.IDENT, "LGTM"},
{IDENT, "_"}, {token.IDENT, "_"},
{IDENT, "_abc123"}, {token.IDENT, "_abc123"},
{IDENT, "abc123_"}, {token.IDENT, "abc123_"},
{IDENT, "_abc_123_"}, {token.IDENT, "_abc_123_"},
{IDENT, "_äöü"}, {token.IDENT, "_äöü"},
{IDENT, "_本"}, {token.IDENT, "_本"},
{IDENT, "äöü"}, {token.IDENT, "äöü"},
{IDENT, "本"}, {token.IDENT, "本"},
{IDENT, "a۰۱۸"}, {token.IDENT, "a۰۱۸"},
{IDENT, "foo६४"}, {token.IDENT, "foo६४"},
{IDENT, "bar９８７６"}, {token.IDENT, "bar９８７６"},
} }
// create artifical source code // create artifical source code
@ -88,10 +90,10 @@ func TestIdent(t *testing.T) {
} }
func TestString(t *testing.T) { func TestString(t *testing.T) {
var tokenList = []token{ var tokenList = []tokenPair{
{STRING, `" "`}, {token.STRING, `" "`},
{STRING, `"a"`}, {token.STRING, `"a"`},
{STRING, `"本"`}, {token.STRING, `"本"`},
// {STRING, `"\a"`}, // {STRING, `"\a"`},
// {STRING, `"\b"`}, // {STRING, `"\b"`},
// {STRING, `"\f"`}, // {STRING, `"\f"`},