scanner: change signature of Scanner
This commit is contained in:
Fatih Arslan
parent
da40013062
commit
3631451bd2
@ -1,10 +1,10 @@
|
||||
// Package scanner implements a scanner for HCL (HashiCorp Configuration
|
||||
// Language) source text.
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
@ -40,37 +40,30 @@ type Scanner struct {
|
||||
// ErrorCount is incremented by one for each error encountered.
|
||||
ErrorCount int
|
||||
|
||||
// Start position of most recently scanned token; set by Scan.
|
||||
// Calling Init or Next invalidates the position (Line == 0).
|
||||
// The Filename field is always left untouched by the Scanner.
|
||||
// If an error is reported (via Error) and Position is invalid,
|
||||
// the scanner is not inside a token. Call Pos to obtain an error
|
||||
// position in that case.
|
||||
// tokPos is the start position of most recently scanned token; set by
|
||||
// Scan. The Filename field is always left untouched by the Scanner. If
|
||||
// an error is reported (via Error) and Position is invalid, the scanner is
|
||||
// not inside a token.
|
||||
tokPos Position
|
||||
}
|
||||
|
||||
// NewScanner returns a new instance of Lexer. Even though src is an io.Reader,
|
||||
// we fully consume the content.
|
||||
func NewScanner(src io.Reader) (*Scanner, error) {
|
||||
buf, err := ioutil.ReadAll(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b := bytes.NewBuffer(buf)
|
||||
// NewScanner returns a new instance of Scanner.
|
||||
func NewScanner(src []byte) *Scanner {
|
||||
b := bytes.NewBuffer(src)
|
||||
s := &Scanner{
|
||||
src: b,
|
||||
srcBuf: b.Bytes(),
|
||||
srcBuf: src, // immutable src
|
||||
}
|
||||
|
||||
// srcPosition always starts with 1
|
||||
s.srcPos.Line = 1
|
||||
return s, nil
|
||||
return s
|
||||
}
|
||||
|
||||
// next reads the next rune from the bufferred reader. Returns the rune(0) if
|
||||
// an error occurs (or io.EOF is returned).
|
||||
func (s *Scanner) next() rune {
|
||||
|
||||
ch, size, err := s.src.ReadRune()
|
||||
if err != nil {
|
||||
// advance for error reporting
|
||||
@ -106,6 +99,7 @@ func (s *Scanner) next() rune {
|
||||
return ch
|
||||
}
|
||||
|
||||
// unread
|
||||
func (s *Scanner) unread() {
|
||||
if err := s.src.UnreadRune(); err != nil {
|
||||
panic(err) // this is user fault, we should catch it
|
||||
@ -113,6 +107,7 @@ func (s *Scanner) unread() {
|
||||
s.srcPos = s.prevPos // put back last position
|
||||
}
|
||||
|
||||
// peek returns the next rune without advancing the reader.
|
||||
func (s *Scanner) peek() rune {
|
||||
peek, _, err := s.src.ReadRune()
|
||||
if err != nil {
|
||||
@ -203,6 +198,26 @@ func (s *Scanner) Scan() (tok token.Token) {
|
||||
return tok
|
||||
}
|
||||
|
||||
// TokenText returns the literal string corresponding to the most recently
|
||||
// scanned token.
|
||||
func (s *Scanner) TokenText() string {
|
||||
if s.tokStart < 0 {
|
||||
// no token text
|
||||
return ""
|
||||
}
|
||||
|
||||
// part of the token text was saved in tokBuf: save the rest in
|
||||
// tokBuf as well and return its content
|
||||
s.tokBuf.Write(s.srcBuf[s.tokStart:s.tokEnd])
|
||||
s.tokStart = s.tokEnd // ensure idempotency of TokenText() call
|
||||
return s.tokBuf.String()
|
||||
}
|
||||
|
||||
// Pos returns the successful position of the most recently scanned token.
|
||||
func (s *Scanner) Pos() (pos Position) {
|
||||
return s.tokPos
|
||||
}
|
||||
|
||||
func (s *Scanner) scanComment(ch rune) {
|
||||
// single line comments
|
||||
if ch == '#' || (ch == '/' && s.peek() != '*') {
|
||||
@ -335,6 +350,7 @@ func (s *Scanner) scanMantissa(ch rune) rune {
|
||||
return ch
|
||||
}
|
||||
|
||||
// scanFraction scans the fraction after the '.' rune
|
||||
func (s *Scanner) scanFraction(ch rune) rune {
|
||||
if ch == '.' {
|
||||
ch = s.peek() // we peek just to see if we can move forward
|
||||
@ -343,6 +359,8 @@ func (s *Scanner) scanFraction(ch rune) rune {
|
||||
return ch
|
||||
}
|
||||
|
||||
// scanExponent scans the remaining parts of an exponent after the 'e' or 'E'
|
||||
// rune.
|
||||
func (s *Scanner) scanExponent(ch rune) rune {
|
||||
if ch == 'e' || ch == 'E' {
|
||||
ch = s.next()
|
||||
@ -431,26 +449,6 @@ func (s *Scanner) scanIdentifier() string {
|
||||
return string(s.srcBuf[offs:s.srcPos.Offset])
|
||||
}
|
||||
|
||||
// TokenText returns the literal string corresponding to the most recently
|
||||
// scanned token.
|
||||
func (s *Scanner) TokenText() string {
|
||||
if s.tokStart < 0 {
|
||||
// no token text
|
||||
return ""
|
||||
}
|
||||
|
||||
// part of the token text was saved in tokBuf: save the rest in
|
||||
// tokBuf as well and return its content
|
||||
s.tokBuf.Write(s.srcBuf[s.tokStart:s.tokEnd])
|
||||
s.tokStart = s.tokEnd // ensure idempotency of TokenText() call
|
||||
return s.tokBuf.String()
|
||||
}
|
||||
|
||||
// Pos returns the successful position of the most recently scanned token.
|
||||
func (s *Scanner) Pos() (pos Position) {
|
||||
return s.tokPos
|
||||
}
|
||||
|
||||
// recentPosition returns the position of the character immediately after the
|
||||
// character or token returned by the last call to Scan.
|
||||
func (s *Scanner) recentPosition() (pos Position) {
|
||||
|
||||
@ -3,7 +3,6 @@ package scanner
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/fatih/hcl/token"
|
||||
@ -185,10 +184,7 @@ func TestPosition(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
s, err := NewScanner(buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := NewScanner(buf.Bytes())
|
||||
|
||||
pos := Position{"", 4, 1, 5}
|
||||
s.Scan()
|
||||
@ -246,6 +242,52 @@ func TestFloat(t *testing.T) {
|
||||
testTokenList(t, tokenLists["float"])
|
||||
}
|
||||
|
||||
func TestComplexHCL(t *testing.T) {
|
||||
// complexHCL = `// This comes from Terraform, as a test
|
||||
// variable "foo" {
|
||||
// default = "bar"
|
||||
// description = "bar"
|
||||
// }
|
||||
//
|
||||
// provider "aws" {
|
||||
// access_key = "foo"
|
||||
// secret_key = "bar"
|
||||
// }
|
||||
//
|
||||
// provider "do" {
|
||||
// api_key = "${var.foo}"
|
||||
// }
|
||||
//
|
||||
// resource "aws_security_group" "firewall" {
|
||||
// count = 5
|
||||
// }
|
||||
//
|
||||
// resource aws_instance "web" {
|
||||
// ami = "${var.foo}"
|
||||
// security_groups = [
|
||||
// "foo",
|
||||
// "${aws_security_group.firewall.foo}"
|
||||
// ]
|
||||
//
|
||||
// network_interface {
|
||||
// device_index = 0
|
||||
// description = "Main network interface"
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// resource "aws_instance" "db" {
|
||||
// security_groups = "${aws_security_group.firewall.*.id}"
|
||||
// VPC = "foo"
|
||||
//
|
||||
// depends_on = ["aws_instance.web"]
|
||||
// }
|
||||
//
|
||||
// output "web_ip" {
|
||||
// value = "${aws_instance.web.private_ip}"
|
||||
// }`
|
||||
|
||||
}
|
||||
|
||||
func TestError(t *testing.T) {
|
||||
testError(t, "\x80", "1:1", "illegal UTF-8 encoding", token.ILLEGAL)
|
||||
testError(t, "\xff", "1:1", "illegal UTF-8 encoding", token.ILLEGAL)
|
||||
@ -269,10 +311,7 @@ func TestError(t *testing.T) {
|
||||
}
|
||||
|
||||
func testError(t *testing.T, src, pos, msg string, tok token.Token) {
|
||||
s, err := NewScanner(strings.NewReader(src))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := NewScanner([]byte(src))
|
||||
|
||||
errorCalled := false
|
||||
s.Error = func(p Position, m string) {
|
||||
@ -307,11 +346,7 @@ func testTokenList(t *testing.T, tokenList []tokenPair) {
|
||||
fmt.Fprintf(buf, "%s\n", ident.text)
|
||||
}
|
||||
|
||||
s, err := NewScanner(buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s := NewScanner(buf.Bytes())
|
||||
for _, ident := range tokenList {
|
||||
tok := s.Scan()
|
||||
if tok != ident.tok {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user