hcl/strconv: custom unquote logic

hcl/strconv/quote.go

@@ -0,0 +1,235 @@
+package strconv
+
+import (
+	"errors"
+	"unicode/utf8"
+)
+
+// ErrSyntax indicates that a value does not have the right syntax for the target type.
+var ErrSyntax = errors.New("invalid syntax")
+
+// Unquote interprets s as a single-quoted, double-quoted,
+// or backquoted Go string literal, returning the string value
+// that s quotes.  (If s is single-quoted, it would be a Go
+// character literal; Unquote returns the corresponding
+// one-character string.)
+func Unquote(s string) (t string, err error) {
+	n := len(s)
+	if n < 2 {
+		return "", ErrSyntax
+	}
+	quote := s[0]
+	if quote != s[n-1] {
+		return "", ErrSyntax
+	}
+	s = s[1 : n-1]
+
+	if quote != '"' {
+		return "", ErrSyntax
+	}
+	if contains(s, '\n') {
+		return "", ErrSyntax
+	}
+
+	// Is it trivial?  Avoid allocation.
+	if !contains(s, '\\') && !contains(s, quote) {
+		switch quote {
+		case '"':
+			return s, nil
+		case '\'':
+			r, size := utf8.DecodeRuneInString(s)
+			if size == len(s) && (r != utf8.RuneError || size != 1) {
+				return s, nil
+			}
+		}
+	}
+
+	var runeTmp [utf8.UTFMax]byte
+	buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations.
+	for len(s) > 0 {
+		// If we're starting a '${}' then let it through un-unquoted.
+		// Specifically: we don't unquote any characters within the `${}`
+		// section.
+		if s[0] == '$' && len(s) > 1 && s[1] == '{' {
+			buf = append(buf, '$', '{')
+			s = s[2:]
+
+			// Continue reading until we find the closing brace, copying as-is
+			braces := 1
+			for len(s) > 0 {
+				r, size := utf8.DecodeRuneInString(s)
+				if r == utf8.RuneError {
+					return "", ErrSyntax
+				}
+
+				s = s[size:]
+
+				n := utf8.EncodeRune(runeTmp[:], r)
+				buf = append(buf, runeTmp[:n]...)
+
+				switch r {
+				case '{':
+					braces++
+				case '}':
+					braces--
+					if braces == 0 {
+						break
+					}
+				}
+			}
+			if braces != 0 {
+				return "", ErrSyntax
+			}
+			if len(s) == 0 {
+				break
+			}
+		}
+
+		c, multibyte, ss, err := unquoteChar(s, quote)
+		if err != nil {
+			return "", err
+		}
+		s = ss
+		if c < utf8.RuneSelf || !multibyte {
+			buf = append(buf, byte(c))
+		} else {
+			n := utf8.EncodeRune(runeTmp[:], c)
+			buf = append(buf, runeTmp[:n]...)
+		}
+		if quote == '\'' && len(s) != 0 {
+			// single-quoted must be single character
+			return "", ErrSyntax
+		}
+	}
+	return string(buf), nil
+}
+
+// contains reports whether the string contains the byte c.
+func contains(s string, c byte) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] == c {
+			return true
+		}
+	}
+	return false
+}
+
+func unhex(b byte) (v rune, ok bool) {
+	c := rune(b)
+	switch {
+	case '0' <= c && c <= '9':
+		return c - '0', true
+	case 'a' <= c && c <= 'f':
+		return c - 'a' + 10, true
+	case 'A' <= c && c <= 'F':
+		return c - 'A' + 10, true
+	}
+	return
+}
+
+func unquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) {
+	// easy cases
+	switch c := s[0]; {
+	case c == quote && (quote == '\'' || quote == '"'):
+		err = ErrSyntax
+		return
+	case c >= utf8.RuneSelf:
+		r, size := utf8.DecodeRuneInString(s)
+		return r, true, s[size:], nil
+	case c != '\\':
+		return rune(s[0]), false, s[1:], nil
+	}
+
+	// hard case: c is backslash
+	if len(s) <= 1 {
+		err = ErrSyntax
+		return
+	}
+	c := s[1]
+	s = s[2:]
+
+	switch c {
+	case 'a':
+		value = '\a'
+	case 'b':
+		value = '\b'
+	case 'f':
+		value = '\f'
+	case 'n':
+		value = '\n'
+	case 'r':
+		value = '\r'
+	case 't':
+		value = '\t'
+	case 'v':
+		value = '\v'
+	case 'x', 'u', 'U':
+		n := 0
+		switch c {
+		case 'x':
+			n = 2
+		case 'u':
+			n = 4
+		case 'U':
+			n = 8
+		}
+		var v rune
+		if len(s) < n {
+			err = ErrSyntax
+			return
+		}
+		for j := 0; j < n; j++ {
+			x, ok := unhex(s[j])
+			if !ok {
+				err = ErrSyntax
+				return
+			}
+			v = v<<4 | x
+		}
+		s = s[n:]
+		if c == 'x' {
+			// single-byte string, possibly not UTF-8
+			value = v
+			break
+		}
+		if v > utf8.MaxRune {
+			err = ErrSyntax
+			return
+		}
+		value = v
+		multibyte = true
+	case '0', '1', '2', '3', '4', '5', '6', '7':
+		v := rune(c) - '0'
+		if len(s) < 2 {
+			err = ErrSyntax
+			return
+		}
+		for j := 0; j < 2; j++ { // one digit already; two more
+			x := rune(s[j]) - '0'
+			if x < 0 || x > 7 {
+				err = ErrSyntax
+				return
+			}
+			v = (v << 3) | x
+		}
+		s = s[2:]
+		if v > 255 {
+			err = ErrSyntax
+			return
+		}
+		value = v
+	case '\\':
+		value = '\\'
+	case '\'', '"':
+		if c != quote {
+			err = ErrSyntax
+			return
+		}
+		value = rune(c)
+	default:
+		err = ErrSyntax
+		return
+	}
+	tail = s
+	return
+}

hcl/strconv/quote_test.go

@@ -0,0 +1,90 @@
+package strconv
+
+import (
+	"testing"
+)
+
+type quoteTest struct {
+	in    string
+	out   string
+	ascii string
+}
+
+var quotetests = []quoteTest{
+	{"\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`, `"\a\b\f\r\n\t\v"`},
+	{"\\", `"\\"`, `"\\"`},
+	{"abc\xffdef", `"abc\xffdef"`, `"abc\xffdef"`},
+	{"\u263a", `"☺"`, `"\u263a"`},
+	{"\U0010ffff", `"\U0010ffff"`, `"\U0010ffff"`},
+	{"\x04", `"\x04"`, `"\x04"`},
+}
+
+type unQuoteTest struct {
+	in  string
+	out string
+}
+
+var unquotetests = []unQuoteTest{
+	{`""`, ""},
+	{`"a"`, "a"},
+	{`"abc"`, "abc"},
+	{`"☺"`, "☺"},
+	{`"hello world"`, "hello world"},
+	{`"\xFF"`, "\xFF"},
+	{`"\377"`, "\377"},
+	{`"\u1234"`, "\u1234"},
+	{`"\U00010111"`, "\U00010111"},
+	{`"\U0001011111"`, "\U0001011111"},
+	{`"\a\b\f\n\r\t\v\\\""`, "\a\b\f\n\r\t\v\\\""},
+	{`"'"`, "'"},
+	{`"${file("foo")}"`, `${file("foo")}`},
+}
+
+var misquoted = []string{
+	``,
+	`"`,
+	`"a`,
+	`"'`,
+	`b"`,
+	`"\"`,
+	`"\9"`,
+	`"\19"`,
+	`"\129"`,
+	`'\'`,
+	`'\9'`,
+	`'\19'`,
+	`'\129'`,
+	`'ab'`,
+	`"\x1!"`,
+	`"\U12345678"`,
+	`"\z"`,
+	"`",
+	"`xxx",
+	"`\"",
+	`"\'"`,
+	`'\"'`,
+	"\"\n\"",
+	"\"\\n\n\"",
+	"'\n'",
+}
+
+func TestUnquote(t *testing.T) {
+	for _, tt := range unquotetests {
+		if out, err := Unquote(tt.in); err != nil && out != tt.out {
+			t.Errorf("Unquote(%#q) = %q, %v want %q, nil", tt.in, out, err, tt.out)
+		}
+	}
+
+	// run the quote tests too, backward
+	for _, tt := range quotetests {
+		if in, err := Unquote(tt.out); in != tt.in {
+			t.Errorf("Unquote(%#q) = %q, %v, want %q, nil", tt.out, in, err, tt.in)
+		}
+	}
+
+	for _, s := range misquoted {
+		if out, err := Unquote(s); out != "" || err != ErrSyntax {
+			t.Errorf("Unquote(%#q) = %q, %v want %q, %v", s, out, err, "", ErrSyntax)
+		}
+	}
+}

hcl/token/token.go

@@ -5,6 +5,8 @@ package token
 import (
 	"fmt"
 	"strconv"
+
+	hclstrconv "github.com/hashicorp/hcl/hcl/strconv"
 )
 
 // Token defines a single HCL token which can be obtained via the Scanner
@@ -136,9 +138,9 @@ func (t Token) Value() interface{} {
 	case IDENT:
 		return t.Text
 	case STRING:
-		v, err := strconv.Unquote(t.Text)
+		v, err := hclstrconv.Unquote(t.Text)
 		if err != nil {
-			panic(err)
+			panic(fmt.Sprintf("unquote %s err: %s", t.Text, err))
 		}
 
 		return v