Add support for indented HEREDOC terminators

This PR adds support for the style of HEREDOC often used in Ruby which
allows the terminating marker to be indented if the HEREDOC is started
with the sequence "<<-" rather than the usual "<<". This allows users to
express documents with more natural indentation:

resource "template_file" "test" {
    template = <<-HEREDOC
        First Line
        Second Line
    HEREDOC
}

Note that this does not attempt to add any semantics around removing
hanging indent from the actual text of the document, so extra
indentation would still be present. We could make use of the canonical
style for HCL herre to remove the hanging indent in the style of Ruby
which would probably be more predictable for users.

decoder_test.go

@@ -91,6 +91,11 @@ func TestDecode_interface(t *testing.T) {
 			false,
 			map[string]interface{}{"foo": testhelper.Unix2dos("bar\nbaz\n")},
 		},
+		{
+			"multiline_indented.hcl",
+			false,
+			map[string]interface{}{"foo": "        bar\n        baz\n"},
+		},
 		{
 			"multiline_no_eof.hcl",
 			false,

hcl/scanner/scanner.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"fmt"
 	"os"
+	"regexp"
 	"unicode"
 	"unicode/utf8"
 
@@ -376,7 +377,7 @@ func (s *Scanner) scanExponent(ch rune) rune {
 	return ch
 }
 
-// scanHeredoc scans a heredoc string.
+// scanHeredoc scans a heredoc string
 func (s *Scanner) scanHeredoc() {
 	// Scan the second '<' in example: '<<EOF'
 	if s.next() != '<' {
@@ -389,6 +390,12 @@ func (s *Scanner) scanHeredoc() {
 
 	// Scan the identifier
 	ch := s.next()
+
+	// Indented heredoc syntax
+	if ch == '-' {
+		ch = s.next()
+	}
+
 	for isLetter(ch) || isDigit(ch) {
 		ch = s.next()
 	}
@@ -414,6 +421,17 @@ func (s *Scanner) scanHeredoc() {
 
 	// Read the identifier
 	identBytes := s.src[offs : s.srcPos.Offset-s.lastCharLen]
+	if len(identBytes) == 0 {
+		s.err("zero-length heredoc anchor")
+		return
+	}
+
+	var identRegexp *regexp.Regexp
+	if identBytes[0] == '-' {
+		identRegexp = regexp.MustCompile(fmt.Sprintf(`[[:space:]]*%s\z`, identBytes[1:]))
+	} else {
+		identRegexp = regexp.MustCompile(fmt.Sprintf(`[[:space:]]*%s\z`, identBytes))
+	}
 
 	// Read the actual string value
 	lineStart := s.srcPos.Offset
@@ -422,12 +440,11 @@ func (s *Scanner) scanHeredoc() {
 
 		// Special newline handling.
 		if ch == '\n' {
-			// Math is fast, so we first compare the byte counts to
-			// see if we have a chance of seeing the same identifier. If those
-			// match, then we compare the string values directly.
+			// Math is fast, so we first compare the byte counts to see if we have a chance
+			// of seeing the same identifier - if the length is less than the number of bytes
+			// in the identifier, this cannot be a valid terminator.
 			lineBytesLen := s.srcPos.Offset - s.lastCharLen - lineStart
-			if lineBytesLen == len(identBytes) &&
-				bytes.Equal(identBytes, s.src[lineStart:s.srcPos.Offset-s.lastCharLen]) {
+			if lineBytesLen >= len(identBytes) && identRegexp.Match(s.src[lineStart:s.srcPos.Offset-s.lastCharLen]) {
 				break
 			}
 

hcl/scanner/scanner_test.go

@@ -5,8 +5,9 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/hashicorp/hcl/hcl/token"
 	"strings"
+
+	"github.com/hashicorp/hcl/hcl/token"
 )
 
 var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
@@ -377,6 +378,14 @@ func TestRealExample(t *testing.T) {
 Main interface
 EOF
 	    }
+	    
+		network_interface {
+	        device_index = 1
+	        description = <<-EOF
+			Outer text
+				Indented text
+			EOF
+		}
 	}`
 
 	literals := []struct {
@@ -435,6 +444,15 @@ EOF
 		{token.ASSIGN, `=`},
 		{token.HEREDOC, "<<EOF\nMain interface\nEOF\n"},
 		{token.RBRACE, `}`},
+		{token.IDENT, `network_interface`},
+		{token.LBRACE, `{`},
+		{token.IDENT, `device_index`},
+		{token.ASSIGN, `=`},
+		{token.NUMBER, `1`},
+		{token.IDENT, `description`},
+		{token.ASSIGN, `=`},
+		{token.HEREDOC, "<<-EOF\n\t\t\tOuter text\n\t\t\t\tIndented text\n\t\t\tEOF\n"},
+		{token.RBRACE, `}`},
 		{token.RBRACE, `}`},
 		{token.EOF, ``},
 	}
@@ -447,7 +465,7 @@ EOF
 		}
 
 		if l.literal != tok.Text {
-			t.Errorf("got: %s want %s\n", tok, l.literal)
+			t.Errorf("got:\n%+v\n%s\n want:\n%+v\n%s\n", []byte(tok.String()), tok, []byte(l.literal), l.literal)
 		}
 	}
 

hcl/token/token.go

@@ -148,7 +148,8 @@ func (t Token) Value() interface{} {
 			panic("heredoc doesn't contain newline")
 		}
 
-		return string(t.Text[idx+1 : len(t.Text)-idx+1])
+		// Trim any trailing whitespace from the start of the marker
+		return strings.TrimRight(string(t.Text[idx+1:len(t.Text)-idx+1]), " \t")
 	case STRING:
 		// Determine the Unquote method to use. If it came from JSON,
 		// then we need to use the built-in unquote since we have to

test-fixtures/multiline_indented.hcl

@@ -0,0 +1,4 @@
+foo = <<-EOF
+        bar
+        baz
+    EOF