Merge pull request #173 from hashicorp/jbardin/crlf

Remove CRLF line endings

appveyor.yml

@@ -4,7 +4,7 @@ clone_folder: c:\gopath\src\github.com\hashicorp\hcl
 environment:
   GOPATH: c:\gopath
 init:
-  - git config --global core.autocrlf true
+  - git config --global core.autocrlf false
 install:
 - cmd: >-
     echo %Path%

decoder_test.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/hashicorp/hcl/hcl/ast"
-	"github.com/hashicorp/hcl/testhelper"
 )
 
 func TestDecode_interface(t *testing.T) {
@@ -89,8 +88,7 @@ func TestDecode_interface(t *testing.T) {
 		{
 			"multiline_literal_with_hil.hcl",
 			false,
-			map[string]interface{}{"multiline_literal_with_hil": testhelper.Unix2dos(`${hello
+			map[string]interface{}{"multiline_literal_with_hil": "${hello\n  world}"},
-  world}`)},
 		},
 		{
 			"multiline_no_marker.hcl",
@@ -100,22 +98,22 @@ func TestDecode_interface(t *testing.T) {
 		{
 			"multiline.hcl",
 			false,
-			map[string]interface{}{"foo": testhelper.Unix2dos("bar\nbaz\n")},
+			map[string]interface{}{"foo": "bar\nbaz\n"},
 		},
 		{
 			"multiline_indented.hcl",
 			false,
-			map[string]interface{}{"foo": testhelper.Unix2dos("  bar\n  baz\n")},
+			map[string]interface{}{"foo": "  bar\n  baz\n"},
 		},
 		{
 			"multiline_no_hanging_indent.hcl",
 			false,
-			map[string]interface{}{"foo": testhelper.Unix2dos("  baz\n    bar\n      foo\n")},
+			map[string]interface{}{"foo": "  baz\n    bar\n      foo\n"},
 		},
 		{
 			"multiline_no_eof.hcl",
 			false,
-			map[string]interface{}{"foo": testhelper.Unix2dos("bar\nbaz\n"), "key": "value"},
+			map[string]interface{}{"foo": "bar\nbaz\n", "key": "value"},
 		},
 		{
 			"multiline.json",
@@ -421,31 +419,32 @@ func TestDecode_interface(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		t.Logf("Testing: %s", tc.File)
+		t.Run(tc.File, func(t *testing.T) {
-		d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.File))
+			d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.File))
-		if err != nil {
+			if err != nil {
-			t.Fatalf("err: %s", err)
+				t.Fatalf("err: %s", err)
-		}
+			}
 
-		var out interface{}
+			var out interface{}
-		err = Decode(&out, string(d))
+			err = Decode(&out, string(d))
-		if (err != nil) != tc.Err {
+			if (err != nil) != tc.Err {
-			t.Fatalf("Input: %s\n\nError: %s", tc.File, err)
+				t.Fatalf("Input: %s\n\nError: %s", tc.File, err)
-		}
+			}
 
-		if !reflect.DeepEqual(out, tc.Out) {
+			if !reflect.DeepEqual(out, tc.Out) {
-			t.Fatalf("Input: %s. Actual, Expected.\n\n%#v\n\n%#v", tc.File, out, tc.Out)
+				t.Fatalf("Input: %s. Actual, Expected.\n\n%#v\n\n%#v", tc.File, out, tc.Out)
-		}
+			}
 
-		var v interface{}
+			var v interface{}
-		err = Unmarshal(d, &v)
+			err = Unmarshal(d, &v)
-		if (err != nil) != tc.Err {
+			if (err != nil) != tc.Err {
-			t.Fatalf("Input: %s\n\nError: %s", tc.File, err)
+				t.Fatalf("Input: %s\n\nError: %s", tc.File, err)
-		}
+			}
 
-		if !reflect.DeepEqual(v, tc.Out) {
+			if !reflect.DeepEqual(v, tc.Out) {
-			t.Fatalf("Input: %s. Actual, Expected.\n\n%#v\n\n%#v", tc.File, out, tc.Out)
+				t.Fatalf("Input: %s. Actual, Expected.\n\n%#v\n\n%#v", tc.File, out, tc.Out)
-		}
+			}
+		})
 	}
 }
 

hcl/parser/parser.go

@@ -3,6 +3,7 @@
 package parser
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"strings"
@@ -36,6 +37,11 @@ func newParser(src []byte) *Parser {
 
 // Parse returns the fully parsed source and returns the abstract syntax tree.
 func Parse(src []byte) (*ast.File, error) {
+	// normalize all line endings
+	// since the scanner and output only work with "\n" line endings, we may
+	// end up with dangling "\r" characters in the parsed data.
+	src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1)
+
 	p := newParser(src)
 	return p.Parse()
 }

hcl/parser/parser_test.go

@@ -393,6 +393,7 @@ func TestCommentGroup(t *testing.T) {
 		groups int
 	}{
 		{"# Hello\n# World", 1},
+		{"# Hello\r\n# Windows", 1},
 	}
 
 	for _, tc := range cases {
@@ -424,6 +425,10 @@ func TestParse(t *testing.T) {
 			"comment.hcl",
 			false,
 		},
+		{
+			"comment_crlf.hcl",
+			false,
+		},
 		{
 			"comment_lastline.hcl",
 			false,
@@ -464,6 +469,10 @@ func TestParse(t *testing.T) {
 			"complex.hcl",
 			false,
 		},
+		{
+			"complex_crlf.hcl",
+			false,
+		},
 		{
 			"types.hcl",
 			false,

hcl/parser/test-fixtures/comment.hcl

@@ -1,15 +1,15 @@
-// Foo
+// Foo
-
+
-/* Bar */
+/* Bar */
-
+
-/*
+/*
-/*
+/*
-Baz
+Baz
-*/
+*/
-
+
-# Another
+# Another
-
+
-# Multiple
+# Multiple
-# Lines
+# Lines
-
+
-foo = "bar"
+foo = "bar"

hcl/parser/test-fixtures/comment_crlf.hcl

@@ -0,0 +1,15 @@
+// Foo
+
+/* Bar */
+
+/*
+/*
+Baz
+*/
+
+# Another
+
+# Multiple
+# Lines
+
+foo = "bar"

hcl/parser/test-fixtures/complex_crlf.hcl

@@ -0,0 +1,42 @@
+variable "foo" {
+	default = "bar"
+	description = "bar"
+}
+
+variable "groups" { }
+
+provider "aws" {
+	access_key = "foo"
+	secret_key = "bar"
+}
+
+provider "do" {
+	api_key = "${var.foo}"
+}
+
+resource "aws_security_group" "firewall" {
+	count = 5
+}
+
+resource aws_instance "web" {
+	ami = "${var.foo}"
+	security_groups = [
+		"foo",
+		"${aws_security_group.firewall.foo}",
+		"${element(split(\",\", var.groups)}",
+	]
+	network_interface = {
+		device_index = 0
+		description = "Main network interface"
+	}
+}
+
+resource "aws_instance" "db" {
+	security_groups = "${aws_security_group.firewall.*.id}"
+	VPC = "foo"
+	depends_on = ["aws_instance.web"]
+}
+
+output "web_ip" {
+	value = "${aws_instance.web.private_ip}"
+}

hcl/printer/printer.go

@@ -62,6 +62,5 @@ func Format(src []byte) ([]byte, error) {
 
 	// Add trailing newline to result
 	buf.WriteString("\n")
-
 	return buf.Bytes(), nil
 }

hcl/printer/printer_test.go

@@ -1,8 +1,3 @@
-// +build !windows
-// TODO(jen20): These need fixing on Windows but printer is not used right now
-// and red CI is making it harder to process other bugs, so ignore until
-// we get around to fixing them.package printer
-
 package printer
 
 import (
@@ -33,6 +28,7 @@ var data = []entry{
 	{"list.input", "list.golden"},
 	{"list_comment.input", "list_comment.golden"},
 	{"comment.input", "comment.golden"},
+	{"comment_crlf.input", "comment.golden"},
 	{"comment_aligned.input", "comment_aligned.golden"},
 	{"comment_array.input", "comment_array.golden"},
 	{"comment_end_file.input", "comment_end_file.golden"},
@@ -109,8 +105,8 @@ func diff(aname, bname string, a, b []byte) error {
 	for i := 0; i < len(a) && i < len(b); i++ {
 		ch := a[i]
 		if ch != b[i] {
-			fmt.Fprintf(&buf, "\n%s:%d:%d: %s", aname, line, i-offs+1, lineAt(a, offs))
+			fmt.Fprintf(&buf, "\n%s:%d:%d: %q", aname, line, i-offs+1, lineAt(a, offs))
-			fmt.Fprintf(&buf, "\n%s:%d:%d: %s", bname, line, i-offs+1, lineAt(b, offs))
+			fmt.Fprintf(&buf, "\n%s:%d:%d: %q", bname, line, i-offs+1, lineAt(b, offs))
 			fmt.Fprintf(&buf, "\n\n")
 			break
 		}

hcl/printer/testdata/comment_crlf.input

@@ -0,0 +1,37 @@
+// A standalone comment is a comment which is not attached to any kind of node
+
+   // This comes from Terraform, as a test
+variable "foo" {
+	# Standalone comment should be still here
+
+       default = "bar"
+    description =     "bar" # yooo
+}
+
+/* This is a multi line standalone
+comment*/
+
+
+// fatih arslan
+/* This is a developer test
+account and a multine comment */
+developer = [     "fatih",       "arslan"] // fatih arslan
+
+# One line here
+numbers = [1,2] // another line here
+
+         # Another comment
+variable = {
+    description =     "bar" # another yooo
+    foo { 
+	# Nested standalone
+
+        bar = "fatih"
+    } 
+}
+
+          // lead comment
+foo { 
+    bar = "fatih"       // line comment 2 
+}        // line comment 3
+