From 86950618a5db5cea7de1b4e6abd6c021e50ea4a9 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 1 Dec 2016 10:01:40 -0500 Subject: [PATCH 1/4] Don't automatically convert to CRLF in AppVeyor We have test files with both types of line endings, leave them as they are for tests. --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 3c8cdf8..4db0b71 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,7 +4,7 @@ clone_folder: c:\gopath\src\github.com\hashicorp\hcl environment: GOPATH: c:\gopath init: - - git config --global core.autocrlf true + - git config --global core.autocrlf false install: - cmd: >- echo %Path% From a97a4ff95fd6b21d1d8983951bec0a064954d730 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Wed, 30 Nov 2016 16:16:03 -0500 Subject: [PATCH 2/4] No longer preserve CRLF line endings Matching the behavior of Go tooling, we are going to accept "\r\n" line endings, but standaradize on "\n" being the official format. Normalize line endings to "\n in the Parse function, so that we never get stray "\r" characters in the source. --- hcl/parser/parser.go | 6 ++++ hcl/parser/parser_test.go | 9 +++++ hcl/parser/test-fixtures/comment_crlf.hcl | 15 ++++++++ hcl/parser/test-fixtures/complex_crlf.hcl | 42 +++++++++++++++++++++++ hcl/printer/printer.go | 1 - hcl/printer/printer_test.go | 10 ++---- hcl/printer/testdata/comment_crlf.input | 37 ++++++++++++++++++++ 7 files changed, 112 insertions(+), 8 deletions(-) create mode 100644 hcl/parser/test-fixtures/comment_crlf.hcl create mode 100644 hcl/parser/test-fixtures/complex_crlf.hcl create mode 100644 hcl/printer/testdata/comment_crlf.input diff --git a/hcl/parser/parser.go b/hcl/parser/parser.go index 476ed04..82641eb 100644 --- a/hcl/parser/parser.go +++ b/hcl/parser/parser.go @@ -3,6 +3,7 @@ package parser import ( + "bytes" "errors" "fmt" "strings" @@ -36,6 +37,11 @@ func newParser(src []byte) *Parser { // Parse returns the fully parsed source and returns the abstract syntax tree. func Parse(src []byte) (*ast.File, error) { + // normalize all line endings + // since the scanner and output only work with "\n" line endings, we may + // end up with dangling "\r" characters in the parsed data. + src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1) + p := newParser(src) return p.Parse() } diff --git a/hcl/parser/parser_test.go b/hcl/parser/parser_test.go index ea0d78f..7f522fc 100644 --- a/hcl/parser/parser_test.go +++ b/hcl/parser/parser_test.go @@ -389,6 +389,7 @@ func TestCommentGroup(t *testing.T) { groups int }{ {"# Hello\n# World", 1}, + {"# Hello\r\n# Windows", 1}, } for _, tc := range cases { @@ -420,6 +421,10 @@ func TestParse(t *testing.T) { "comment.hcl", false, }, + { + "comment_crlf.hcl", + false, + }, { "comment_lastline.hcl", false, @@ -460,6 +465,10 @@ func TestParse(t *testing.T) { "complex.hcl", false, }, + { + "complex_crlf.hcl", + false, + }, { "types.hcl", false, diff --git a/hcl/parser/test-fixtures/comment_crlf.hcl b/hcl/parser/test-fixtures/comment_crlf.hcl new file mode 100644 index 0000000..51dbcfd --- /dev/null +++ b/hcl/parser/test-fixtures/comment_crlf.hcl @@ -0,0 +1,15 @@ +// Foo + +/* Bar */ + +/* +/* +Baz +*/ + +# Another + +# Multiple +# Lines + +foo = "bar" diff --git a/hcl/parser/test-fixtures/complex_crlf.hcl b/hcl/parser/test-fixtures/complex_crlf.hcl new file mode 100644 index 0000000..9b071d1 --- /dev/null +++ b/hcl/parser/test-fixtures/complex_crlf.hcl @@ -0,0 +1,42 @@ +variable "foo" { + default = "bar" + description = "bar" +} + +variable "groups" { } + +provider "aws" { + access_key = "foo" + secret_key = "bar" +} + +provider "do" { + api_key = "${var.foo}" +} + +resource "aws_security_group" "firewall" { + count = 5 +} + +resource aws_instance "web" { + ami = "${var.foo}" + security_groups = [ + "foo", + "${aws_security_group.firewall.foo}", + "${element(split(\",\", var.groups)}", + ] + network_interface = { + device_index = 0 + description = "Main network interface" + } +} + +resource "aws_instance" "db" { + security_groups = "${aws_security_group.firewall.*.id}" + VPC = "foo" + depends_on = ["aws_instance.web"] +} + +output "web_ip" { + value = "${aws_instance.web.private_ip}" +} diff --git a/hcl/printer/printer.go b/hcl/printer/printer.go index a296fc8..6617ab8 100644 --- a/hcl/printer/printer.go +++ b/hcl/printer/printer.go @@ -62,6 +62,5 @@ func Format(src []byte) ([]byte, error) { // Add trailing newline to result buf.WriteString("\n") - return buf.Bytes(), nil } diff --git a/hcl/printer/printer_test.go b/hcl/printer/printer_test.go index 573cf91..5a1f579 100644 --- a/hcl/printer/printer_test.go +++ b/hcl/printer/printer_test.go @@ -1,8 +1,3 @@ -// +build !windows -// TODO(jen20): These need fixing on Windows but printer is not used right now -// and red CI is making it harder to process other bugs, so ignore until -// we get around to fixing them.package printer - package printer import ( @@ -32,6 +27,7 @@ var data = []entry{ {"complexhcl.input", "complexhcl.golden"}, {"list.input", "list.golden"}, {"comment.input", "comment.golden"}, + {"comment_crlf.input", "comment.golden"}, {"comment_aligned.input", "comment_aligned.golden"}, {"comment_array.input", "comment_array.golden"}, {"comment_multiline_indent.input", "comment_multiline_indent.golden"}, @@ -104,8 +100,8 @@ func diff(aname, bname string, a, b []byte) error { for i := 0; i < len(a) && i < len(b); i++ { ch := a[i] if ch != b[i] { - fmt.Fprintf(&buf, "\n%s:%d:%d: %s", aname, line, i-offs+1, lineAt(a, offs)) - fmt.Fprintf(&buf, "\n%s:%d:%d: %s", bname, line, i-offs+1, lineAt(b, offs)) + fmt.Fprintf(&buf, "\n%s:%d:%d: %q", aname, line, i-offs+1, lineAt(a, offs)) + fmt.Fprintf(&buf, "\n%s:%d:%d: %q", bname, line, i-offs+1, lineAt(b, offs)) fmt.Fprintf(&buf, "\n\n") break } diff --git a/hcl/printer/testdata/comment_crlf.input b/hcl/printer/testdata/comment_crlf.input new file mode 100644 index 0000000..5d27206 --- /dev/null +++ b/hcl/printer/testdata/comment_crlf.input @@ -0,0 +1,37 @@ +// A standalone comment is a comment which is not attached to any kind of node + + // This comes from Terraform, as a test +variable "foo" { + # Standalone comment should be still here + + default = "bar" + description = "bar" # yooo +} + +/* This is a multi line standalone +comment*/ + + +// fatih arslan +/* This is a developer test +account and a multine comment */ +developer = [ "fatih", "arslan"] // fatih arslan + +# One line here +numbers = [1,2] // another line here + + # Another comment +variable = { + description = "bar" # another yooo + foo { + # Nested standalone + + bar = "fatih" + } +} + + // lead comment +foo { + bar = "fatih" // line comment 2 +} // line comment 3 + From aaca00993566dec65d4efefc5254d8c4f3e3dd2f Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 1 Dec 2016 10:22:15 -0500 Subject: [PATCH 3/4] Don't convert windows newlines If the canonical newline is a single LF, don't insert CRLF in tests --- decoder_test.go | 55 ++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/decoder_test.go b/decoder_test.go index 07b369d..62ecf9c 100644 --- a/decoder_test.go +++ b/decoder_test.go @@ -9,7 +9,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/testhelper" ) func TestDecode_interface(t *testing.T) { @@ -89,8 +88,7 @@ func TestDecode_interface(t *testing.T) { { "multiline_literal_with_hil.hcl", false, - map[string]interface{}{"multiline_literal_with_hil": testhelper.Unix2dos(`${hello - world}`)}, + map[string]interface{}{"multiline_literal_with_hil": "${hello\n world}"}, }, { "multiline_no_marker.hcl", @@ -100,22 +98,22 @@ func TestDecode_interface(t *testing.T) { { "multiline.hcl", false, - map[string]interface{}{"foo": testhelper.Unix2dos("bar\nbaz\n")}, + map[string]interface{}{"foo": "bar\nbaz\n"}, }, { "multiline_indented.hcl", false, - map[string]interface{}{"foo": testhelper.Unix2dos(" bar\n baz\n")}, + map[string]interface{}{"foo": " bar\n baz\n"}, }, { "multiline_no_hanging_indent.hcl", false, - map[string]interface{}{"foo": testhelper.Unix2dos(" baz\n bar\n foo\n")}, + map[string]interface{}{"foo": " baz\n bar\n foo\n"}, }, { "multiline_no_eof.hcl", false, - map[string]interface{}{"foo": testhelper.Unix2dos("bar\nbaz\n"), "key": "value"}, + map[string]interface{}{"foo": "bar\nbaz\n", "key": "value"}, }, { "multiline.json", @@ -394,31 +392,32 @@ func TestDecode_interface(t *testing.T) { } for _, tc := range cases { - t.Logf("Testing: %s", tc.File) - d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.File)) - if err != nil { - t.Fatalf("err: %s", err) - } + t.Run(tc.File, func(t *testing.T) { + d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.File)) + if err != nil { + t.Fatalf("err: %s", err) + } - var out interface{} - err = Decode(&out, string(d)) - if (err != nil) != tc.Err { - t.Fatalf("Input: %s\n\nError: %s", tc.File, err) - } + var out interface{} + err = Decode(&out, string(d)) + if (err != nil) != tc.Err { + t.Fatalf("Input: %s\n\nError: %s", tc.File, err) + } - if !reflect.DeepEqual(out, tc.Out) { - t.Fatalf("Input: %s. Actual, Expected.\n\n%#v\n\n%#v", tc.File, out, tc.Out) - } + if !reflect.DeepEqual(out, tc.Out) { + t.Fatalf("Input: %s. Actual, Expected.\n\n%#v\n\n%#v", tc.File, out, tc.Out) + } - var v interface{} - err = Unmarshal(d, &v) - if (err != nil) != tc.Err { - t.Fatalf("Input: %s\n\nError: %s", tc.File, err) - } + var v interface{} + err = Unmarshal(d, &v) + if (err != nil) != tc.Err { + t.Fatalf("Input: %s\n\nError: %s", tc.File, err) + } - if !reflect.DeepEqual(v, tc.Out) { - t.Fatalf("Input: %s. Actual, Expected.\n\n%#v\n\n%#v", tc.File, out, tc.Out) - } + if !reflect.DeepEqual(v, tc.Out) { + t.Fatalf("Input: %s. Actual, Expected.\n\n%#v\n\n%#v", tc.File, out, tc.Out) + } + }) } } From dd5495f43c43240bc0618beff7023e71e421c2a7 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 1 Dec 2016 13:05:58 -0500 Subject: [PATCH 4/4] Fix newlines in test-fixture hcl/parser/test-fixtures/comment.hcl already contained CRLF line endings. Convert that to unix and remove the extra cariiage returns from comment_crlf.hcl. --- hcl/parser/test-fixtures/comment.hcl | 30 +++++++++++------------ hcl/parser/test-fixtures/comment_crlf.hcl | 30 +++++++++++------------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/hcl/parser/test-fixtures/comment.hcl b/hcl/parser/test-fixtures/comment.hcl index 1ff7f29..e32be87 100644 --- a/hcl/parser/test-fixtures/comment.hcl +++ b/hcl/parser/test-fixtures/comment.hcl @@ -1,15 +1,15 @@ -// Foo - -/* Bar */ - -/* -/* -Baz -*/ - -# Another - -# Multiple -# Lines - -foo = "bar" +// Foo + +/* Bar */ + +/* +/* +Baz +*/ + +# Another + +# Multiple +# Lines + +foo = "bar" diff --git a/hcl/parser/test-fixtures/comment_crlf.hcl b/hcl/parser/test-fixtures/comment_crlf.hcl index 51dbcfd..1ff7f29 100644 --- a/hcl/parser/test-fixtures/comment_crlf.hcl +++ b/hcl/parser/test-fixtures/comment_crlf.hcl @@ -1,15 +1,15 @@ -// Foo - -/* Bar */ - -/* -/* -Baz -*/ - -# Another - -# Multiple -# Lines - -foo = "bar" +// Foo + +/* Bar */ + +/* +/* +Baz +*/ + +# Another + +# Multiple +# Lines + +foo = "bar"