From a5c0f7fdcce278f8aab81a222c9800cd62c907d3 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sun, 12 Aug 2018 10:08:27 -0700 Subject: [PATCH] cmd/hclspecsuite: Check for expected diagnostics When a test file declares one or more expected diagnostics, we check those instead of checking the result value. The severities and source ranges must match. We don't test the error messages themselves because they are not part of the specification and may vary between implementations or, in future, be translated into other languages. --- cmd/hclspecsuite/diagnostics.go | 19 +++ cmd/hclspecsuite/runner.go | 126 ++++++++++++++---- .../structure/attributes/singleline_bad.hcl | 1 + .../attributes/singleline_bad.hcldec | 3 + .../structure/attributes/singleline_bad.t | 19 +++ 5 files changed, 140 insertions(+), 28 deletions(-) create mode 100644 specsuite/tests/structure/attributes/singleline_bad.hcl create mode 100644 specsuite/tests/structure/attributes/singleline_bad.hcldec create mode 100644 specsuite/tests/structure/attributes/singleline_bad.t diff --git a/cmd/hclspecsuite/diagnostics.go b/cmd/hclspecsuite/diagnostics.go index 802c0cb..a559f1b 100644 --- a/cmd/hclspecsuite/diagnostics.go +++ b/cmd/hclspecsuite/diagnostics.go @@ -87,3 +87,22 @@ func decodeJSONDiagnostics(src []byte) hcl.Diagnostics { return diags } + +func severityString(severity hcl.DiagnosticSeverity) string { + switch severity { + case hcl.DiagError: + return "error" + case hcl.DiagWarning: + return "warning" + default: + return "unsupported-severity" + } +} + +func rangeString(rng hcl.Range) string { + return fmt.Sprintf( + "from line %d column %d byte %d to line %d column %d byte %d", + rng.Start.Line, rng.Start.Column, rng.Start.Byte, + rng.End.Line, rng.End.Column, rng.End.Byte, + ) +} diff --git a/cmd/hclspecsuite/runner.go b/cmd/hclspecsuite/runner.go index 977a3d7..e3d1235 100644 --- a/cmd/hclspecsuite/runner.go +++ b/cmd/hclspecsuite/runner.go @@ -169,44 +169,114 @@ func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile) } - val, moreDiags := r.hcldecTransform(specFilename, inputFilename) - diags = append(diags, moreDiags...) - if moreDiags.HasErrors() { - // If hcldec failed then there's no point in continuing. - return diags - } + val, transformDiags := r.hcldecTransform(specFilename, inputFilename) + if len(tf.ExpectedDiags) == 0 { + diags = append(diags, transformDiags...) + if transformDiags.HasErrors() { + // If hcldec failed then there's no point in continuing. + return diags + } - if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Incorrect result type", - Detail: fmt.Sprintf( - "Input file %s produced %s, but was expecting %s.", - inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType), - ), - }) - } - - if tf.Result != cty.NilVal { - cmpVal, err := convert.Convert(tf.Result, tf.ResultType) - if err != nil { + if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: "Incorrect type for result value", + Summary: "Incorrect result type", Detail: fmt.Sprintf( - "Result does not conform to the given result type: %s.", err, + "Input file %s produced %s, but was expecting %s.", + inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType), ), - Subject: &tf.ResultRange, }) - } else { - if !val.RawEquals(cmpVal) { + } + + if tf.Result != cty.NilVal { + cmpVal, err := convert.Convert(tf.Result, tf.ResultType) + if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: "Incorrect result value", + Summary: "Incorrect type for result value", Detail: fmt.Sprintf( - "Input file %s produced %#v, but was expecting %#v.", - inputFilename, val, tf.Result, + "Result does not conform to the given result type: %s.", err, ), + Subject: &tf.ResultRange, + }) + } else { + if !val.RawEquals(cmpVal) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Incorrect result value", + Detail: fmt.Sprintf( + "Input file %s produced %#v, but was expecting %#v.", + inputFilename, val, tf.Result, + ), + }) + } + } + } + } else { + // We're expecting diagnostics, and so we'll need to correlate the + // severities and source ranges of our actual diagnostics against + // what we were expecting. + type DiagnosticEntry struct { + Severity hcl.DiagnosticSeverity + Range hcl.Range + } + got := make(map[DiagnosticEntry]*hcl.Diagnostic) + want := make(map[DiagnosticEntry]hcl.Range) + for _, diag := range transformDiags { + if diag.Subject == nil { + // Sourceless diagnostics can never be expected, so we'll just + // pass these through as-is and assume they are hcldec + // operational errors. + diags = append(diags, diag) + continue + } + if diag.Subject.Filename != inputFilename { + // If the problem is for something other than the input file + // then it can't be expected. + diags = append(diags, diag) + continue + } + entry := DiagnosticEntry{ + Severity: diag.Severity, + Range: *diag.Subject, + } + got[entry] = diag + } + for _, e := range tf.ExpectedDiags { + e.Range.Filename = inputFilename // assumed here, since we don't allow any other filename to be expected + entry := DiagnosticEntry{ + Severity: e.Severity, + Range: e.Range, + } + want[entry] = e.DeclRange + } + + for gotEntry, diag := range got { + if _, wanted := want[gotEntry]; !wanted { + // Pass through the diagnostic itself so the user can see what happened + diags = append(diags, diag) + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unexpected diagnostic", + Detail: fmt.Sprintf( + "No %s diagnostic was expected %s. The unexpected diagnostic was shown above.", + severityString(gotEntry.Severity), rangeString(gotEntry.Range), + ), + Subject: &gotEntry.Range, + }) + } + } + + for wantEntry, declRange := range want { + if _, gotted := got[wantEntry]; !gotted { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing expected diagnostic", + Detail: fmt.Sprintf( + "No %s diagnostic was generated %s.", + severityString(wantEntry.Severity), rangeString(wantEntry.Range), + ), + Subject: &declRange, }) } } diff --git a/specsuite/tests/structure/attributes/singleline_bad.hcl b/specsuite/tests/structure/attributes/singleline_bad.hcl new file mode 100644 index 0000000..975b177 --- /dev/null +++ b/specsuite/tests/structure/attributes/singleline_bad.hcl @@ -0,0 +1 @@ +a = "a value", b = "b value" diff --git a/specsuite/tests/structure/attributes/singleline_bad.hcldec b/specsuite/tests/structure/attributes/singleline_bad.hcldec new file mode 100644 index 0000000..5d4a5b6 --- /dev/null +++ b/specsuite/tests/structure/attributes/singleline_bad.hcldec @@ -0,0 +1,3 @@ +literal { + value = null +} diff --git a/specsuite/tests/structure/attributes/singleline_bad.t b/specsuite/tests/structure/attributes/singleline_bad.t new file mode 100644 index 0000000..98610b7 --- /dev/null +++ b/specsuite/tests/structure/attributes/singleline_bad.t @@ -0,0 +1,19 @@ +# This test verifies that comma-separated attributes on the same line are +# reported as an error, rather than being parsed like an object constructor +# expression. + +diagnostics { + error { + # Message like "missing newline after argument" or "each argument must be on its own line" + from { + line = 1 + column = 14 + byte = 13 + } + to { + line = 1 + column = 15 + byte = 14 + } + } +}