diff --git a/cmd/hclspecsuite/diagnostics.go b/cmd/hclspecsuite/diagnostics.go index 802c0cb..a559f1b 100644 --- a/cmd/hclspecsuite/diagnostics.go +++ b/cmd/hclspecsuite/diagnostics.go @@ -87,3 +87,22 @@ func decodeJSONDiagnostics(src []byte) hcl.Diagnostics { return diags } + +func severityString(severity hcl.DiagnosticSeverity) string { + switch severity { + case hcl.DiagError: + return "error" + case hcl.DiagWarning: + return "warning" + default: + return "unsupported-severity" + } +} + +func rangeString(rng hcl.Range) string { + return fmt.Sprintf( + "from line %d column %d byte %d to line %d column %d byte %d", + rng.Start.Line, rng.Start.Column, rng.Start.Byte, + rng.End.Line, rng.End.Column, rng.End.Byte, + ) +} diff --git a/cmd/hclspecsuite/runner.go b/cmd/hclspecsuite/runner.go index 977a3d7..e3d1235 100644 --- a/cmd/hclspecsuite/runner.go +++ b/cmd/hclspecsuite/runner.go @@ -169,44 +169,114 @@ func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile) } - val, moreDiags := r.hcldecTransform(specFilename, inputFilename) - diags = append(diags, moreDiags...) - if moreDiags.HasErrors() { - // If hcldec failed then there's no point in continuing. - return diags - } + val, transformDiags := r.hcldecTransform(specFilename, inputFilename) + if len(tf.ExpectedDiags) == 0 { + diags = append(diags, transformDiags...) + if transformDiags.HasErrors() { + // If hcldec failed then there's no point in continuing. + return diags + } - if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Incorrect result type", - Detail: fmt.Sprintf( - "Input file %s produced %s, but was expecting %s.", - inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType), - ), - }) - } - - if tf.Result != cty.NilVal { - cmpVal, err := convert.Convert(tf.Result, tf.ResultType) - if err != nil { + if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: "Incorrect type for result value", + Summary: "Incorrect result type", Detail: fmt.Sprintf( - "Result does not conform to the given result type: %s.", err, + "Input file %s produced %s, but was expecting %s.", + inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType), ), - Subject: &tf.ResultRange, }) - } else { - if !val.RawEquals(cmpVal) { + } + + if tf.Result != cty.NilVal { + cmpVal, err := convert.Convert(tf.Result, tf.ResultType) + if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: "Incorrect result value", + Summary: "Incorrect type for result value", Detail: fmt.Sprintf( - "Input file %s produced %#v, but was expecting %#v.", - inputFilename, val, tf.Result, + "Result does not conform to the given result type: %s.", err, ), + Subject: &tf.ResultRange, + }) + } else { + if !val.RawEquals(cmpVal) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Incorrect result value", + Detail: fmt.Sprintf( + "Input file %s produced %#v, but was expecting %#v.", + inputFilename, val, tf.Result, + ), + }) + } + } + } + } else { + // We're expecting diagnostics, and so we'll need to correlate the + // severities and source ranges of our actual diagnostics against + // what we were expecting. + type DiagnosticEntry struct { + Severity hcl.DiagnosticSeverity + Range hcl.Range + } + got := make(map[DiagnosticEntry]*hcl.Diagnostic) + want := make(map[DiagnosticEntry]hcl.Range) + for _, diag := range transformDiags { + if diag.Subject == nil { + // Sourceless diagnostics can never be expected, so we'll just + // pass these through as-is and assume they are hcldec + // operational errors. + diags = append(diags, diag) + continue + } + if diag.Subject.Filename != inputFilename { + // If the problem is for something other than the input file + // then it can't be expected. + diags = append(diags, diag) + continue + } + entry := DiagnosticEntry{ + Severity: diag.Severity, + Range: *diag.Subject, + } + got[entry] = diag + } + for _, e := range tf.ExpectedDiags { + e.Range.Filename = inputFilename // assumed here, since we don't allow any other filename to be expected + entry := DiagnosticEntry{ + Severity: e.Severity, + Range: e.Range, + } + want[entry] = e.DeclRange + } + + for gotEntry, diag := range got { + if _, wanted := want[gotEntry]; !wanted { + // Pass through the diagnostic itself so the user can see what happened + diags = append(diags, diag) + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unexpected diagnostic", + Detail: fmt.Sprintf( + "No %s diagnostic was expected %s. The unexpected diagnostic was shown above.", + severityString(gotEntry.Severity), rangeString(gotEntry.Range), + ), + Subject: &gotEntry.Range, + }) + } + } + + for wantEntry, declRange := range want { + if _, gotted := got[wantEntry]; !gotted { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing expected diagnostic", + Detail: fmt.Sprintf( + "No %s diagnostic was generated %s.", + severityString(wantEntry.Severity), rangeString(wantEntry.Range), + ), + Subject: &declRange, }) } } diff --git a/specsuite/tests/structure/attributes/singleline_bad.hcl b/specsuite/tests/structure/attributes/singleline_bad.hcl new file mode 100644 index 0000000..975b177 --- /dev/null +++ b/specsuite/tests/structure/attributes/singleline_bad.hcl @@ -0,0 +1 @@ +a = "a value", b = "b value" diff --git a/specsuite/tests/structure/attributes/singleline_bad.hcldec b/specsuite/tests/structure/attributes/singleline_bad.hcldec new file mode 100644 index 0000000..5d4a5b6 --- /dev/null +++ b/specsuite/tests/structure/attributes/singleline_bad.hcldec @@ -0,0 +1,3 @@ +literal { + value = null +} diff --git a/specsuite/tests/structure/attributes/singleline_bad.t b/specsuite/tests/structure/attributes/singleline_bad.t new file mode 100644 index 0000000..98610b7 --- /dev/null +++ b/specsuite/tests/structure/attributes/singleline_bad.t @@ -0,0 +1,19 @@ +# This test verifies that comma-separated attributes on the same line are +# reported as an error, rather than being parsed like an object constructor +# expression. + +diagnostics { + error { + # Message like "missing newline after argument" or "each argument must be on its own line" + from { + line = 1 + column = 14 + byte = 13 + } + to { + line = 1 + column = 15 + byte = 14 + } + } +}