gohcl: retain nested blocks while decoding

Currently, if nonzero struct is passed to the DecodeBody function,
decoding process will keep already initialized top-level fields values or
overwrite them, if they are specified in HCL. This behaviour is useful,
as it allows to have some default values for top-level fields.

However, if the field is a type block or slice (multiple blocks), then the
entire block is overwritten, which erases the existing values. Because of
that, setting default values in nested structs is not possible.

With this commit, decode functions will check if the value is
nil and only then set them to empty struct, which allows for appending
to existing structs.

In case of a slice, either new empty element will be added, or existing
element will be used for setting new value (so values will be merged).

Also, to keep the same behavior as json.Unmarshal, if retained list
have more elements than new list, additional elements will be removed
and existing elements will be merged. This allows to have default values
also for positional elements.

Behavior added by this patch is the same as in json.Unmarshal and
yaml.Unmarshal, which both retain nested structs during unmarshaling
process, so I believe this is an expected behavior from user
perspective.
This commit is contained in:
Mateusz Gozdek 2019-10-10 18:15:24 +02:00 committed by Martin Atkins
parent 6ca13f360e
commit 42351b1d15
2 changed files with 191 additions and 63 deletions

View File

@ -147,7 +147,9 @@ func decodeBodyToStruct(body hcl.Body, ctx *hcl.EvalContext, val reflect.Value)
if len(blocks) == 0 { if len(blocks) == 0 {
if isSlice || isPtr { if isSlice || isPtr {
val.Field(fieldIdx).Set(reflect.Zero(field.Type)) if val.Field(fieldIdx).IsNil() {
val.Field(fieldIdx).Set(reflect.Zero(field.Type))
}
} else { } else {
diags = append(diags, &hcl.Diagnostic{ diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError, Severity: hcl.DiagError,
@ -166,11 +168,20 @@ func decodeBodyToStruct(body hcl.Body, ctx *hcl.EvalContext, val reflect.Value)
if isPtr { if isPtr {
elemType = reflect.PtrTo(ty) elemType = reflect.PtrTo(ty)
} }
sli := reflect.MakeSlice(reflect.SliceOf(elemType), len(blocks), len(blocks)) sli := val.Field(fieldIdx)
if sli.IsNil() {
sli = reflect.MakeSlice(reflect.SliceOf(elemType), len(blocks), len(blocks))
}
for i, block := range blocks { for i, block := range blocks {
if isPtr { if isPtr {
v := reflect.New(ty) if i >= sli.Len() {
sli = reflect.Append(sli, reflect.New(ty))
}
v := sli.Index(i)
if v.IsNil() {
v = reflect.New(ty)
}
diags = append(diags, decodeBlockToValue(block, ctx, v.Elem())...) diags = append(diags, decodeBlockToValue(block, ctx, v.Elem())...)
sli.Index(i).Set(v) sli.Index(i).Set(v)
} else { } else {
@ -178,12 +189,19 @@ func decodeBodyToStruct(body hcl.Body, ctx *hcl.EvalContext, val reflect.Value)
} }
} }
if sli.Len() > len(blocks) {
sli.SetLen(len(blocks))
}
val.Field(fieldIdx).Set(sli) val.Field(fieldIdx).Set(sli)
default: default:
block := blocks[0] block := blocks[0]
if isPtr { if isPtr {
v := reflect.New(ty) v := val.Field(fieldIdx)
if v.IsNil() {
v = reflect.New(ty)
}
diags = append(diags, decodeBlockToValue(block, ctx, v.Elem())...) diags = append(diags, decodeBlockToValue(block, ctx, v.Elem())...)
val.Field(fieldIdx).Set(v) val.Field(fieldIdx).Set(v)
} else { } else {

View File

@ -23,23 +23,37 @@ func TestDecodeBody(t *testing.T) {
Name hcl.Expression `hcl:"name"` Name hcl.Expression `hcl:"name"`
} }
type withTwoAttributes struct {
A string `hcl:"a,optional"`
B string `hcl:"b,optional"`
}
type withNestedBlock struct {
Plain string `hcl:"plain,optional"`
Nested *withTwoAttributes `hcl:"nested,block"`
}
type withListofNestedBlocks struct {
Nested []*withTwoAttributes `hcl:"nested,block"`
}
tests := []struct { tests := []struct {
Body map[string]interface{} Body map[string]interface{}
Target interface{} Target func() interface{}
Check func(v interface{}) bool Check func(v interface{}) bool
DiagCount int DiagCount int
}{ }{
{ {
map[string]interface{}{}, map[string]interface{}{},
struct{}{}, makeInstantiateType(struct{}{}),
deepEquals(struct{}{}), deepEquals(struct{}{}),
0, 0,
}, },
{ {
map[string]interface{}{}, map[string]interface{}{},
struct { makeInstantiateType(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
}{}, }{}),
deepEquals(struct { deepEquals(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
}{}), }{}),
@ -47,9 +61,9 @@ func TestDecodeBody(t *testing.T) {
}, },
{ {
map[string]interface{}{}, map[string]interface{}{},
struct { makeInstantiateType(struct {
Name *string `hcl:"name"` Name *string `hcl:"name"`
}{}, }{}),
deepEquals(struct { deepEquals(struct {
Name *string `hcl:"name"` Name *string `hcl:"name"`
}{}), }{}),
@ -57,9 +71,9 @@ func TestDecodeBody(t *testing.T) {
}, // name nil }, // name nil
{ {
map[string]interface{}{}, map[string]interface{}{},
struct { makeInstantiateType(struct {
Name string `hcl:"name,optional"` Name string `hcl:"name,optional"`
}{}, }{}),
deepEquals(struct { deepEquals(struct {
Name string `hcl:"name,optional"` Name string `hcl:"name,optional"`
}{}), }{}),
@ -67,7 +81,7 @@ func TestDecodeBody(t *testing.T) {
}, // name optional }, // name optional
{ {
map[string]interface{}{}, map[string]interface{}{},
withNameExpression{}, makeInstantiateType(withNameExpression{}),
func(v interface{}) bool { func(v interface{}) bool {
if v == nil { if v == nil {
return false return false
@ -95,7 +109,7 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"name": "Ermintrude", "name": "Ermintrude",
}, },
withNameExpression{}, makeInstantiateType(withNameExpression{}),
func(v interface{}) bool { func(v interface{}) bool {
if v == nil { if v == nil {
return false return false
@ -123,9 +137,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"name": "Ermintrude", "name": "Ermintrude",
}, },
struct { makeInstantiateType(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
}{}, }{}),
deepEquals(struct { deepEquals(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
}{"Ermintrude"}), }{"Ermintrude"}),
@ -136,9 +150,9 @@ func TestDecodeBody(t *testing.T) {
"name": "Ermintrude", "name": "Ermintrude",
"age": 23, "age": 23,
}, },
struct { makeInstantiateType(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
}{}, }{}),
deepEquals(struct { deepEquals(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
}{"Ermintrude"}), }{"Ermintrude"}),
@ -149,10 +163,10 @@ func TestDecodeBody(t *testing.T) {
"name": "Ermintrude", "name": "Ermintrude",
"age": 50, "age": 50,
}, },
struct { makeInstantiateType(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
Attrs hcl.Attributes `hcl:",remain"` Attrs hcl.Attributes `hcl:",remain"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
got := gotI.(struct { got := gotI.(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
@ -167,10 +181,10 @@ func TestDecodeBody(t *testing.T) {
"name": "Ermintrude", "name": "Ermintrude",
"age": 50, "age": 50,
}, },
struct { makeInstantiateType(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
Remain hcl.Body `hcl:",remain"` Remain hcl.Body `hcl:",remain"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
got := gotI.(struct { got := gotI.(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
@ -188,10 +202,10 @@ func TestDecodeBody(t *testing.T) {
"name": "Ermintrude", "name": "Ermintrude",
"living": true, "living": true,
}, },
struct { makeInstantiateType(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
Remain map[string]cty.Value `hcl:",remain"` Remain map[string]cty.Value `hcl:",remain"`
}{}, }{}),
deepEquals(struct { deepEquals(struct {
Name string `hcl:"name"` Name string `hcl:"name"`
Remain map[string]cty.Value `hcl:",remain"` Remain map[string]cty.Value `hcl:",remain"`
@ -207,9 +221,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": map[string]interface{}{}, "noodle": map[string]interface{}{},
}, },
struct { makeInstantiateType(struct {
Noodle struct{} `hcl:"noodle,block"` Noodle struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
// Generating no diagnostics is good enough for this one. // Generating no diagnostics is good enough for this one.
return true return true
@ -220,9 +234,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": []map[string]interface{}{{}}, "noodle": []map[string]interface{}{{}},
}, },
struct { makeInstantiateType(struct {
Noodle struct{} `hcl:"noodle,block"` Noodle struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
// Generating no diagnostics is good enough for this one. // Generating no diagnostics is good enough for this one.
return true return true
@ -233,9 +247,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": []map[string]interface{}{{}, {}}, "noodle": []map[string]interface{}{{}, {}},
}, },
struct { makeInstantiateType(struct {
Noodle struct{} `hcl:"noodle,block"` Noodle struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
// Generating one diagnostic is good enough for this one. // Generating one diagnostic is good enough for this one.
return true return true
@ -244,9 +258,9 @@ func TestDecodeBody(t *testing.T) {
}, },
{ {
map[string]interface{}{}, map[string]interface{}{},
struct { makeInstantiateType(struct {
Noodle struct{} `hcl:"noodle,block"` Noodle struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
// Generating one diagnostic is good enough for this one. // Generating one diagnostic is good enough for this one.
return true return true
@ -257,9 +271,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": []map[string]interface{}{}, "noodle": []map[string]interface{}{},
}, },
struct { makeInstantiateType(struct {
Noodle struct{} `hcl:"noodle,block"` Noodle struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
// Generating one diagnostic is good enough for this one. // Generating one diagnostic is good enough for this one.
return true return true
@ -270,9 +284,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": map[string]interface{}{}, "noodle": map[string]interface{}{},
}, },
struct { makeInstantiateType(struct {
Noodle *struct{} `hcl:"noodle,block"` Noodle *struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
return gotI.(struct { return gotI.(struct {
Noodle *struct{} `hcl:"noodle,block"` Noodle *struct{} `hcl:"noodle,block"`
@ -284,9 +298,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": []map[string]interface{}{{}}, "noodle": []map[string]interface{}{{}},
}, },
struct { makeInstantiateType(struct {
Noodle *struct{} `hcl:"noodle,block"` Noodle *struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
return gotI.(struct { return gotI.(struct {
Noodle *struct{} `hcl:"noodle,block"` Noodle *struct{} `hcl:"noodle,block"`
@ -298,9 +312,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": []map[string]interface{}{}, "noodle": []map[string]interface{}{},
}, },
struct { makeInstantiateType(struct {
Noodle *struct{} `hcl:"noodle,block"` Noodle *struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
return gotI.(struct { return gotI.(struct {
Noodle *struct{} `hcl:"noodle,block"` Noodle *struct{} `hcl:"noodle,block"`
@ -312,9 +326,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": []map[string]interface{}{{}, {}}, "noodle": []map[string]interface{}{{}, {}},
}, },
struct { makeInstantiateType(struct {
Noodle *struct{} `hcl:"noodle,block"` Noodle *struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
// Generating one diagnostic is good enough for this one. // Generating one diagnostic is good enough for this one.
return true return true
@ -325,9 +339,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": []map[string]interface{}{}, "noodle": []map[string]interface{}{},
}, },
struct { makeInstantiateType(struct {
Noodle []struct{} `hcl:"noodle,block"` Noodle []struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
noodle := gotI.(struct { noodle := gotI.(struct {
Noodle []struct{} `hcl:"noodle,block"` Noodle []struct{} `hcl:"noodle,block"`
@ -340,9 +354,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": []map[string]interface{}{{}}, "noodle": []map[string]interface{}{{}},
}, },
struct { makeInstantiateType(struct {
Noodle []struct{} `hcl:"noodle,block"` Noodle []struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
noodle := gotI.(struct { noodle := gotI.(struct {
Noodle []struct{} `hcl:"noodle,block"` Noodle []struct{} `hcl:"noodle,block"`
@ -355,9 +369,9 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": []map[string]interface{}{{}, {}}, "noodle": []map[string]interface{}{{}, {}},
}, },
struct { makeInstantiateType(struct {
Noodle []struct{} `hcl:"noodle,block"` Noodle []struct{} `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
noodle := gotI.(struct { noodle := gotI.(struct {
Noodle []struct{} `hcl:"noodle,block"` Noodle []struct{} `hcl:"noodle,block"`
@ -370,11 +384,11 @@ func TestDecodeBody(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"noodle": map[string]interface{}{}, "noodle": map[string]interface{}{},
}, },
struct { makeInstantiateType(struct {
Noodle struct { Noodle struct {
Name string `hcl:"name,label"` Name string `hcl:"name,label"`
} `hcl:"noodle,block"` } `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
// Generating two diagnostics is good enough for this one. // Generating two diagnostics is good enough for this one.
// (one for the missing noodle block and the other for // (one for the missing noodle block and the other for
@ -390,11 +404,11 @@ func TestDecodeBody(t *testing.T) {
"foo_foo": map[string]interface{}{}, "foo_foo": map[string]interface{}{},
}, },
}, },
struct { makeInstantiateType(struct {
Noodle struct { Noodle struct {
Name string `hcl:"name,label"` Name string `hcl:"name,label"`
} `hcl:"noodle,block"` } `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
noodle := gotI.(struct { noodle := gotI.(struct {
Noodle struct { Noodle struct {
@ -412,11 +426,11 @@ func TestDecodeBody(t *testing.T) {
"bar_baz": map[string]interface{}{}, "bar_baz": map[string]interface{}{},
}, },
}, },
struct { makeInstantiateType(struct {
Noodle struct { Noodle struct {
Name string `hcl:"name,label"` Name string `hcl:"name,label"`
} `hcl:"noodle,block"` } `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
// One diagnostic is enough for this one. // One diagnostic is enough for this one.
return true return true
@ -430,11 +444,11 @@ func TestDecodeBody(t *testing.T) {
"bar_baz": map[string]interface{}{}, "bar_baz": map[string]interface{}{},
}, },
}, },
struct { makeInstantiateType(struct {
Noodles []struct { Noodles []struct {
Name string `hcl:"name,label"` Name string `hcl:"name,label"`
} `hcl:"noodle,block"` } `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
noodles := gotI.(struct { noodles := gotI.(struct {
Noodles []struct { Noodles []struct {
@ -453,12 +467,12 @@ func TestDecodeBody(t *testing.T) {
}, },
}, },
}, },
struct { makeInstantiateType(struct {
Noodle struct { Noodle struct {
Name string `hcl:"name,label"` Name string `hcl:"name,label"`
Type string `hcl:"type"` Type string `hcl:"type"`
} `hcl:"noodle,block"` } `hcl:"noodle,block"`
}{}, }{}),
func(gotI interface{}) bool { func(gotI interface{}) bool {
noodle := gotI.(struct { noodle := gotI.(struct {
Noodle struct { Noodle struct {
@ -476,7 +490,7 @@ func TestDecodeBody(t *testing.T) {
"name": "Ermintrude", "name": "Ermintrude",
"age": 34, "age": 34,
}, },
map[string]string(nil), makeInstantiateType(map[string]string(nil)),
deepEquals(map[string]string{ deepEquals(map[string]string{
"name": "Ermintrude", "name": "Ermintrude",
"age": "34", "age": "34",
@ -488,7 +502,7 @@ func TestDecodeBody(t *testing.T) {
"name": "Ermintrude", "name": "Ermintrude",
"age": 89, "age": 89,
}, },
map[string]*hcl.Attribute(nil), makeInstantiateType(map[string]*hcl.Attribute(nil)),
func(gotI interface{}) bool { func(gotI interface{}) bool {
got := gotI.(map[string]*hcl.Attribute) got := gotI.(map[string]*hcl.Attribute)
return len(got) == 2 && got["name"] != nil && got["age"] != nil return len(got) == 2 && got["name"] != nil && got["age"] != nil
@ -500,7 +514,7 @@ func TestDecodeBody(t *testing.T) {
"name": "Ermintrude", "name": "Ermintrude",
"age": 13, "age": 13,
}, },
map[string]hcl.Expression(nil), makeInstantiateType(map[string]hcl.Expression(nil)),
func(gotI interface{}) bool { func(gotI interface{}) bool {
got := gotI.(map[string]hcl.Expression) got := gotI.(map[string]hcl.Expression)
return len(got) == 2 && got["name"] != nil && got["age"] != nil return len(got) == 2 && got["name"] != nil && got["age"] != nil
@ -512,13 +526,103 @@ func TestDecodeBody(t *testing.T) {
"name": "Ermintrude", "name": "Ermintrude",
"living": true, "living": true,
}, },
map[string]cty.Value(nil), makeInstantiateType(map[string]cty.Value(nil)),
deepEquals(map[string]cty.Value{ deepEquals(map[string]cty.Value{
"name": cty.StringVal("Ermintrude"), "name": cty.StringVal("Ermintrude"),
"living": cty.True, "living": cty.True,
}), }),
0, 0,
}, },
{
// Retain "nested" block while decoding
map[string]interface{}{
"plain": "foo",
},
func() interface{} {
return &withNestedBlock{
Plain: "bar",
Nested: &withTwoAttributes{
A: "bar",
},
}
},
func(gotI interface{}) bool {
foo := gotI.(withNestedBlock)
return foo.Plain == "foo" && foo.Nested != nil && foo.Nested.A == "bar"
},
0,
},
{
// Retain values in "nested" block while decoding
map[string]interface{}{
"nested": map[string]interface{}{
"a": "foo",
},
},
func() interface{} {
return &withNestedBlock{
Nested: &withTwoAttributes{
B: "bar",
},
}
},
func(gotI interface{}) bool {
foo := gotI.(withNestedBlock)
return foo.Nested.A == "foo" && foo.Nested.B == "bar"
},
0,
},
{
// Retain values in "nested" block list while decoding
map[string]interface{}{
"nested": []map[string]interface{}{
{
"a": "foo",
},
},
},
func() interface{} {
return &withListofNestedBlocks{
Nested: []*withTwoAttributes{
&withTwoAttributes{
B: "bar",
},
},
}
},
func(gotI interface{}) bool {
n := gotI.(withListofNestedBlocks)
return n.Nested[0].A == "foo" && n.Nested[0].B == "bar"
},
0,
},
{
// Remove additional elements from the list while decoding nested blocks
map[string]interface{}{
"nested": []map[string]interface{}{
{
"a": "foo",
},
},
},
func() interface{} {
return &withListofNestedBlocks{
Nested: []*withTwoAttributes{
&withTwoAttributes{
B: "bar",
},
&withTwoAttributes{
B: "bar",
},
},
}
},
func(gotI interface{}) bool {
n := gotI.(withListofNestedBlocks)
return len(n.Nested) == 1
},
0,
},
} }
for i, test := range tests { for i, test := range tests {
@ -535,7 +639,7 @@ func TestDecodeBody(t *testing.T) {
t.Fatalf("diagnostics while parsing: %s", diags.Error()) t.Fatalf("diagnostics while parsing: %s", diags.Error())
} }
targetVal := reflect.New(reflect.TypeOf(test.Target)) targetVal := reflect.ValueOf(test.Target())
diags = DecodeBody(file.Body, nil, targetVal.Interface()) diags = DecodeBody(file.Body, nil, targetVal.Interface())
if len(diags) != test.DiagCount { if len(diags) != test.DiagCount {
@ -643,3 +747,9 @@ func (e *fixedExpression) StartRange() (r hcl.Range) {
func (e *fixedExpression) Variables() []hcl.Traversal { func (e *fixedExpression) Variables() []hcl.Traversal {
return nil return nil
} }
func makeInstantiateType(target interface{}) func() interface{} {
return func() interface{} {
return reflect.New(reflect.TypeOf(target)).Interface()
}
}