Skip to content

Commit fc77e65

Browse files
authored
YAML multi-document support (#32)
Add YAML multi-document support
1 parent 191be70 commit fc77e65

9 files changed

Lines changed: 167 additions & 21 deletions

File tree

README.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ Comparable to [jq](https://github.com/stedolan/jq) / [yq](https://github.com/kis
3939
* [Put](#put)
4040
* [Put Object](#put-object)
4141
* [Supported file types](#supported-file-types)
42+
* [JSON](#json)
43+
* [TOML](#toml)
44+
* [YAML](#yaml)
45+
* [XML](#xml)
4246
* [Selectors](#selectors)
4347
* [Property](#property)
4448
* [Child](#child-elements)
@@ -334,7 +338,9 @@ Using [gopkg.in/yaml.v2](https://gopkg.in/yaml.v2).
334338
335339
#### Multi-document files
336340
337-
Multi-document files are not supported. This is due to [limitations in the yaml parser used](https://github.com/go-yaml/yaml/tree/v2.3.0#compatibility).
341+
Multi-document files are decoded into an array, with `[0]` being the first document, `[1]` being the second and so on.
342+
343+
Once decoded, you can access them using any of the standard selectors provided by Dasel.
338344
339345
### XML
340346
```bash

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
77
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
88
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
99
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
10-
github.com/clbanning/mxj v1.8.5-0.20201012155914-b957cfd48b51 h1:PK4/gQyzsZnkfJdEaWSybHdjsYORx+u34oqN9Sd63Rs=
11-
github.com/clbanning/mxj v1.8.5-0.20201012155914-b957cfd48b51/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
1210
github.com/clbanning/mxj/v2 v2.3.2 h1:DSkU65zfrBHtrggxd54X9pK1z/Lw2OwSW5D8p+x1toE=
1311
github.com/clbanning/mxj/v2 v2.3.2/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
1412
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -103,6 +101,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
103101
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
104102
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
105103
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
104+
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
106105
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
107106
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
108107
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -122,6 +121,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
122121
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
123122
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
124123
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
124+
golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
125125
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
126126
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
127127
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

internal/command/put.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,7 @@ func writeNodeToOutput(opts writeNodeToOutputOpts, cmd *cobra.Command) error {
133133
}
134134
return nil
135135
}
136-
137-
if err := storage.Write(opts.Parser, opts.Node.InterfaceValue(), opts.Writer); err != nil {
136+
if err := storage.Write(opts.Parser, opts.Node.InterfaceValue(), opts.Node.OriginalValue, opts.Writer); err != nil {
138137
return fmt.Errorf("could not write to output file: %w", err)
139138
}
140139

internal/command/root_put_test.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,10 @@ func TestRootCMD_Put_JSON(t *testing.T) {
131131
func TestRootCMD_Put_YAML(t *testing.T) {
132132
t.Run("String", putStringTest(`
133133
id: "x"
134+
name: "Tom"
134135
`, "yaml", "id", "y", `
135136
id: "y"
137+
name: Tom
136138
`, nil))
137139
t.Run("Int", putIntTest(`
138140
id: 123
@@ -160,6 +162,19 @@ numbers:
160162
rank: 2
161163
- number: three
162164
rank: 3
165+
`, nil))
166+
t.Run("StringInMultiDocument", putStringTest(`
167+
id: "x"
168+
---
169+
id: "y"
170+
---
171+
id: "z"
172+
`, "yaml", "[1].id", "1", `
173+
id: x
174+
---
175+
id: "1"
176+
---
177+
id: z
163178
`, nil))
164179
}
165180

internal/storage/parser.go

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,13 @@ func Load(p Parser, reader io.Reader) (interface{}, error) {
7979
}
8080

8181
// Write writes the value to the given io.Writer.
82-
func Write(p Parser, value interface{}, writer io.Writer) error {
82+
func Write(p Parser, value interface{}, originalValue interface{}, writer io.Writer) error {
83+
switch typed := originalValue.(type) {
84+
case OriginalRequired:
85+
if typed.OriginalRequired() {
86+
value = originalValue
87+
}
88+
}
8389
byteData, err := p.ToBytes(value)
8490
if err != nil {
8591
return fmt.Errorf("could not get byte data for file: %w", err)
@@ -89,3 +95,25 @@ func Write(p Parser, value interface{}, writer io.Writer) error {
8995
}
9096
return nil
9197
}
98+
99+
// OriginalRequired can be used in conjunction with RealValue to allow parsers to be more intelligent
100+
// with the data they read/write.
101+
type OriginalRequired interface {
102+
// OriginalRequired tells dasel if the parser requires the original value when converting to bytes.
103+
OriginalRequired() bool
104+
}
105+
106+
// RealValue can be used in conjunction with OriginalRequired to allow parsers to be more intelligent
107+
// with the data they read/write.
108+
type RealValue interface {
109+
// RealValue returns the real value that dasel should use when processing data.
110+
RealValue() interface{}
111+
}
112+
113+
type originalRequired struct {
114+
}
115+
116+
// OriginalRequired tells dasel if the parser requires the original value when converting to bytes.
117+
func (d originalRequired) OriginalRequired() bool {
118+
return true
119+
}

internal/storage/parser_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ func (fp *failingReader) Read(_ []byte) (n int, err error) {
172172
func TestWrite(t *testing.T) {
173173
t.Run("Success", func(t *testing.T) {
174174
var buf bytes.Buffer
175-
if err := storage.Write(&storage.JSONParser{}, map[string]interface{}{"name": "Tom"}, &buf); err != nil {
175+
if err := storage.Write(&storage.JSONParser{}, map[string]interface{}{"name": "Tom"}, nil, &buf); err != nil {
176176
t.Errorf("unexpected error: %s", err)
177177
return
178178
}
@@ -187,14 +187,14 @@ func TestWrite(t *testing.T) {
187187

188188
t.Run("ParserErrHandled", func(t *testing.T) {
189189
var buf bytes.Buffer
190-
if err := storage.Write(&failingParser{}, map[string]interface{}{"name": "Tom"}, &buf); !errors.Is(err, errFailingParserErr) {
190+
if err := storage.Write(&failingParser{}, map[string]interface{}{"name": "Tom"}, nil, &buf); !errors.Is(err, errFailingParserErr) {
191191
t.Errorf("unexpected error: %v", err)
192192
return
193193
}
194194
})
195195

196196
t.Run("WriterErrHandled", func(t *testing.T) {
197-
if err := storage.Write(&storage.JSONParser{}, map[string]interface{}{"name": "Tom"}, &failingWriter{}); !errors.Is(err, errFailingWriterErr) {
197+
if err := storage.Write(&storage.JSONParser{}, map[string]interface{}{"name": "Tom"}, nil, &failingWriter{}); !errors.Is(err, errFailingWriterErr) {
198198
t.Errorf("unexpected error: %v", err)
199199
return
200200
}

internal/storage/yaml.go

Lines changed: 67 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,86 @@
11
package storage
22

33
import (
4+
"bytes"
45
"fmt"
56
"gopkg.in/yaml.v2"
7+
"io"
68
)
79

810
// YAMLParser is a Parser implementation to handle yaml files.
911
type YAMLParser struct {
1012
}
1113

14+
// YAMLSingleDocument represents a decoded single-document YAML file.
15+
type YAMLSingleDocument struct {
16+
originalRequired
17+
Value interface{}
18+
}
19+
20+
// RealValue returns the real value that dasel should use when processing data.
21+
func (d *YAMLSingleDocument) RealValue() interface{} {
22+
return d.Value
23+
}
24+
25+
// YAMLMultiDocument represents a decoded multi-document YAML file.
26+
type YAMLMultiDocument struct {
27+
originalRequired
28+
Values []interface{}
29+
}
30+
31+
// RealValue returns the real value that dasel should use when processing data.
32+
func (d *YAMLMultiDocument) RealValue() interface{} {
33+
return d.Values
34+
}
35+
1236
// FromBytes returns some Data that is represented by the given bytes.
1337
func (p *YAMLParser) FromBytes(byteData []byte) (interface{}, error) {
14-
var data interface{}
15-
if err := yaml.Unmarshal(byteData, &data); err != nil {
16-
return data, fmt.Errorf("could not unmarshal config data: %w", err)
38+
res := make([]interface{}, 0)
39+
40+
decoder := yaml.NewDecoder(bytes.NewBuffer(byteData))
41+
42+
docLoop:
43+
for {
44+
var docData interface{}
45+
if err := decoder.Decode(&docData); err != nil {
46+
if err == io.EOF {
47+
break docLoop
48+
}
49+
return nil, fmt.Errorf("could not unmarshal config data: %w", err)
50+
}
51+
res = append(res, docData)
52+
}
53+
switch len(res) {
54+
case 0:
55+
return nil, nil
56+
case 1:
57+
return &YAMLSingleDocument{Value: res[0]}, nil
58+
default:
59+
return &YAMLMultiDocument{Values: res}, nil
1760
}
18-
return data, nil
1961
}
2062

2163
// ToBytes returns a slice of bytes that represents the given value.
2264
func (p *YAMLParser) ToBytes(value interface{}) ([]byte, error) {
23-
return yaml.Marshal(value)
65+
buffer := new(bytes.Buffer)
66+
encoder := yaml.NewEncoder(buffer)
67+
defer encoder.Close()
68+
69+
switch v := value.(type) {
70+
case *YAMLSingleDocument:
71+
if err := encoder.Encode(v.Value); err != nil {
72+
return nil, fmt.Errorf("could not encode single document: %w", err)
73+
}
74+
case *YAMLMultiDocument:
75+
for index, d := range v.Values {
76+
if err := encoder.Encode(d); err != nil {
77+
return nil, fmt.Errorf("could not encode multi document [%d]: %w", index, err)
78+
}
79+
}
80+
default:
81+
if err := encoder.Encode(v); err != nil {
82+
return nil, fmt.Errorf("could not encode default document type: %w", err)
83+
}
84+
}
85+
return buffer.Bytes(), nil
2486
}

internal/storage/yaml_test.go

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,33 @@ func TestYAMLParser_FromBytes(t *testing.T) {
2121
t.Errorf("unexpected error: %s", err)
2222
return
2323
}
24-
if !reflect.DeepEqual(yamlMap, got) {
25-
t.Errorf("expected %v, got %v", yamlMap, got)
24+
fmt.Printf("%T", got)
25+
exp := &storage.YAMLSingleDocument{Value: yamlMap}
26+
if !reflect.DeepEqual(exp, got) {
27+
t.Errorf("expected %v, got %v", exp, got)
28+
}
29+
})
30+
t.Run("ValidMultiDocument", func(t *testing.T) {
31+
got, err := (&storage.YAMLParser{}).FromBytes([]byte(`
32+
name: Tom
33+
---
34+
name: Jim
35+
`))
36+
if err != nil {
37+
t.Errorf("unexpected error: %s", err)
38+
return
39+
}
40+
exp := &storage.YAMLMultiDocument{Values: []interface{}{
41+
map[interface{}]interface{}{
42+
"name": "Tom",
43+
},
44+
map[interface{}]interface{}{
45+
"name": "Jim",
46+
},
47+
}}
48+
49+
if !reflect.DeepEqual(exp, got) {
50+
t.Errorf("expected %v, got %v", exp, got)
2651
}
2752
})
2853
t.Run("Invalid", func(t *testing.T) {

node.go

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package dasel
22

33
import (
44
"fmt"
5+
"github.com/tomwright/dasel/internal/storage"
56
"reflect"
67
"regexp"
78
"strconv"
@@ -32,7 +33,10 @@ type Node struct {
3233
Previous *Node `json:"-"`
3334
// Next is the next node in the chain.
3435
Next *Node `json:"next,omitempty"`
35-
36+
// OriginalValue is the value returned from the parser.
37+
// In most cases this is the same as Value, but is different for thr YAML parser
38+
// as it contains information on the original document.
39+
OriginalValue interface{} `json:"-"`
3640
// Value is the value of the current node.
3741
Value reflect.Value `json:"value"`
3842
// Selector is the selector for the current node.
@@ -147,11 +151,18 @@ func ParseSelector(selector string) (Selector, error) {
147151

148152
// New returns a new root node with the given value.
149153
func New(value interface{}) *Node {
150-
baseValue := reflect.ValueOf(value)
154+
var baseValue reflect.Value
155+
switch typed := value.(type) {
156+
case storage.RealValue:
157+
baseValue = reflect.ValueOf(typed.RealValue())
158+
default:
159+
baseValue = reflect.ValueOf(value)
160+
}
151161
rootNode := &Node{
152-
Previous: nil,
153-
Next: nil,
154-
Value: baseValue,
162+
Previous: nil,
163+
Next: nil,
164+
OriginalValue: value,
165+
Value: baseValue,
155166
Selector: Selector{
156167
Raw: ".",
157168
Current: ".",

0 commit comments

Comments
 (0)