fix(api): fix YAML merge functionality
- implement recursive merging of YAML with patch support - preserve comments and formatting during merge operations - add tests for various merge scenarios in config_test.go
This commit is contained in:
Sergey Krashevich
+83
-26
@@ -1,11 +1,15 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
|
||||
"github.com/AlexxIT/go2rtc/internal/app"
|
||||
pkgyaml "github.com/AlexxIT/go2rtc/pkg/yaml"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -55,47 +59,100 @@ func configHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func mergeYAML(file1 string, yaml2 []byte) ([]byte, error) {
|
||||
// Read the contents of the first YAML file
|
||||
data1, err := os.ReadFile(file1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Unmarshal the first YAML file into a map
|
||||
var config1 map[string]any
|
||||
if err = yaml.Unmarshal(data1, &config1); err != nil {
|
||||
var patch map[string]any
|
||||
if err = yaml.Unmarshal(yaml2, &patch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Unmarshal the second YAML document into a map
|
||||
var config2 map[string]any
|
||||
if err = yaml.Unmarshal(yaml2, &config2); err != nil {
|
||||
data1, err = mergeYAMLMap(data1, nil, patch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Merge the two maps
|
||||
config1 = merge(config1, config2)
|
||||
// validate config after merge
|
||||
if err = yaml.Unmarshal(data1, map[string]any{}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Marshal the merged map into YAML
|
||||
return yaml.Marshal(&config1)
|
||||
return data1, nil
|
||||
}
|
||||
|
||||
func merge(dst, src map[string]any) map[string]any {
|
||||
for k, v := range src {
|
||||
if vv, ok := dst[k]; ok {
|
||||
switch vv := vv.(type) {
|
||||
case map[string]any:
|
||||
v := v.(map[string]any)
|
||||
dst[k] = merge(vv, v)
|
||||
case []any:
|
||||
v := v.([]any)
|
||||
dst[k] = v
|
||||
default:
|
||||
dst[k] = v
|
||||
// mergeYAMLMap recursively applies patch values onto config bytes.
|
||||
func mergeYAMLMap(data []byte, path []string, patch map[string]any) ([]byte, error) {
|
||||
for _, key := range slices.Sorted(maps.Keys(patch)) {
|
||||
value := patch[key]
|
||||
currPath := append(append([]string(nil), path...), key)
|
||||
|
||||
if valueMap, ok := value.(map[string]any); ok {
|
||||
isMap, exists, err := pathIsMapping(data, currPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
dst[k] = v
|
||||
|
||||
if exists && isMap {
|
||||
data, err = mergeYAMLMap(data, currPath, valueMap)
|
||||
} else {
|
||||
data, err = pkgyaml.Patch(data, currPath, valueMap)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var err error
|
||||
data, err = pkgyaml.Patch(data, currPath, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return dst
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// pathIsMapping reports whether path exists and ends with a mapping node.
|
||||
func pathIsMapping(data []byte, path []string) (isMap, exists bool, err error) {
|
||||
var root yaml.Node
|
||||
if err = yaml.Unmarshal(data, &root); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
if len(root.Content) == 0 {
|
||||
return false, false, nil
|
||||
}
|
||||
|
||||
if len(root.Content) != 1 || root.Content[0].Kind != yaml.MappingNode {
|
||||
return false, false, errors.New("yaml: expected mapping document")
|
||||
}
|
||||
|
||||
node := root.Content[0]
|
||||
for i, part := range path {
|
||||
idx := -1
|
||||
for j := 0; j < len(node.Content); j += 2 {
|
||||
if node.Content[j].Value == part {
|
||||
idx = j
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
return false, false, nil
|
||||
}
|
||||
|
||||
valueNode := node.Content[idx+1]
|
||||
if i == len(path)-1 {
|
||||
return valueNode.Kind == yaml.MappingNode, true, nil
|
||||
}
|
||||
|
||||
if valueNode.Kind != yaml.MappingNode {
|
||||
return false, false, nil
|
||||
}
|
||||
|
||||
node = valueNode
|
||||
}
|
||||
|
||||
return false, false, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestMergeYAMLPreserveCommentedStreamList(t *testing.T) {
|
||||
base := `streams:
|
||||
yard:
|
||||
- #http://1.1.1.1
|
||||
- #http://2.2.2.2
|
||||
- http://3.3.3.3
|
||||
- #http://4.4.4.4
|
||||
log:
|
||||
level: trace
|
||||
`
|
||||
patch := `log:
|
||||
api: debug
|
||||
`
|
||||
|
||||
path := filepath.Join(t.TempDir(), "go2rtc.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(base), 0o644))
|
||||
|
||||
out, err := mergeYAML(path, []byte(patch))
|
||||
require.NoError(t, err)
|
||||
|
||||
merged := string(out)
|
||||
require.Contains(t, merged, "#http://1.1.1.1")
|
||||
require.Contains(t, merged, "#http://2.2.2.2")
|
||||
require.Contains(t, merged, "#http://4.4.4.4")
|
||||
require.Contains(t, merged, "- http://3.3.3.3")
|
||||
require.NotContains(t, merged, "- null")
|
||||
require.Contains(t, merged, "api: debug")
|
||||
|
||||
var cfg map[string]any
|
||||
require.NoError(t, yaml.Unmarshal(out, &cfg))
|
||||
}
|
||||
|
||||
func TestMergeYAMLPreserveUnchangedComments(t *testing.T) {
|
||||
base := `api:
|
||||
username: admin
|
||||
streams:
|
||||
yard:
|
||||
- #http://1.1.1.1
|
||||
- http://3.3.3.3
|
||||
`
|
||||
patch := `api:
|
||||
password: secret
|
||||
`
|
||||
|
||||
path := filepath.Join(t.TempDir(), "go2rtc.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(base), 0o644))
|
||||
|
||||
out, err := mergeYAML(path, []byte(patch))
|
||||
require.NoError(t, err)
|
||||
|
||||
merged := string(out)
|
||||
require.Contains(t, merged, "username: admin")
|
||||
require.Contains(t, merged, "password: secret")
|
||||
require.Contains(t, merged, "#http://1.1.1.1")
|
||||
require.NotContains(t, merged, "- null")
|
||||
}
|
||||
|
||||
func TestMergeYAMLPreserveCommentsAndFormattingAcrossSections(t *testing.T) {
|
||||
base := `# global config comment
|
||||
api: # api section comment
|
||||
username: admin # inline username comment
|
||||
streams:
|
||||
# stream comment
|
||||
yard:
|
||||
- #http://1.1.1.1
|
||||
- http://3.3.3.3
|
||||
log:
|
||||
format: |
|
||||
line1
|
||||
line2
|
||||
`
|
||||
patch := `api:
|
||||
password: "secret value"
|
||||
ffmpeg:
|
||||
bin: /usr/bin/ffmpeg
|
||||
`
|
||||
|
||||
path := filepath.Join(t.TempDir(), "go2rtc.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(base), 0o644))
|
||||
|
||||
out, err := mergeYAML(path, []byte(patch))
|
||||
require.NoError(t, err)
|
||||
|
||||
merged := string(out)
|
||||
require.Contains(t, merged, "# global config comment")
|
||||
require.Contains(t, merged, "# api section comment")
|
||||
require.Contains(t, merged, "# inline username comment")
|
||||
require.Contains(t, merged, "# stream comment")
|
||||
require.Contains(t, merged, "#http://1.1.1.1")
|
||||
require.Contains(t, merged, "password: secret value")
|
||||
require.Contains(t, merged, "format: |")
|
||||
require.NotContains(t, merged, "- null")
|
||||
|
||||
assertOrder(t, merged, "api:", "streams:", "log:", "ffmpeg:")
|
||||
|
||||
var cfg map[string]any
|
||||
require.NoError(t, yaml.Unmarshal(out, &cfg))
|
||||
require.Equal(t, "admin", cfg["api"].(map[string]any)["username"])
|
||||
require.Equal(t, "secret value", cfg["api"].(map[string]any)["password"])
|
||||
require.Equal(t, "/usr/bin/ffmpeg", cfg["ffmpeg"].(map[string]any)["bin"])
|
||||
}
|
||||
|
||||
func TestMergeYAMLPreserveQuotedValuesAndNestedStructure(t *testing.T) {
|
||||
base := `api:
|
||||
username: "admin user"
|
||||
listen: ":1984"
|
||||
webrtc:
|
||||
candidates:
|
||||
- "stun:stun.l.google.com:19302"
|
||||
streams:
|
||||
porch:
|
||||
- "rtsp://cam.local/stream?token=a:b"
|
||||
- #disabled source
|
||||
`
|
||||
patch := `webrtc:
|
||||
ice_servers:
|
||||
- urls:
|
||||
- stun:stun.cloudflare.com:3478
|
||||
`
|
||||
|
||||
path := filepath.Join(t.TempDir(), "go2rtc.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(base), 0o644))
|
||||
|
||||
out, err := mergeYAML(path, []byte(patch))
|
||||
require.NoError(t, err)
|
||||
|
||||
merged := string(out)
|
||||
require.Contains(t, merged, `username: "admin user"`)
|
||||
require.Contains(t, merged, `listen: ":1984"`)
|
||||
require.Contains(t, merged, `"rtsp://cam.local/stream?token=a:b"`)
|
||||
require.Contains(t, merged, "#disabled source")
|
||||
require.Contains(t, merged, "ice_servers:")
|
||||
require.NotContains(t, merged, "- null")
|
||||
|
||||
var cfg map[string]any
|
||||
require.NoError(t, yaml.Unmarshal(out, &cfg))
|
||||
require.Equal(t, "admin user", cfg["api"].(map[string]any)["username"])
|
||||
require.Equal(t, ":1984", cfg["api"].(map[string]any)["listen"])
|
||||
require.NotNil(t, cfg["streams"])
|
||||
require.NotNil(t, cfg["webrtc"].(map[string]any)["candidates"])
|
||||
require.NotNil(t, cfg["webrtc"].(map[string]any)["ice_servers"])
|
||||
}
|
||||
|
||||
func TestMergeYAMLPatchLogKeepsCommentedYardEntriesInline(t *testing.T) {
|
||||
base := `api:
|
||||
listen: :1984
|
||||
read_only: false
|
||||
static_dir: www
|
||||
log:
|
||||
level: trace
|
||||
mcp:
|
||||
enabled: true
|
||||
http: true
|
||||
sse: true
|
||||
streams:
|
||||
cam_main:
|
||||
- https://example.local/stream.m3u8
|
||||
yard:
|
||||
- #http://camera.local/disabled-source-a
|
||||
- #ffmpeg:http://camera.local/disabled-source-b#video=h264
|
||||
- ffmpeg:yard#video=mjpeg
|
||||
- #homekit://camera.local/disabled-source-c
|
||||
- homekit://camera.local/enabled-source
|
||||
`
|
||||
patch := `log:
|
||||
api: debug
|
||||
`
|
||||
|
||||
path := filepath.Join(t.TempDir(), "go2rtc.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(base), 0o644))
|
||||
|
||||
out, err := mergeYAML(path, []byte(patch))
|
||||
require.NoError(t, err)
|
||||
|
||||
merged := string(out)
|
||||
require.Contains(t, merged, " level: trace")
|
||||
require.Contains(t, merged, " api: debug")
|
||||
require.Contains(t, merged, " - #http://camera.local/disabled-source-a")
|
||||
require.Contains(t, merged, " - #ffmpeg:http://camera.local/disabled-source-b#video=h264")
|
||||
require.Contains(t, merged, " - #homekit://camera.local/disabled-source-c")
|
||||
require.NotContains(t, merged, "\n -\n")
|
||||
|
||||
var cfg map[string]any
|
||||
require.NoError(t, yaml.Unmarshal(out, &cfg))
|
||||
require.Equal(t, "debug", cfg["log"].(map[string]any)["api"])
|
||||
}
|
||||
|
||||
func TestMergeYAMLPatchLogWithTrailingSpaces(t *testing.T) {
|
||||
// trailing spaces on "- #comment" lines could confuse node parsing
|
||||
base := "api:\n listen: :1984\nlog:\n level: trace\nstreams:\n yard:\n" +
|
||||
" - #http://192.168.88.100/long/path/to/resource \n" +
|
||||
" - #ffmpeg:http://192.168.88.100/path#video=h264 \n" +
|
||||
" - ffmpeg:yard#video=mjpeg\n"
|
||||
|
||||
patch := "log:\n api: debug\n"
|
||||
|
||||
path := filepath.Join(t.TempDir(), "go2rtc.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(base), 0o644))
|
||||
|
||||
out, err := mergeYAML(path, []byte(patch))
|
||||
require.NoError(t, err)
|
||||
|
||||
merged := string(out)
|
||||
require.Contains(t, merged, " - #http://192.168.88.100/long/path/to/resource")
|
||||
require.Contains(t, merged, " - #ffmpeg:http://192.168.88.100/path#video=h264")
|
||||
require.NotContains(t, merged, "\n -\n")
|
||||
}
|
||||
|
||||
func TestMergeYAMLPatchLogPreservesLongCommentedURLs(t *testing.T) {
|
||||
base := "api:\n" +
|
||||
" listen: :1984\n" +
|
||||
" read_only: false\n" +
|
||||
" static_dir: www\n" +
|
||||
"log:\n" +
|
||||
" level: trace\n" +
|
||||
"mcp:\n" +
|
||||
" enabled: true\n" +
|
||||
" http: true\n" +
|
||||
" sse: true\n" +
|
||||
"streams:\n" +
|
||||
" sf_i280_us101:\n" +
|
||||
" - https://wzmedia.dot.ca.gov/D4/N280_at_JCT_101.stream/playlist.m3u8\n" +
|
||||
" testsrc_h264:\n" +
|
||||
" - exec:ffmpeg -hide_banner -re -f lavfi -i testsrc=size=320x240:rate=15 -c:v libx264 -preset ultrafast -tune zerolatency -profile:v baseline -pix_fmt yuv420p -crf 28 -f h264 -\n" +
|
||||
" yard:\n" +
|
||||
" - #http://192.168.88.100/c17d5873fa8f1ca5e0f94daa46e29343/live/files/high/index.m3u8\n" +
|
||||
" - #ffmpeg:http://192.168.88.100/c17d5873fa8f1ca5e0f94daa46e29343/live/files/high/index.m3u8#audio=opus/16000#video=h264\n" +
|
||||
" - ffmpeg:yard#video=mjpeg\n" +
|
||||
" - #homekit://192.168.88.100:5001?client_id=00000000-0000-0000-0000-000000000001&client_private=0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001&device_id=00:00:00:00:00:01&device_public=0000000000000000000000000000000000000000000000000000000000000001\n" +
|
||||
" - homekit://192.168.88.100:5001?client_id=00000000-0000-0000-0000-000000000002&client_private=0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002&device_id=00:00:00:00:00:02&device_public=0000000000000000000000000000000000000000000000000000000000000002\n"
|
||||
|
||||
patch := "log:\n api: debug\n"
|
||||
|
||||
path := filepath.Join(t.TempDir(), "go2rtc.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(base), 0o644))
|
||||
|
||||
out, err := mergeYAML(path, []byte(patch))
|
||||
require.NoError(t, err)
|
||||
|
||||
merged := string(out)
|
||||
|
||||
// patch applied
|
||||
require.Contains(t, merged, " api: debug")
|
||||
require.Contains(t, merged, " level: trace")
|
||||
|
||||
// commented entries must stay on same line as dash
|
||||
require.Contains(t, merged, " - #http://192.168.88.100/")
|
||||
require.Contains(t, merged, " - #ffmpeg:http://192.168.88.100/")
|
||||
require.Contains(t, merged, " - #homekit://192.168.88.100:")
|
||||
require.NotContains(t, merged, "\n -\n")
|
||||
|
||||
// non-commented entries preserved
|
||||
require.Contains(t, merged, " - ffmpeg:yard#video=mjpeg")
|
||||
require.Contains(t, merged, " - homekit://192.168.88.100:5001?client_id=00000000-0000-0000-0000-000000000002")
|
||||
|
||||
var cfg map[string]any
|
||||
require.NoError(t, yaml.Unmarshal(out, &cfg))
|
||||
require.Equal(t, "debug", cfg["log"].(map[string]any)["api"])
|
||||
}
|
||||
|
||||
func assertOrder(t *testing.T, s string, items ...string) {
|
||||
t.Helper()
|
||||
|
||||
last := -1
|
||||
for _, item := range items {
|
||||
idx := strings.Index(s, item)
|
||||
require.NotEqualf(t, -1, idx, "expected %q in output", item)
|
||||
require.Greaterf(t, idx, last, "expected %q after previous sections", item)
|
||||
last = idx
|
||||
}
|
||||
}
|
||||
+28
-18
@@ -59,38 +59,41 @@ func patch(in []byte, path []string, value any) ([]byte, error) {
|
||||
|
||||
n := len(path) - 1
|
||||
|
||||
// parent node key/value
|
||||
pKey, pVal := findNode(nodes, path[:n])
|
||||
if pKey == nil {
|
||||
// no parent node
|
||||
return addToEnd(in, path, value)
|
||||
}
|
||||
|
||||
var paste []byte
|
||||
|
||||
if value != nil {
|
||||
// nil value means delete key
|
||||
var err error
|
||||
v := map[string]any{path[n]: value}
|
||||
if paste, err = Encode(v, 2); err != nil {
|
||||
if paste, err = Encode(map[string]any{path[n]: value}, 2); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// top-level key
|
||||
if n == 0 {
|
||||
for i := 0; i < len(nodes); i += 2 {
|
||||
if nodes[i].Value == path[0] {
|
||||
i0, i1 := nodeBounds(in, nodes[i])
|
||||
return join(in[:i0], paste, in[i1:]), nil
|
||||
}
|
||||
}
|
||||
return join(in, paste), nil
|
||||
}
|
||||
|
||||
// nested key
|
||||
pKey, pVal := findNode(nodes, path[:n])
|
||||
if pKey == nil {
|
||||
return addToEnd(in, path, value)
|
||||
}
|
||||
|
||||
iKey, _ := findNode(pVal.Content, path[n:])
|
||||
if iKey != nil {
|
||||
// key item not nil (replace value)
|
||||
paste = addIndent(paste, iKey.Column-1)
|
||||
|
||||
i0, i1 := nodeBounds(in, iKey)
|
||||
return join(in[:i0], paste, in[i1:]), nil
|
||||
}
|
||||
|
||||
if pVal.Content != nil {
|
||||
// parent value not nil (use first child indent)
|
||||
paste = addIndent(paste, pVal.Column-1)
|
||||
} else {
|
||||
// parent value is nil (use parent indent + 2)
|
||||
paste = addIndent(paste, pKey.Column+1)
|
||||
}
|
||||
|
||||
@@ -138,13 +141,20 @@ func nodeBounds(in []byte, node *yaml.Node) (offset0, offset1 int) {
|
||||
}
|
||||
|
||||
func addToEnd(in []byte, path []string, value any) ([]byte, error) {
|
||||
if len(path) != 2 || value == nil {
|
||||
if value == nil {
|
||||
return nil, errors.New("yaml: path not exist")
|
||||
}
|
||||
|
||||
v := map[string]map[string]any{
|
||||
path[0]: {path[1]: value},
|
||||
var v any
|
||||
switch len(path) {
|
||||
case 1:
|
||||
v = map[string]any{path[0]: value}
|
||||
case 2:
|
||||
v = map[string]map[string]any{path[0]: {path[1]: value}}
|
||||
default:
|
||||
return nil, errors.New("yaml: path not exist")
|
||||
}
|
||||
|
||||
paste, err := Encode(v, 2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -98,6 +98,20 @@ func TestPatch(t *testing.T) {
|
||||
value: []string{"val1"},
|
||||
expect: "streams:\n camera1: url1\nhomekit:\n camera1:\n name: dummy\n pairings:\n - val1\n",
|
||||
},
|
||||
{
|
||||
name: "top-level replace",
|
||||
src: "api:\n listen: :1984\nlog:\n level: trace\n",
|
||||
path: []string{"log"},
|
||||
value: map[string]any{"level": "debug"},
|
||||
expect: "api:\n listen: :1984\nlog:\n level: debug\n",
|
||||
},
|
||||
{
|
||||
name: "top-level add",
|
||||
src: "api:\n listen: :1984\n",
|
||||
path: []string{"ffmpeg"},
|
||||
value: map[string]any{"bin": "/usr/bin/ffmpeg"},
|
||||
expect: "api:\n listen: :1984\nffmpeg:\n bin: /usr/bin/ffmpeg\n",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user