From bf45f64a7e3f9eaab0c977e2555c4541da240f72 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 26 May 2025 21:56:45 +0200 Subject: [PATCH] - refactor secrets - add support for env in config - redact sensitive information in logs/responses --- internal/api/api.go | 20 ++++- internal/app/secrets.go | 162 +++++++++-------------------------- internal/echo/echo.go | 2 +- internal/expr/expr.go | 3 +- internal/hass/hass.go | 3 +- internal/hls/hls.go | 7 +- internal/onvif/onvif.go | 5 +- internal/streams/dot.go | 4 +- internal/streams/handlers.go | 13 --- internal/streams/producer.go | 11 +-- internal/streams/streams.go | 3 +- pkg/core/helpers.go | 6 ++ pkg/core/listener.go | 6 -- pkg/shell/shell.go | 117 +++++++++++++++++++++++++ 14 files changed, 202 insertions(+), 160 deletions(-) diff --git a/internal/api/api.go b/internal/api/api.go index 419e2bdf..241bedf3 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -1,6 +1,7 @@ package api import ( + "bytes" "crypto/tls" "encoding/json" "fmt" @@ -14,6 +15,7 @@ import ( "time" "github.com/AlexxIT/go2rtc/internal/app" + "github.com/AlexxIT/go2rtc/pkg/shell" "github.com/rs/zerolog" ) @@ -164,10 +166,20 @@ func ResponseJSON(w http.ResponseWriter, v any) { } func ResponsePrettyJSON(w http.ResponseWriter, v any) { - w.Header().Set("Content-Type", MimeJSON) - enc := json.NewEncoder(w) + w.Header().Set("Content-Type", "application/json") + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) enc.SetIndent("", " ") - _ = enc.Encode(v) + err := enc.Encode(v) + + if err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + redactedJSON := shell.Redact(buf.String()) + w.Write([]byte(redactedJSON)) } func Response(w http.ResponseWriter, body any, contentType string) { @@ -190,7 +202,7 @@ var log zerolog.Logger func middlewareLog(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - log.Trace().Msgf("[api] %s %s %s", r.Method, r.URL, r.RemoteAddr) + log.Trace().Msgf("[api] %s %s %s", r.Method, shell.Redact(r.URL.String()), r.RemoteAddr) next.ServeHTTP(w, r) }) } diff --git a/internal/app/secrets.go b/internal/app/secrets.go index c3c4f95e..4735c27c 100644 --- a/internal/app/secrets.go +++ b/internal/app/secrets.go @@ -1,23 +1,19 @@ package app import ( - "fmt" - "regexp" - "strings" "sync" "github.com/AlexxIT/go2rtc/pkg/yaml" ) -var secrets = make(map[string]*Secret) -var secretsMu sync.Mutex - -var templateRegex = regexp.MustCompile(`\{\{\s*([^\}]+)\s*\}\}`) +var ( + secrets = make(map[string]*Secret) + secretsMu sync.Mutex +) type Secrets interface { Get(key string) any Set(key string, value any) - Parse(template string) string Marshal(v any) ([]byte, error) Unmarshal(v any) error Save() error @@ -27,37 +23,36 @@ type Secret struct { Secrets Name string - Values map[string]any + Values map[string]string } -func NewSecret(name string, values interface{}) *Secret { +func NewSecret(name string, values interface{}) (*Secret, error) { secretsMu.Lock() defer secretsMu.Unlock() if s, exists := secrets[name]; exists { - return s + return s, nil } - s := &Secret{Name: name, Values: make(map[string]any)} + s := &Secret{Name: name, Values: make(map[string]string)} - switch v := values.(type) { - case map[string]any: - s.Values = v - default: - data, err := yaml.Encode(values, 2) - if err == nil { - var mapValues map[string]any - if err := yaml.Unmarshal(data, &mapValues); err == nil { - s.Values = mapValues - } - } + data, err := yaml.Encode(values, 2) + if err != nil { + return nil, err + } + + if err := yaml.Unmarshal(data, &s.Values); err != nil { + return nil, err } secrets[name] = s - return s + + return s, nil } func GetSecret(name string) *Secret { + secretsMu.Lock() + defer secretsMu.Unlock() return secrets[name] } @@ -65,79 +60,50 @@ func (s *Secret) Get(key string) any { secretsMu.Lock() defer secretsMu.Unlock() + if s.Values == nil { + return nil + } + return s.Values[key] } -func (s *Secret) Set(key string, value any) { +func (s *Secret) Set(key string, value string) { secretsMu.Lock() defer secretsMu.Unlock() if s.Values == nil { - s.Values = make(map[string]any) + s.Values = make(map[string]string) } s.Values[key] = value - secrets[s.Name] = s } -func (s *Secret) Parse(template string) string { - if !templateRegex.MatchString(template) { - return template - } - - secretsMu.Lock() - defer secretsMu.Unlock() - - if _, exists := secrets[s.Name]; !exists { - return template - } - - result := templateRegex.ReplaceAllStringFunc(template, func(match string) string { - varName := strings.TrimSpace(templateRegex.FindStringSubmatch(match)[1]) - pathParts := strings.Split(varName, ".") - value := getNestedValue(s.Values, pathParts) - - if value != nil { - return stringify(value) - } - - return "" - }) - - return result -} - -func (s *Secret) Marshal(v any) ([]byte, error) { +func (s *Secret) Marshal() (interface{}, error) { secretsMu.Lock() defer secretsMu.Unlock() if s.Values == nil { - return nil, fmt.Errorf("no values in secret %s", s.Name) + return make(map[string]any), nil } - data, err := yaml.Encode(s.Values, 2) - if err != nil { - return nil, fmt.Errorf("error encoding secret values: %w", err) - } - - return data, nil + return s.Values, nil } -func (s *Secret) Unmarshal(v any) error { +func (s *Secret) Unmarshal(value any) error { secretsMu.Lock() defer secretsMu.Unlock() if s.Values == nil { - return fmt.Errorf("no values in secret %s", s.Name) + s.Values = make(map[string]string) } - data, err := yaml.Encode(s.Values, 2) + data, err := yaml.Encode(value, 2) if err != nil { - return fmt.Errorf("error encoding secret values: %w", err) + return err } - if err := yaml.Unmarshal(data, v); err != nil { - return fmt.Errorf("error unmarshaling secret values: %w", err) + if err := yaml.Unmarshal(data, value); err != nil { + return err } return nil @@ -151,17 +117,9 @@ func (s *Secret) Save() error { func initSecrets() { var cfg struct { - Secrets map[string]map[string]any `yaml:"secrets"` + Secrets map[string]map[string]string `yaml:"secrets"` } - /* - Example config: - secrets: - test_camera: - username: test - password: test - */ - LoadConfig(&cfg) if cfg.Secrets == nil { @@ -172,53 +130,13 @@ func initSecrets() { defer secretsMu.Unlock() for name, values := range cfg.Secrets { - secrets[name] = &Secret{Name: name, Values: values} - } -} - -func saveSecret(name string, secret map[string]any) error { - return PatchConfig([]string{"secrets", name}, secret) -} - -func getNestedValue(m map[string]any, path []string) interface{} { - if len(path) == 0 || m == nil { - return nil - } - - key := path[0] - value, exists := m[key] - if !exists { - return nil - } - - if len(path) == 1 { - return value - } - - // Check nested maps - switch nextMap := value.(type) { - case map[string]any: - return getNestedValue(nextMap, path[1:]) - case map[interface{}]interface{}: - stringMap := make(map[string]any) - for k, v := range nextMap { - if keyStr, ok := k.(string); ok { - stringMap[keyStr] = v - } + secrets[name] = &Secret{ + Name: name, + Values: values, } - return getNestedValue(stringMap, path[1:]) - default: - return nil } } -func stringify(value interface{}) string { - switch v := value.(type) { - case string: - return v - case int, int64, float64, bool: - return fmt.Sprintf("%v", v) - default: - return "" - } +func saveSecret(name string, secretValues map[string]string) error { + return PatchConfig([]string{"secrets", name}, secretValues) } diff --git a/internal/echo/echo.go b/internal/echo/echo.go index fb105cec..df88dd64 100644 --- a/internal/echo/echo.go +++ b/internal/echo/echo.go @@ -22,7 +22,7 @@ func Init() { b = bytes.TrimSpace(b) - log.Debug().Str("url", url).Msgf("[echo] %s", b) + log.Debug().Str("url", shell.Redact(url)).Msgf("[echo] %s", b) return string(b), nil }) diff --git a/internal/expr/expr.go b/internal/expr/expr.go index 8fd6c9c2..4d8aa5ce 100644 --- a/internal/expr/expr.go +++ b/internal/expr/expr.go @@ -6,6 +6,7 @@ import ( "github.com/AlexxIT/go2rtc/internal/app" "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/pkg/expr" + "github.com/AlexxIT/go2rtc/pkg/shell" ) func Init() { @@ -17,7 +18,7 @@ func Init() { return "", err } - log.Debug().Msgf("[expr] url=%s", url) + log.Debug().Msgf("[expr] url=%s", shell.Redact(url)) if url = v.(string); url == "" { return "", errors.New("expr: result is empty") diff --git a/internal/hass/hass.go b/internal/hass/hass.go index ea172b02..e2132ad6 100644 --- a/internal/hass/hass.go +++ b/internal/hass/hass.go @@ -15,6 +15,7 @@ import ( "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/hass" + "github.com/AlexxIT/go2rtc/pkg/shell" "github.com/rs/zerolog" ) @@ -177,7 +178,7 @@ func importConfig(config string) error { continue } - log.Debug().Str("url", "hass:"+entrie.Title).Msg("[hass] load config") + log.Debug().Str("url", "hass:"+shell.Redact(entrie.Title)).Msg("[hass] load config") //streams.Get("hass:" + entrie.Title) } diff --git a/internal/hls/hls.go b/internal/hls/hls.go index 5c136450..2344b62e 100644 --- a/internal/hls/hls.go +++ b/internal/hls/hls.go @@ -12,6 +12,7 @@ import ( "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/mp4" "github.com/AlexxIT/go2rtc/pkg/mpegts" + "github.com/AlexxIT/go2rtc/pkg/shell" "github.com/rs/zerolog" ) @@ -142,7 +143,7 @@ func handlerSegmentTS(w http.ResponseWriter, r *http.Request) { data := session.Segment() if data == nil { - log.Warn().Msgf("[hls] can't get segment %s", r.URL.RawQuery) + log.Warn().Msgf("[hls] can't get segment %s", shell.Redact(r.URL.RawQuery)) http.NotFound(w, r) return } @@ -172,7 +173,7 @@ func handlerInit(w http.ResponseWriter, r *http.Request) { data := session.Init() if data == nil { - log.Warn().Msgf("[hls] can't get init %s", r.URL.RawQuery) + log.Warn().Msgf("[hls] can't get init %s", shell.Redact(r.URL.RawQuery)) http.NotFound(w, r) return } @@ -206,7 +207,7 @@ func handlerSegmentMP4(w http.ResponseWriter, r *http.Request) { data := session.Segment() if data == nil { - log.Warn().Msgf("[hls] can't get segment %s", r.URL.RawQuery) + log.Warn().Msgf("[hls] can't get segment %s", shell.Redact(r.URL.RawQuery)) http.NotFound(w, r) return } diff --git a/internal/onvif/onvif.go b/internal/onvif/onvif.go index 6dfa633a..36b3843c 100644 --- a/internal/onvif/onvif.go +++ b/internal/onvif/onvif.go @@ -15,6 +15,7 @@ import ( "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/onvif" + "github.com/AlexxIT/go2rtc/pkg/shell" "github.com/rs/zerolog" ) @@ -165,12 +166,12 @@ func apiOnvif(w http.ResponseWriter, r *http.Request) { for _, rawURL := range urls { u, err := url.Parse(rawURL) if err != nil { - log.Warn().Str("url", rawURL).Msg("[onvif] broken") + log.Warn().Str("url", shell.Redact(rawURL)).Msg("[onvif] broken") continue } if u.Scheme != "http" { - log.Warn().Str("url", rawURL).Msg("[onvif] unsupported") + log.Warn().Str("url", shell.Redact(rawURL)).Msg("[onvif] unsupported") continue } diff --git a/internal/streams/dot.go b/internal/streams/dot.go index e0417972..2c357f77 100644 --- a/internal/streams/dot.go +++ b/internal/streams/dot.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "strings" + + "github.com/AlexxIT/go2rtc/pkg/shell" ) func AppendDOT(dot []byte, stream *Stream) []byte { @@ -166,7 +168,7 @@ func (c *conn) label() string { sb.WriteString("\nsource=" + c.Source) } if c.URL != "" { - sb.WriteString("\nurl=" + c.URL) + sb.WriteString("\nurl=" + shell.Redact(c.URL)) } if c.UserAgent != "" { sb.WriteString("\nuser_agent=" + c.UserAgent) diff --git a/internal/streams/handlers.go b/internal/streams/handlers.go index 991b69c9..3240abb5 100644 --- a/internal/streams/handlers.go +++ b/internal/streams/handlers.go @@ -4,7 +4,6 @@ import ( "errors" "strings" - "github.com/AlexxIT/go2rtc/internal/app" "github.com/AlexxIT/go2rtc/pkg/core" ) @@ -47,18 +46,6 @@ func GetProducer(url string) (core.Producer, error) { } if handler, ok := handlers[scheme]; ok { - index := strings.IndexByte(url, '#') - if index > 0 { - _, query := url[:index], ParseQuery(url[index+1:]) - secretsName := query.Get("secrets") - if secretsName != "" { - secrets := app.GetSecret(secretsName) - if secrets != nil { - url = secrets.Parse(url) - } - } - } - return handler(url) } } diff --git a/internal/streams/producer.go b/internal/streams/producer.go index 09e2dcc5..4260198e 100644 --- a/internal/streams/producer.go +++ b/internal/streams/producer.go @@ -8,6 +8,7 @@ import ( "time" "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/shell" ) type state byte @@ -149,7 +150,7 @@ func (p *Producer) start() { return } - log.Debug().Msgf("[streams] start producer url=%s", p.url) + log.Debug().Msgf("[streams] start producer url=%s", shell.Redact(p.url)) p.state = stateStart p.workerID++ @@ -167,7 +168,7 @@ func (p *Producer) worker(conn core.Producer, workerID int) { return } - log.Warn().Err(err).Str("url", p.url).Caller().Send() + log.Warn().Err(err).Str("url", shell.Redact(p.url)).Caller().Send() } p.reconnect(workerID, 0) @@ -178,11 +179,11 @@ func (p *Producer) reconnect(workerID, retry int) { defer p.mu.Unlock() if p.workerID != workerID { - log.Trace().Msgf("[streams] stop reconnect url=%s", p.url) + log.Trace().Msgf("[streams] stop reconnect url=%s", shell.Redact(p.url)) return } - log.Debug().Msgf("[streams] retry=%d to url=%s", retry, p.url) + log.Debug().Msgf("[streams] retry=%d to url=%s", retry, shell.Redact(p.url)) conn, err := GetProducer(p.url) if err != nil { @@ -257,7 +258,7 @@ func (p *Producer) stop() { p.workerID++ } - log.Debug().Msgf("[streams] stop producer url=%s", p.url) + log.Debug().Msgf("[streams] stop producer url=%s", shell.Redact(p.url)) if p.conn != nil { _ = p.conn.Stop() diff --git a/internal/streams/streams.go b/internal/streams/streams.go index dcbaba28..8731ae68 100644 --- a/internal/streams/streams.go +++ b/internal/streams/streams.go @@ -9,6 +9,7 @@ import ( "github.com/AlexxIT/go2rtc/internal/api" "github.com/AlexxIT/go2rtc/internal/app" + "github.com/AlexxIT/go2rtc/pkg/shell" "github.com/rs/zerolog" ) @@ -127,7 +128,7 @@ func GetOrPatch(query url.Values) *Stream { // check if name param provided if name := query.Get("name"); name != "" { - log.Info().Msgf("[streams] create new stream url=%s", source) + log.Info().Msgf("[streams] create new stream url=%s", shell.Redact(source)) return Patch(name, source) } diff --git a/pkg/core/helpers.go b/pkg/core/helpers.go index 72afe897..f39f53b4 100644 --- a/pkg/core/helpers.go +++ b/pkg/core/helpers.go @@ -6,6 +6,8 @@ import ( "strconv" "strings" "time" + + "github.com/AlexxIT/go2rtc/internal/app" ) const ( @@ -77,3 +79,7 @@ func Caller() string { _, file, line, _ := runtime.Caller(1) return file + ":" + strconv.Itoa(line) } + +func NewSecret(name string, defaultValues interface{}) (*app.Secret, error) { + return app.NewSecret(name, defaultValues) +} diff --git a/pkg/core/listener.go b/pkg/core/listener.go index 7a512741..75d9202a 100644 --- a/pkg/core/listener.go +++ b/pkg/core/listener.go @@ -1,7 +1,5 @@ package core -import "github.com/AlexxIT/go2rtc/internal/app" - type EventFunc func(msg any) // Listener base struct for all classes with support feedback @@ -18,7 +16,3 @@ func (l *Listener) Fire(msg any) { f(msg) } } - -func (l *Listener) NewSecret(name string, defaultValues interface{}) *app.Secret { - return app.NewSecret(name, defaultValues) -} diff --git a/pkg/shell/shell.go b/pkg/shell/shell.go index 75df671f..5388503a 100644 --- a/pkg/shell/shell.go +++ b/pkg/shell/shell.go @@ -1,12 +1,22 @@ package shell import ( + "fmt" "os" "os/signal" "path/filepath" "regexp" "strings" + "sync" "syscall" + + "github.com/AlexxIT/go2rtc/pkg/yaml" +) + +var ( + secretReplacer *strings.Replacer + secretValues map[string]bool // Tracker für alle bekannten Secret-Werte + secretMutex sync.RWMutex ) func QuoteSplit(s string) []string { @@ -40,6 +50,15 @@ func QuoteSplit(s string) []string { // ReplaceEnvVars - support format ${CAMERA_PASSWORD} and ${RTSP_USER:admin} func ReplaceEnvVars(text string) string { + var cfg struct { + Env map[string]string `yaml:"env"` + Secrets map[string]map[string]string `yaml:"secrets"` + } + + yaml.Unmarshal([]byte(text), &cfg) + + buildSecretReplacer(cfg) + re := regexp.MustCompile(`\${([^}{]+)}`) return re.ReplaceAllStringFunc(text, func(match string) string { key := match[2 : len(match)-1] @@ -63,6 +82,23 @@ func ReplaceEnvVars(text string) string { return value } + if cfg.Env != nil { + if value, ok := cfg.Env[key]; ok { + return value + } + } + + if cfg.Secrets != nil { + for secretName, secretValues := range cfg.Secrets { + for k, v := range secretValues { + name := fmt.Sprintf("%s_%s", secretName, k) + if key == name { + return v + } + } + } + } + if dok { return def } @@ -76,3 +112,84 @@ func RunUntilSignal() { signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) println("exit with signal:", (<-sigs).String()) } + +func Redact(text string) string { + secretMutex.RLock() + defer secretMutex.RUnlock() + + if secretReplacer == nil { + return text + } + + return secretReplacer.Replace(text) +} + +func buildSecretReplacer(cfg struct { + Env map[string]string `yaml:"env"` + Secrets map[string]map[string]string `yaml:"secrets"` +}) { + secretMutex.Lock() + defer secretMutex.Unlock() + + if secretValues == nil { + secretValues = make(map[string]bool) + } + + var newSecrets []string + + if dir, ok := os.LookupEnv("CREDENTIALS_DIRECTORY"); ok { + entries, err := os.ReadDir(dir) + if err == nil { + for _, entry := range entries { + if !entry.IsDir() { + value, err := os.ReadFile(filepath.Join(dir, entry.Name())) + if err == nil { + cleanValue := strings.TrimSpace(string(value)) + if len(cleanValue) > 0 && !secretValues[cleanValue] { + secretValues[cleanValue] = true + newSecrets = append(newSecrets, cleanValue) + } + } + } + } + } + } + + if cfg.Env != nil { + for _, value := range cfg.Env { + if len(value) > 0 && !secretValues[value] { + secretValues[value] = true + newSecrets = append(newSecrets, value) + } + } + } + + if cfg.Secrets != nil { + for _, secretMap := range cfg.Secrets { + for _, value := range secretMap { + if len(value) > 0 && !secretValues[value] { + secretValues[value] = true + newSecrets = append(newSecrets, value) + } + } + } + } + + if len(newSecrets) > 0 { + rebuildReplacer() + } +} + +func rebuildReplacer() { + var replacements []string + + for secret := range secretValues { + replacements = append(replacements, secret, "*****") + } + + if len(replacements) > 0 { + secretReplacer = strings.NewReplacer(replacements...) + } else { + secretReplacer = nil + } +} \ No newline at end of file