From e74fc6f198bbb1d178b8ee004438c0eecd2e7a62 Mon Sep 17 00:00:00 2001 From: seydx Date: Sat, 10 May 2025 18:34:02 +0200 Subject: [PATCH 01/60] add tuya source --- go.sum | 7 +- internal/tuya/tuya.go | 13 ++ main.go | 2 + pkg/tuya/api.go | 285 ++++++++++++++++++++++++++++++++++++++++++ pkg/tuya/client.go | 246 ++++++++++++++++++++++++++++++++++++ pkg/tuya/mqtt.go | 274 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 822 insertions(+), 5 deletions(-) create mode 100644 internal/tuya/tuya.go create mode 100644 pkg/tuya/api.go create mode 100644 pkg/tuya/client.go create mode 100644 pkg/tuya/mqtt.go diff --git a/go.sum b/go.sum index 7e1b0cee..4dfebcf6 100644 --- a/go.sum +++ b/go.sum @@ -7,9 +7,10 @@ github.com/asticode/go-astits v1.13.0 h1:XOgkaadfZODnyZRR5Y0/DWkA9vrkLLPLeeOvDwf github.com/asticode/go-astits v1.13.0/go.mod h1:QSHmknZ51pf6KJdHKZHJTLlMegIrhega3LPWz3ND/iI= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= +github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= github.com/expr-lang/expr v1.17.2 h1:o0A99O/Px+/DTjEnQiodAgOIK9PPxL8DtXhBRKC+Iso= github.com/expr-lang/expr v1.17.2/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/expr-lang/expr v1.17.5 h1:i1WrMvcdLF249nSNlpQZN1S6NXuW9WaOfF5tPi3aw3k= @@ -86,7 +87,6 @@ github.com/pion/webrtc/v4 v4.1.3 h1:YZ67Boj9X/hk190jJZ8+HFGQ6DqSZ/fYP3sLAZv7c3c= github.com/pion/webrtc/v4 v4.1.3/go.mod h1:rsq+zQ82ryfR9vbb0L1umPJ6Ogq7zm8mcn9fcGnxomM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= @@ -98,8 +98,6 @@ github.com/sigurn/crc16 v0.0.0-20240131213347-83fcde1e29d1 h1:NVK+OqnavpyFmUiKfU github.com/sigurn/crc16 v0.0.0-20240131213347-83fcde1e29d1/go.mod h1:9/etS5gpQq9BJsJMWg1wpLbfuSnkm8dPF6FdW2JXVhA= github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f h1:1R9KdKjCNSd7F8iGTxIpoID9prlYH8nuNYKt0XvweHA= github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f/go.mod h1:vQhwQ4meQEDfahT5kd61wLAF5AAeh5ZPLVI4JJ/tYo8= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 h1:aeN+ghOV0b2VCmKKO3gqnDQ8mLbpABZgRR2FVYx4ouI= @@ -135,6 +133,5 @@ golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/tuya/tuya.go b/internal/tuya/tuya.go new file mode 100644 index 00000000..c3b34e4a --- /dev/null +++ b/internal/tuya/tuya.go @@ -0,0 +1,13 @@ +package tuya + +import ( + "github.com/AlexxIT/go2rtc/internal/streams" + "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/tuya" +) + +func Init() { + streams.HandleFunc("tuya", func(source string) (core.Producer, error) { + return tuya.Dial(source) + }) +} diff --git a/main.go b/main.go index e85c5900..3bbf632f 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ import ( "github.com/AlexxIT/go2rtc/internal/srtp" "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/internal/tapo" + "github.com/AlexxIT/go2rtc/internal/tuya" "github.com/AlexxIT/go2rtc/internal/v4l2" "github.com/AlexxIT/go2rtc/internal/webrtc" "github.com/AlexxIT/go2rtc/internal/webtorrent" @@ -88,6 +89,7 @@ func main() { roborock.Init() // roborock source homekit.Init() // homekit source ring.Init() // ring source + tuya.Init() // tuya source nest.Init() // nest source bubble.Init() // bubble source expr.Init() // expr source diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go new file mode 100644 index 00000000..4ea609ca --- /dev/null +++ b/pkg/tuya/api.go @@ -0,0 +1,285 @@ +package tuya + +import ( + "bytes" + "crypto/md5" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/webrtc" + "github.com/google/uuid" + pionWebrtc "github.com/pion/webrtc/v4" +) + +type TuyaClient struct { + httpClient *http.Client + mqtt *TuyaMQTT + apiURL string + sessionID string + clientID string + deviceID string + accessToken string + refreshToken string + secret string + expireTime int64 + uid string + motoID string + auth string + iceServers []pionWebrtc.ICEServer +} + +type Token struct { + UID string `json:"uid"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpireTime int64 `json:"expire_time"` +} + +type AudioAttributes struct { + CallMode []int `json:"call_mode"` + HardwareCapability []int `json:"hardware_capability"` +} + +type OpenApiICE struct { + Urls string `json:"urls"` + Username string `json:"username"` + Credential string `json:"credential"` + TTL int `json:"ttl"` +} + +type WebICE struct { + Urls string `json:"urls"` + Username string `json:"username,omitempty"` + Credential string `json:"credential,omitempty"` +} + +type P2PConfig struct { + Ices []OpenApiICE `json:"ices"` +} + +type WebRTConfig struct { + AudioAttributes AudioAttributes `json:"audio_attributes"` + Auth string `json:"auth"` + ID string `json:"id"` + MotoID string `json:"moto_id"` + P2PConfig P2PConfig `json:"p2p_config"` + Skill string `json:"skill"` + SupportsWebRTC bool `json:"supports_webrtc"` + VideoClaritiy int `json:"video_clarity"` +} + +type TokenResponse struct { + Result Token `json:"result"` +} + +type WebRTCConfigResponse struct { + Result WebRTConfig `json:"result"` +} + +type OpenIoTHubConfigRequest struct { + UID string `json:"uid"` + UniqueID string `json:"unique_id"` + LinkType string `json:"link_type"` + Topics string `json:"topics"` +} + +type OpenIoTHubConfigResponse struct { + Success bool `json:"success"` + Result OpenIoTHubConfig `json:"result"` +} + +type OpenIoTHubConfig struct { + Url string `json:"url"` + ClientID string `json:"client_id"` + Username string `json:"username"` + Password string `json:"password"` + + SinkTopic struct { + IPC string `json:"ipc"` + } `json:"sink_topic"` + + SourceSink struct { + IPC string `json:"ipc"` + } `json:"source_topic"` + + ExpireTime int `json:"expire_time"` +} + +const ( + defaultTimeout = 5 * time.Second +) + +func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID string, secret string) (*TuyaClient, error) { + client := &TuyaClient{ + httpClient: &http.Client{Timeout: defaultTimeout}, + mqtt: &TuyaMQTT{waiter: core.Waiter{}}, + apiURL: openAPIURL, + sessionID: core.RandString(6, 62), + clientID: clientID, + deviceID: deviceID, + secret: secret, + uid: uid, + } + + if err := client.InitToken(); err != nil { + return nil, fmt.Errorf("failed to initialize token: %w", err) + } + + if err := client.InitDevice(); err != nil { + return nil, fmt.Errorf("failed to initialize device: %w", err) + } + + if err := client.StartMQTT(); err != nil { + return nil, fmt.Errorf("failed to start MQTT: %w", err) + } + + return client, nil +} + +func (c *TuyaClient) Close() { + c.StopMQTT() + c.httpClient.CloseIdleConnections() +} + +func(c *TuyaClient) Request(method string, url string, body any) ([]byte, error) { + var bodyReader io.Reader + if body != nil { + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request body: %w", err) + } + bodyReader = bytes.NewReader(jsonBody) + } + + req, err := http.NewRequest(method, url, bodyReader) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + ts := time.Now().UnixNano() / 1000000 + sign := c.calBusinessSign(ts) + + req.Header.Set("Accept", "*") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Access-Control-Allow-Origin", "*") + req.Header.Set("Access-Control-Allow-Methods", "*") + req.Header.Set("Access-Control-Allow-Headers", "*") + req.Header.Set("mode", "no-cors") + req.Header.Set("client_id", c.clientID) + req.Header.Set("access_token", c.accessToken) + req.Header.Set("sign", sign) + req.Header.Set("t", strconv.FormatInt(ts, 10)) + + response, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer response.Body.Close() + + res, err := io.ReadAll(response.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("request failed with status code %d: %s", response.StatusCode, string(res)) + } + + return res, nil +} + +func(c *TuyaClient) InitToken() (err error) { + url := fmt.Sprintf("https://%s/v1.0/token?grant_type=1", c.apiURL) + + c.accessToken = "" + c.refreshToken = "" + + body, err := c.Request("GET", url, nil) + if err != nil { + return fmt.Errorf("failed to get token: %w", err) + } + + var tokenResponse TokenResponse + err = json.Unmarshal(body, &tokenResponse) + if err != nil { + return fmt.Errorf("failed to unmarshal token response: %w", err) + } + + c.accessToken = tokenResponse.Result.AccessToken + c.refreshToken = tokenResponse.Result.RefreshToken + c.expireTime = tokenResponse.Result.ExpireTime + + return nil +} + +func(c *TuyaClient) InitDevice() (err error) { + url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.apiURL, c.uid, c.deviceID) + + body, err := c.Request("GET", url, nil) + if err != nil { + return fmt.Errorf("failed to get webrtc-configs: %w", err) + } + + var webRTCConfigResponse WebRTCConfigResponse + err = json.Unmarshal(body, &webRTCConfigResponse) + if err != nil { + return fmt.Errorf("failed to unmarshal webrtc-configs response: %w", err) + } + + c.motoID = webRTCConfigResponse.Result.MotoID + c.auth = webRTCConfigResponse.Result.Auth + + iceServersBytes, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) + if err != nil { + return fmt.Errorf("failed to marshal ICE servers: %w", err) + } + + + c.iceServers, err = webrtc.UnmarshalICEServers([]byte(iceServersBytes)) + if err != nil { + return fmt.Errorf("failed to unmarshal ICE servers: %w", err) + } + + return nil +} + + +func(c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { + url := fmt.Sprintf("https://%s/v2.0/open-iot-hub/access/config", c.apiURL) + + request := &OpenIoTHubConfigRequest{ + UID: c.uid, + UniqueID: uuid.New().String(), + LinkType: "mqtt", + Topics: "ipc", + } + + var openIoTHubConfigResponse OpenIoTHubConfigResponse + body, err := c.Request("POST", url, request) + if err != nil { + return nil, fmt.Errorf("failed to get OpenIoTHub config: %w", err) + } + + err = json.Unmarshal(body, &openIoTHubConfigResponse) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal OpenIoTHub config response: %w", err) + } + + if !openIoTHubConfigResponse.Success { + return nil, fmt.Errorf("failed to get OpenIoTHub config: %s", string(body)) + } + + return &openIoTHubConfigResponse.Result, nil +} + +func(c *TuyaClient) calBusinessSign(ts int64) string { + data := fmt.Sprintf("%s%s%s%d", c.clientID, c.accessToken, c.secret, ts) + val := md5.Sum([]byte(data)) + res := fmt.Sprintf("%X", val) + return res +} \ No newline at end of file diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go new file mode 100644 index 00000000..c36cd502 --- /dev/null +++ b/pkg/tuya/client.go @@ -0,0 +1,246 @@ +package tuya + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "strconv" + + "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/webrtc" + pion "github.com/pion/webrtc/v4" +) + +type Client struct { + api *TuyaClient + prod core.Producer + done chan struct{} +} + +const ( + DefaultCnURL = "openapi.tuyacn.com" + DefaultWestUsURL = "openapi.tuyaus.com" + DefaultEastUsURL = "openapi-ueaz.tuyaus.com" + DefaultCentralEuURL = "openapi.tuyaeu.com" + DefaultWestEuURL = "openapi-weaz.tuyaeu.com" + DefaultInURL = "openapi.tuyain.com" +) + +func Dial(rawURL string) (*Client, error) { + // Parse URL and validate basic params + u, err := url.Parse(rawURL) + if err != nil { + return nil, err + } + + query := u.Query() + deviceID := query.Get("device_id") + uid := query.Get("uid") + clientID := query.Get("client_id") + secret := query.Get("secret") + resolution := query.Get("resolution") + + if deviceID == "" || uid == "" || clientID == "" || secret == "" { + return nil, errors.New("tuya: wrong query") + } + + // Initialize Tuya API client + tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientID, secret) + if err != nil { + return nil, err + } + + client := &Client{ + api: tuyaAPI, + done: make(chan struct{}), + } + + conf := pion.Configuration{ + ICEServers: client.api.iceServers, + // ICETransportPolicy: pion.ICETransportPolicyAll, + // BundlePolicy: pion.BundlePolicyMaxBundle, + } + + api, err := webrtc.NewAPI() + if err != nil { + client.api.Close() + return nil, err + } + + pc, err := api.NewPeerConnection(conf) + if err != nil { + client.api.Close() + return nil, err + } + + // protect from sending ICE candidate before Offer + var sendOffer core.Waiter + + // protect from blocking on errors + defer sendOffer.Done(nil) + + // waiter will wait PC error or WS error or nil (connection OK) + var connState core.Waiter + + prod := webrtc.NewConn(pc) + prod.FormatName = "tuya/webrtc" + prod.Mode = core.ModeActiveProducer + prod.Protocol = "mqtt" + prod.URL = rawURL + + client.prod = prod + + // Set up MQTT handlers + client.api.mqtt.handleAnswer = func(answer AnswerFrame) { + desc := pion.SessionDescription{ + Type: pion.SDPTypePranswer, + SDP: answer.Sdp, + } + + if err = pc.SetRemoteDescription(desc); err != nil { + return + } + + prod.SetAnswer(answer.Sdp) + if err != nil { + client.Stop() + } + + prod.SDP = answer.Sdp + } + + client.api.mqtt.handleCandidate = func(candidate CandidateFrame) { + if candidate.Candidate != "" { + prod.AddCandidate(candidate.Candidate) + if err != nil { + client.Stop() + } + } + } + + client.api.mqtt.handleDisconnect = func() { + client.Stop() + } + + client.api.mqtt.handleError = func(err error) { + fmt.Printf("Tuya error: %s\n", err.Error()) + client.Stop() + } + + prod.Listen(func(msg any) { + switch msg := msg.(type) { + case *pion.ICECandidate: + _ = sendOffer.Wait() + client.api.sendCandidate("a=" + msg.ToJSON().Candidate) + + case pion.PeerConnectionState: + switch msg { + case pion.PeerConnectionStateNew: + break + case pion.PeerConnectionStateConnecting: + break + case pion.PeerConnectionStateConnected: + connState.Done(nil) + default: + connState.Done(errors.New("webrtc: " + msg.String())) + } + } + }) + + medias := []*core.Media{ + { + Kind: core.KindAudio, + Direction: core.DirectionSendRecv, + Codecs: []*core.Codec{ + { + Name: "PCMU", + ClockRate: 8000, + Channels: 1, + }, + }, + }, + { + Kind: core.KindVideo, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: "H264", + ClockRate: 90000, + }, + }, + }, + } + + // Create offer + offer, err := prod.CreateOffer(medias) + if err != nil { + client.api.Close() + return nil, err + } + + // Send offer + client.api.sendOffer(offer) + sendOffer.Done(nil) + + if err = connState.Wait(); err != nil { + return nil, err + } + + if resolution != "" { + value, err := strconv.Atoi(resolution) + if err == nil { + client.api.sendResolution(value) + } + } + + return client, nil +} + +func (c *Client) GetMedias() []*core.Media { + return c.prod.GetMedias() +} + +func (c *Client) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, error) { + return c.prod.GetTrack(media, codec) +} + +func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiver) error { + if webrtcProd, ok := c.prod.(*webrtc.Conn); ok { + return webrtcProd.AddTrack(media, codec, track) + } + + return fmt.Errorf("add track not supported") +} + +func (c *Client) Start() error { + return c.prod.Start() +} + +func (c *Client) Stop() error { + select { + case <-c.done: + return nil + default: + close(c.done) + } + + if c.prod != nil { + _ = c.prod.Stop() + } + + if c.api != nil { + c.api.Close() + c.api = nil + } + + return nil +} + +func (c *Client) MarshalJSON() ([]byte, error) { + if webrtcProd, ok := c.prod.(*webrtc.Conn); ok { + return webrtcProd.MarshalJSON() + } + + return json.Marshal(c.prod) +} diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go new file mode 100644 index 00000000..eff9d8e7 --- /dev/null +++ b/pkg/tuya/mqtt.go @@ -0,0 +1,274 @@ +package tuya + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/AlexxIT/go2rtc/pkg/core" + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +type TuyaMQTT struct { + client mqtt.Client + waiter core.Waiter + publishTopic string + subscribeTopic string + uid string + closed bool + handleAnswer func(answer AnswerFrame) + handleCandidate func(candidate CandidateFrame) + handleDisconnect func() + handleError func(err error) +} + +type MqttFrameHeader struct { + Type string `json:"type"` + From string `json:"from"` + To string `json:"to"` + SubDevID string `json:"sub_dev_id"` + SessionID string `json:"sessionid"` + MotoID string `json:"moto_id"` + TransactionID string `json:"tid"` +} + +type MqttFrame struct { + Header MqttFrameHeader `json:"header"` + Message json.RawMessage `json:"msg"` +} + +type OfferFrame struct { + Mode string `json:"mode"` + Sdp string `json:"sdp"` + StreamType uint32 `json:"stream_type"` + Auth string `json:"auth"` +} + +type AnswerFrame struct { + Mode string `json:"mode"` + Sdp string `json:"sdp"` +} + +type CandidateFrame struct { + Mode string `json:"mode"` + Candidate string `json:"candidate"` +} + +type ResolutionFrame struct { + Mode string `json:"mode"` + Value int `json:"value"` +} + +type DisconnectFrame struct { + Mode string `json:"mode"` +} + +type MqttMessage struct { + Protocol int `json:"protocol"` + Pv string `json:"pv"` + T int64 `json:"t"` + Data MqttFrame `json:"data"` +} + +func(c *TuyaClient) StartMQTT() error { + hubConfig, err := c.LoadHubConfig() + if err != nil { + return fmt.Errorf("failed to load hub config: %w", err) + } + + c.mqtt.publishTopic = hubConfig.SinkTopic.IPC + c.mqtt.subscribeTopic = hubConfig.SourceSink.IPC + + c.mqtt.publishTopic = strings.Replace(c.mqtt.publishTopic, "moto_id", c.motoID, 1) + c.mqtt.publishTopic = strings.Replace(c.mqtt.publishTopic, "{device_id}", c.deviceID, 1) + + parts := strings.Split(c.mqtt.subscribeTopic, "/") + c.mqtt.uid = parts[3] + + opts := mqtt.NewClientOptions().AddBroker(hubConfig.Url). + SetClientID(hubConfig.ClientID). + SetUsername(hubConfig.Username). + SetPassword(hubConfig.Password). + SetOnConnectHandler(c.onConnect). + SetConnectTimeout(10 * time.Second) + + c.mqtt.client = mqtt.NewClient(opts) + + if token := c.mqtt.client.Connect(); token.Wait() && token.Error() != nil { + return fmt.Errorf("failed to connect to MQTT broker: %w", token.Error()) + } + + if err := c.mqtt.waiter.Wait(); err != nil { + return err + } + + return nil +} + +func(c *TuyaClient) StopMQTT() { + c.sendDisconnect() + + if c.mqtt.client != nil { + c.mqtt.client.Disconnect(1000) + } +} + +func(c *TuyaClient) onConnect(client mqtt.Client) { + if token := client.Subscribe(c.mqtt.subscribeTopic, 1, c.consume); token.Wait() && token.Error() != nil { + c.mqtt.waiter.Done(token.Error()) + return + } + + c.mqtt.waiter.Done(nil) +} + +func(c *TuyaClient) consume(client mqtt.Client, msg mqtt.Message) { + var rmqtt MqttMessage + if err := json.Unmarshal(msg.Payload(), &rmqtt); err != nil { + c.mqtt.onError(fmt.Errorf("unmarshal mqtt message fail: %s, payload: %s", err.Error(), string(msg.Payload()))) + return + } + + if rmqtt.Data.Header.SessionID != c.sessionID { + return + } + + switch rmqtt.Data.Header.Type { + case "answer": + c.mqtt.onMqttAnswer(&rmqtt) + case "candidate": + c.mqtt.onMqttCandidate(&rmqtt) + case "disconnect": + c.mqtt.onMqttDisconnect() + } +} + +func(c *TuyaMQTT) onMqttAnswer(msg *MqttMessage) { + var answerFrame AnswerFrame + if err := json.Unmarshal(msg.Data.Message, &answerFrame); err != nil { + c.onError(fmt.Errorf("unmarshal mqtt answer frame fail: %s, session: %s, frame: %s", + err.Error(), + msg.Data.Header.SessionID, + string(msg.Data.Message))) + return + } + + c.onAnswer(answerFrame) +} + +func(c *TuyaMQTT) onMqttCandidate(msg *MqttMessage) { + var candidateFrame CandidateFrame + if err := json.Unmarshal(msg.Data.Message, &candidateFrame); err != nil { + c.onError(fmt.Errorf("unmarshal mqtt candidate frame fail: %s, session: %s, frame: %s", + err.Error(), + msg.Data.Header.SessionID, + string(msg.Data.Message))) + return + } + + // candidate from device start with "a=", end with "\r\n", which are not needed by Chrome webRTC + candidateFrame.Candidate = strings.TrimPrefix(candidateFrame.Candidate, "a=") + candidateFrame.Candidate = strings.TrimSuffix(candidateFrame.Candidate, "\r\n") + + c.onCandidate(candidateFrame) +} + +func(c *TuyaMQTT) onMqttDisconnect() { + c.closed = true + c.onDisconnect() +} + +func(c *TuyaMQTT) onAnswer(answer AnswerFrame) { + if c.handleAnswer != nil { + c.handleAnswer(answer) + } +} + +func(c *TuyaMQTT) onCandidate(candidate CandidateFrame) { + if c.handleCandidate != nil { + c.handleCandidate(candidate) + } +} + +func(c *TuyaMQTT) onDisconnect() { + if c.handleDisconnect != nil { + c.handleDisconnect() + } +} + +func(c *TuyaMQTT) onError(err error) { + if c.handleError != nil { + c.handleError(err) + } +} + +func (c *TuyaClient) sendOffer(sdp string) { + c.sendMqttMessage("offer", 302, "", OfferFrame{ + Mode: "webrtc", + Sdp: sdp, + StreamType: 1, + Auth: c.auth, + }) +} + +func (c *TuyaClient) sendCandidate(candidate string) { + c.sendMqttMessage("candidate", 302, "", CandidateFrame{ + Mode: "webrtc", + Candidate: candidate, + }) +} + +func (c *TuyaClient) sendResolution(resolution int) { + c.sendMqttMessage("resolution", 302, "", ResolutionFrame{ + Mode: "webrtc", + Value: resolution, + }) +} + +func(c *TuyaClient) sendDisconnect() { + c.sendMqttMessage("disconnect", 302, "", DisconnectFrame{ + Mode: "webrtc", + }) +} + +func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transactionID string, data interface{}) { + if c.mqtt.closed { + c.mqtt.onError(fmt.Errorf("mqtt client is closed, send mqtt message fail")) + return + } + + jsonMessage, err := json.Marshal(data) + if err != nil { + c.mqtt.onError(fmt.Errorf("marshal mqtt message fail: %s", err.Error())) + return + } + + msg := &MqttMessage{ + Protocol: protocol, + Pv: "2.2", + T: time.Now().Unix(), + Data: MqttFrame{ + Header: MqttFrameHeader{ + Type: messageType, + From: c.mqtt.uid, + To: c.deviceID, + SessionID: c.sessionID, + MotoID: c.motoID, + TransactionID: transactionID, + }, + Message: jsonMessage, + }, + } + + payload, err := json.Marshal(msg) + if err != nil { + c.mqtt.onError(fmt.Errorf("marshal mqtt message fail: %s", err.Error())) + return + } + + token := c.mqtt.client.Publish(c.mqtt.publishTopic, 1, false, payload) + if token.Wait() && token.Error() != nil { + c.mqtt.onError(fmt.Errorf("mqtt publish fail: %s, topic: %s", token.Error().Error(), c.mqtt.publishTopic)) + } +} From 30d48e139ced9ee0a1373eff26dcc3bacdf3a94c Mon Sep 17 00:00:00 2001 From: seydx Date: Sun, 11 May 2025 00:51:17 +0200 Subject: [PATCH 02/60] implement skill handling and media configuration --- pkg/tuya/api.go | 97 +++++++++++++++++++++++++++++++++++++++++--- pkg/tuya/client.go | 40 ++++-------------- pkg/webrtc/client.go | 4 +- 3 files changed, 101 insertions(+), 40 deletions(-) diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index 4ea609ca..a6290494 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -31,6 +31,7 @@ type TuyaClient struct { motoID string auth string iceServers []pionWebrtc.ICEServer + medias []*core.Media } type Token struct { @@ -41,8 +42,8 @@ type Token struct { } type AudioAttributes struct { - CallMode []int `json:"call_mode"` - HardwareCapability []int `json:"hardware_capability"` + CallMode []int `json:"call_mode"` // 1 = one way, 2 = two way + HardwareCapability []int `json:"hardware_capability"` // 1 = mic, 2 = speaker } type OpenApiICE struct { @@ -62,6 +63,24 @@ type P2PConfig struct { Ices []OpenApiICE `json:"ices"` } +type Skill struct { + WebRTC int `json:"webrtc"` + Audios []struct { + Channels int `json:"channels"` + DataBit int `json:"dataBit"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` + } `json:"audios"` + Videos []struct { + StreamType int `json:"streamType"` // streamType = 2 => H265 and streamType = 4 => H264 + ProfileId string `json:"profileId"` + Width int `json:"width"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` + Height int `json:"height"` + } `json:"videos"` +} + type WebRTConfig struct { AudioAttributes AudioAttributes `json:"audio_attributes"` Auth string `json:"auth"` @@ -73,14 +92,14 @@ type WebRTConfig struct { VideoClaritiy int `json:"video_clarity"` } -type TokenResponse struct { - Result Token `json:"result"` -} - type WebRTCConfigResponse struct { Result WebRTConfig `json:"result"` } +type TokenResponse struct { + Result Token `json:"result"` +} + type OpenIoTHubConfigRequest struct { UID string `json:"uid"` UniqueID string `json:"unique_id"` @@ -234,6 +253,63 @@ func(c *TuyaClient) InitDevice() (err error) { c.motoID = webRTCConfigResponse.Result.MotoID c.auth = webRTCConfigResponse.Result.Auth + var skill Skill + err = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), &skill) + if err != nil { + return fmt.Errorf("failed to unmarshal skill: %w", err) + } + + var audioDirection string + if contains(webRTCConfigResponse.Result.AudioAttributes.CallMode, 2) && contains(webRTCConfigResponse.Result.AudioAttributes.HardwareCapability, 1) { + audioDirection = core.DirectionSendRecv + } else { + audioDirection = core.DirectionRecvonly + } + + c.medias = make([]*core.Media, 0) + for _, audio := range skill.Audios { + media := &core.Media{ + Kind: core.KindAudio, + Direction: audioDirection, + Codecs: []*core.Codec{ + { + Name: "PCMU", + ClockRate: uint32(audio.SampleRate), + Channels: uint8(audio.Channels), + }, + }, + } + + c.medias = append(c.medias, media) + } + + // take only the first video codec + video := skill.Videos[0] + + var name string + switch video.CodecType { + case 4: + name = core.CodecH265 + case 2: + name = core.CodecH264 + default: + name = core.CodecH264 + } + + media := &core.Media{ + Kind: core.KindVideo, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: name, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }, + }, + } + + c.medias = append(c.medias, media) + iceServersBytes, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) if err != nil { return fmt.Errorf("failed to marshal ICE servers: %w", err) @@ -282,4 +358,13 @@ func(c *TuyaClient) calBusinessSign(ts int64) string { val := md5.Sum([]byte(data)) res := fmt.Sprintf("%X", val) return res +} + +func contains(slice []int, val int) bool { + for _, item := range slice { + if item == val { + return true + } + } + return false } \ No newline at end of file diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index c36cd502..e0cbce4c 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -58,8 +58,8 @@ func Dial(rawURL string) (*Client, error) { conf := pion.Configuration{ ICEServers: client.api.iceServers, - // ICETransportPolicy: pion.ICETransportPolicyAll, - // BundlePolicy: pion.BundlePolicyMaxBundle, + ICETransportPolicy: pion.ICETransportPolicyAll, + BundlePolicy: pion.BundlePolicyMaxBundle, } api, err := webrtc.NewAPI() @@ -99,14 +99,15 @@ func Dial(rawURL string) (*Client, error) { } if err = pc.SetRemoteDescription(desc); err != nil { + client.Stop() return } - - prod.SetAnswer(answer.Sdp) - if err != nil { + + if err = prod.SetAnswer(answer.Sdp); err != nil { client.Stop() + return } - + prod.SDP = answer.Sdp } @@ -148,32 +149,8 @@ func Dial(rawURL string) (*Client, error) { } }) - medias := []*core.Media{ - { - Kind: core.KindAudio, - Direction: core.DirectionSendRecv, - Codecs: []*core.Codec{ - { - Name: "PCMU", - ClockRate: 8000, - Channels: 1, - }, - }, - }, - { - Kind: core.KindVideo, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: "H264", - ClockRate: 90000, - }, - }, - }, - } - // Create offer - offer, err := prod.CreateOffer(medias) + offer, err := prod.CreateOffer(client.api.medias) if err != nil { client.api.Close() return nil, err @@ -231,7 +208,6 @@ func (c *Client) Stop() error { if c.api != nil { c.api.Close() - c.api = nil } return nil diff --git a/pkg/webrtc/client.go b/pkg/webrtc/client.go index 84e9e86b..bc2c4f87 100644 --- a/pkg/webrtc/client.go +++ b/pkg/webrtc/client.go @@ -63,12 +63,12 @@ func (c *Conn) SetAnswer(answer string) (err error) { SDP: fakeFormatsInAnswer(c.pc.LocalDescription().SDP, answer), } if err = c.pc.SetRemoteDescription(desc); err != nil { - return + return err } sd := &sdp.SessionDescription{} if err = sd.Unmarshal([]byte(answer)); err != nil { - return + return err } c.Medias = UnmarshalMedias(sd.MediaDescriptions) From 6e35f1a389c9c2f91a1d2a4a7daa9adeee7168d1 Mon Sep 17 00:00:00 2001 From: seydx Date: Sun, 11 May 2025 01:21:20 +0200 Subject: [PATCH 03/60] add rtsp support --- pkg/tuya/api.go | 58 ++++++++++++++++++++++++++++++++++++++++------ pkg/tuya/client.go | 14 +++++++++-- pkg/tuya/mqtt.go | 3 +-- 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index a6290494..e3c9d5b0 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -20,6 +20,7 @@ type TuyaClient struct { httpClient *http.Client mqtt *TuyaMQTT apiURL string + rtspURL string sessionID string clientID string deviceID string @@ -41,6 +42,17 @@ type Token struct { ExpireTime int64 `json:"expire_time"` } +type RTSPRequest struct { + Type string `json:"type"` +} + +type RTSPResponse struct { + Success bool `json:"success"` + Result struct { + URL string `json:"url"` + } `json:"result"` +} + type AudioAttributes struct { CallMode []int `json:"call_mode"` // 1 = one way, 2 = two way HardwareCapability []int `json:"hardware_capability"` // 1 = mic, 2 = speaker @@ -133,7 +145,7 @@ const ( defaultTimeout = 5 * time.Second ) -func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID string, secret string) (*TuyaClient, error) { +func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID string, secret string, useRTSP bool) (*TuyaClient, error) { client := &TuyaClient{ httpClient: &http.Client{Timeout: defaultTimeout}, mqtt: &TuyaMQTT{waiter: core.Waiter{}}, @@ -149,12 +161,18 @@ func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID stri return nil, fmt.Errorf("failed to initialize token: %w", err) } - if err := client.InitDevice(); err != nil { - return nil, fmt.Errorf("failed to initialize device: %w", err) - } + if useRTSP { + if err := client.GetRTSP(); err != nil { + return nil, fmt.Errorf("failed to get RTSP URL: %w", err) + } + } else { + if err := client.InitDevice(); err != nil { + return nil, fmt.Errorf("failed to initialize device: %w", err) + } - if err := client.StartMQTT(); err != nil { - return nil, fmt.Errorf("failed to start MQTT: %w", err) + if err := client.StartMQTT(); err != nil { + return nil, fmt.Errorf("failed to start MQTT: %w", err) + } } return client, nil @@ -324,6 +342,32 @@ func(c *TuyaClient) InitDevice() (err error) { return nil } +func(c *TuyaClient) GetRTSP() (err error) { + url := fmt.Sprintf("https://%s/v1.0/devices/%s/stream/actions/allocate", c.apiURL, c.deviceID) + + request := &RTSPRequest{ + Type: "rtsp", + } + + body, err := c.Request("POST", url, request) + if err != nil { + return fmt.Errorf("failed to get rtsp url: %w", err) + } + + var rtspResponse RTSPResponse + err = json.Unmarshal(body, &rtspResponse) + if err != nil { + return fmt.Errorf("failed to unmarshal rtsp response: %w", err) + } + + if !rtspResponse.Success { + return fmt.Errorf("failed to get rtsp url: %s", string(body)) + } + + c.rtspURL = rtspResponse.Result.URL + + return nil +} func(c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { url := fmt.Sprintf("https://%s/v2.0/open-iot-hub/access/config", c.apiURL) @@ -335,12 +379,12 @@ func(c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { Topics: "ipc", } - var openIoTHubConfigResponse OpenIoTHubConfigResponse body, err := c.Request("POST", url, request) if err != nil { return nil, fmt.Errorf("failed to get OpenIoTHub config: %w", err) } + var openIoTHubConfigResponse OpenIoTHubConfigResponse err = json.Unmarshal(body, &openIoTHubConfigResponse) if err != nil { return nil, fmt.Errorf("failed to unmarshal OpenIoTHub config response: %w", err) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index e0cbce4c..1f1bc827 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -7,6 +7,7 @@ import ( "net/url" "strconv" + "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/webrtc" pion "github.com/pion/webrtc/v4" @@ -27,7 +28,7 @@ const ( DefaultInURL = "openapi.tuyain.com" ) -func Dial(rawURL string) (*Client, error) { +func Dial(rawURL string) (core.Producer, error) { // Parse URL and validate basic params u, err := url.Parse(rawURL) if err != nil { @@ -40,13 +41,14 @@ func Dial(rawURL string) (*Client, error) { clientID := query.Get("client_id") secret := query.Get("secret") resolution := query.Get("resolution") + useRTSP := query.Get("use_rtsp") == "1" if deviceID == "" || uid == "" || clientID == "" || secret == "" { return nil, errors.New("tuya: wrong query") } // Initialize Tuya API client - tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientID, secret) + tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientID, secret, useRTSP) if err != nil { return nil, err } @@ -56,6 +58,14 @@ func Dial(rawURL string) (*Client, error) { done: make(chan struct{}), } + if useRTSP { + if client.api.rtspURL == "" { + return nil, errors.New("tuya: no rtsp url") + } + + return streams.GetProducer(client.api.rtspURL) + } + conf := pion.Configuration{ ICEServers: client.api.iceServers, ICETransportPolicy: pion.ICETransportPolicyAll, diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index eff9d8e7..3da27c08 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -107,9 +107,8 @@ func(c *TuyaClient) StartMQTT() error { } func(c *TuyaClient) StopMQTT() { - c.sendDisconnect() - if c.mqtt.client != nil { + c.sendDisconnect() c.mqtt.client.Disconnect(1000) } } From b797a2fcd18007359917801b27c02f08a61d7fe6 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 12 May 2025 05:23:14 +0200 Subject: [PATCH 04/60] add HLS support and fix skill response --- pkg/tuya/api.go | 130 +++++++++++++++++++++++++++++---------------- pkg/tuya/client.go | 15 +++++- 2 files changed, 97 insertions(+), 48 deletions(-) diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index e3c9d5b0..4ba4f1eb 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -21,6 +21,7 @@ type TuyaClient struct { mqtt *TuyaMQTT apiURL string rtspURL string + hlsURL string sessionID string clientID string deviceID string @@ -42,11 +43,11 @@ type Token struct { ExpireTime int64 `json:"expire_time"` } -type RTSPRequest struct { +type AllocateRequest struct { Type string `json:"type"` } -type RTSPResponse struct { +type AllocateResponse struct { Success bool `json:"success"` Result struct { URL string `json:"url"` @@ -145,7 +146,7 @@ const ( defaultTimeout = 5 * time.Second ) -func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID string, secret string, useRTSP bool) (*TuyaClient, error) { +func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID string, secret string, useRTSP bool, useHLS bool) (*TuyaClient, error) { client := &TuyaClient{ httpClient: &http.Client{Timeout: defaultTimeout}, mqtt: &TuyaMQTT{waiter: core.Waiter{}}, @@ -162,9 +163,13 @@ func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID stri } if useRTSP { - if err := client.GetRTSP(); err != nil { + if err := client.GetStreamUrl("rtsp"); err != nil { return nil, fmt.Errorf("failed to get RTSP URL: %w", err) } + } else if useHLS { + if err := client.GetStreamUrl("hls"); err != nil { + return nil, fmt.Errorf("failed to get HLS URL: %w", err) + } } else { if err := client.InitDevice(); err != nil { return nil, fmt.Errorf("failed to initialize device: %w", err) @@ -285,48 +290,72 @@ func(c *TuyaClient) InitDevice() (err error) { } c.medias = make([]*core.Media, 0) - for _, audio := range skill.Audios { - media := &core.Media{ + if len(skill.Audios) > 0 { + for _, audio := range skill.Audios { + c.medias = append(c.medias, &core.Media{ + Kind: core.KindAudio, + Direction: audioDirection, + Codecs: []*core.Codec{ + { + Name: "PCMU", + ClockRate: uint32(audio.SampleRate), + Channels: uint8(audio.Channels), + }, + }, + }) + } + } else { + c.medias = append(c.medias, &core.Media{ Kind: core.KindAudio, - Direction: audioDirection, + Direction: core.DirectionRecvonly, Codecs: []*core.Codec{ { Name: "PCMU", - ClockRate: uint32(audio.SampleRate), - Channels: uint8(audio.Channels), + ClockRate: uint32(8000), + Channels: uint8(1), }, }, + }) + } + + if len(skill.Videos) > 0 { + // take only the first video codec + video := skill.Videos[0] + + var name string + switch video.CodecType { + case 4: + name = core.CodecH265 + case 2: + name = core.CodecH264 + default: + name = core.CodecH264 } - - c.medias = append(c.medias, media) - } - - // take only the first video codec - video := skill.Videos[0] - - var name string - switch video.CodecType { - case 4: - name = core.CodecH265 - case 2: - name = core.CodecH264 - default: - name = core.CodecH264 - } - - media := &core.Media{ - Kind: core.KindVideo, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: name, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, + + c.medias = append(c.medias, &core.Media{ + Kind: core.KindVideo, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: name, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }, }, - }, + }) + } else { + c.medias = append(c.medias, &core.Media{ + Kind: core.KindVideo, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: core.CodecH264, + ClockRate: uint32(90000), + PayloadType: 96, + }, + }, + }) } - - c.medias = append(c.medias, media) iceServersBytes, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) if err != nil { @@ -342,11 +371,11 @@ func(c *TuyaClient) InitDevice() (err error) { return nil } -func(c *TuyaClient) GetRTSP() (err error) { +func(c *TuyaClient) GetStreamUrl(streamType string) (err error) { url := fmt.Sprintf("https://%s/v1.0/devices/%s/stream/actions/allocate", c.apiURL, c.deviceID) - request := &RTSPRequest{ - Type: "rtsp", + request := &AllocateRequest{ + Type: streamType, } body, err := c.Request("POST", url, request) @@ -354,17 +383,26 @@ func(c *TuyaClient) GetRTSP() (err error) { return fmt.Errorf("failed to get rtsp url: %w", err) } - var rtspResponse RTSPResponse - err = json.Unmarshal(body, &rtspResponse) + var allosResponse AllocateResponse + err = json.Unmarshal(body, &allosResponse) if err != nil { - return fmt.Errorf("failed to unmarshal rtsp response: %w", err) + return fmt.Errorf("failed to unmarshal stream response: %w", err) } - if !rtspResponse.Success { - return fmt.Errorf("failed to get rtsp url: %s", string(body)) + if !allosResponse.Success { + return fmt.Errorf("failed to get stream url: %s", string(body)) } - c.rtspURL = rtspResponse.Result.URL + switch streamType { + case "rtsp": + c.rtspURL = allosResponse.Result.URL + fmt.Printf("RTSP URL: %s\n", c.rtspURL) + case "hls": + c.hlsURL = "ffmpeg:" + allosResponse.Result.URL + "#video=copy" + fmt.Printf("HLS URL: %s\n", c.hlsURL) + default: + return fmt.Errorf("unsupported stream type: %s", streamType) + } return nil } diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 1f1bc827..9019e931 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -41,14 +41,15 @@ func Dial(rawURL string) (core.Producer, error) { clientID := query.Get("client_id") secret := query.Get("secret") resolution := query.Get("resolution") - useRTSP := query.Get("use_rtsp") == "1" + useRTSP := query.Get("rtsp") == "1" + useHLS := query.Get("hls") == "1" if deviceID == "" || uid == "" || clientID == "" || secret == "" { return nil, errors.New("tuya: wrong query") } // Initialize Tuya API client - tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientID, secret, useRTSP) + tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientID, secret, useRTSP, useHLS) if err != nil { return nil, err } @@ -58,6 +59,7 @@ func Dial(rawURL string) (core.Producer, error) { done: make(chan struct{}), } + // RTSP if useRTSP { if client.api.rtspURL == "" { return nil, errors.New("tuya: no rtsp url") @@ -66,6 +68,15 @@ func Dial(rawURL string) (core.Producer, error) { return streams.GetProducer(client.api.rtspURL) } + // HLS + if useHLS { + if client.api.hlsURL == "" { + return nil, errors.New("tuya: no hls url") + } + return streams.GetProducer(client.api.hlsURL) + } + + // Default to WebRTC conf := pion.Configuration{ ICEServers: client.api.iceServers, ICETransportPolicy: pion.ICETransportPolicyAll, From a7e76db464a82bc277ca116b91e3e442cae7580b Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 12 May 2025 13:31:45 +0200 Subject: [PATCH 05/60] change hls url and query and add more checks --- pkg/tuya/api.go | 2 +- pkg/tuya/client.go | 246 ++++++++++++++++++++++++--------------------- 2 files changed, 130 insertions(+), 118 deletions(-) diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index 4ba4f1eb..cef83af1 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -398,7 +398,7 @@ func(c *TuyaClient) GetStreamUrl(streamType string) (err error) { c.rtspURL = allosResponse.Result.URL fmt.Printf("RTSP URL: %s\n", c.rtspURL) case "hls": - c.hlsURL = "ffmpeg:" + allosResponse.Result.URL + "#video=copy" + c.hlsURL = allosResponse.Result.URL fmt.Printf("HLS URL: %s\n", c.hlsURL) default: return fmt.Errorf("unsupported stream type: %s", streamType) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 9019e931..3a11b595 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -41,13 +41,31 @@ func Dial(rawURL string) (core.Producer, error) { clientID := query.Get("client_id") secret := query.Get("secret") resolution := query.Get("resolution") - useRTSP := query.Get("rtsp") == "1" - useHLS := query.Get("hls") == "1" + streamType := query.Get("type") + useRTSP := streamType == "rtsp" + useHLS := streamType == "hls" + useWebRTC := streamType == "webrtc" || streamType == "" + + // check if host is correct + switch u.Hostname() { + case DefaultCnURL: + case DefaultWestUsURL: + case DefaultEastUsURL: + case DefaultCentralEuURL: + case DefaultWestEuURL: + case DefaultInURL: + default: + return nil, fmt.Errorf("tuya: wrong host %s", u.Hostname()) + } if deviceID == "" || uid == "" || clientID == "" || secret == "" { return nil, errors.New("tuya: wrong query") } + if !useRTSP && !useHLS && !useWebRTC { + return nil, errors.New("tuya: wrong stream type") + } + // Initialize Tuya API client tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientID, secret, useRTSP, useHLS) if err != nil { @@ -59,140 +77,134 @@ func Dial(rawURL string) (core.Producer, error) { done: make(chan struct{}), } - // RTSP if useRTSP { if client.api.rtspURL == "" { return nil, errors.New("tuya: no rtsp url") } - return streams.GetProducer(client.api.rtspURL) - } - - // HLS - if useHLS { + } else if useHLS { if client.api.hlsURL == "" { return nil, errors.New("tuya: no hls url") } return streams.GetProducer(client.api.hlsURL) - } - - // Default to WebRTC - conf := pion.Configuration{ - ICEServers: client.api.iceServers, - ICETransportPolicy: pion.ICETransportPolicyAll, - BundlePolicy: pion.BundlePolicyMaxBundle, - } - - api, err := webrtc.NewAPI() - if err != nil { - client.api.Close() - return nil, err - } - - pc, err := api.NewPeerConnection(conf) - if err != nil { - client.api.Close() - return nil, err - } - - // protect from sending ICE candidate before Offer - var sendOffer core.Waiter - - // protect from blocking on errors - defer sendOffer.Done(nil) - - // waiter will wait PC error or WS error or nil (connection OK) - var connState core.Waiter - - prod := webrtc.NewConn(pc) - prod.FormatName = "tuya/webrtc" - prod.Mode = core.ModeActiveProducer - prod.Protocol = "mqtt" - prod.URL = rawURL - - client.prod = prod - - // Set up MQTT handlers - client.api.mqtt.handleAnswer = func(answer AnswerFrame) { - desc := pion.SessionDescription{ - Type: pion.SDPTypePranswer, - SDP: answer.Sdp, + } else { + conf := pion.Configuration{ + ICEServers: client.api.iceServers, + ICETransportPolicy: pion.ICETransportPolicyAll, + BundlePolicy: pion.BundlePolicyMaxBundle, } - if err = pc.SetRemoteDescription(desc); err != nil { - client.Stop() - return + api, err := webrtc.NewAPI() + if err != nil { + client.api.Close() + return nil, err } - - if err = prod.SetAnswer(answer.Sdp); err != nil { + + pc, err := api.NewPeerConnection(conf) + if err != nil { + client.api.Close() + return nil, err + } + + // protect from sending ICE candidate before Offer + var sendOffer core.Waiter + + // protect from blocking on errors + defer sendOffer.Done(nil) + + // waiter will wait PC error or WS error or nil (connection OK) + var connState core.Waiter + + prod := webrtc.NewConn(pc) + prod.FormatName = "tuya/webrtc" + prod.Mode = core.ModeActiveProducer + prod.Protocol = "mqtt" + prod.URL = rawURL + + client.prod = prod + + // Set up MQTT handlers + client.api.mqtt.handleAnswer = func(answer AnswerFrame) { + desc := pion.SessionDescription{ + Type: pion.SDPTypePranswer, + SDP: answer.Sdp, + } + + if err = pc.SetRemoteDescription(desc); err != nil { + client.Stop() + return + } + + if err = prod.SetAnswer(answer.Sdp); err != nil { + client.Stop() + return + } + + prod.SDP = answer.Sdp + } + + client.api.mqtt.handleCandidate = func(candidate CandidateFrame) { + if candidate.Candidate != "" { + prod.AddCandidate(candidate.Candidate) + if err != nil { + client.Stop() + } + } + } + + client.api.mqtt.handleDisconnect = func() { client.Stop() - return } - prod.SDP = answer.Sdp - } + client.api.mqtt.handleError = func(err error) { + fmt.Printf("Tuya error: %s\n", err.Error()) + client.Stop() + } - client.api.mqtt.handleCandidate = func(candidate CandidateFrame) { - if candidate.Candidate != "" { - prod.AddCandidate(candidate.Candidate) - if err != nil { - client.Stop() + prod.Listen(func(msg any) { + switch msg := msg.(type) { + case *pion.ICECandidate: + _ = sendOffer.Wait() + client.api.sendCandidate("a=" + msg.ToJSON().Candidate) + + case pion.PeerConnectionState: + switch msg { + case pion.PeerConnectionStateNew: + break + case pion.PeerConnectionStateConnecting: + break + case pion.PeerConnectionStateConnected: + connState.Done(nil) + default: + connState.Done(errors.New("webrtc: " + msg.String())) + } + } + }) + + // Create offer + offer, err := prod.CreateOffer(client.api.medias) + if err != nil { + client.api.Close() + return nil, err + } + + // Send offer + client.api.sendOffer(offer) + sendOffer.Done(nil) + + if err = connState.Wait(); err != nil { + return nil, err + } + + if resolution != "" { + value, err := strconv.Atoi(resolution) + if err == nil { + client.api.sendResolution(value) } } + + return client, nil } - - client.api.mqtt.handleDisconnect = func() { - client.Stop() - } - - client.api.mqtt.handleError = func(err error) { - fmt.Printf("Tuya error: %s\n", err.Error()) - client.Stop() - } - - prod.Listen(func(msg any) { - switch msg := msg.(type) { - case *pion.ICECandidate: - _ = sendOffer.Wait() - client.api.sendCandidate("a=" + msg.ToJSON().Candidate) - - case pion.PeerConnectionState: - switch msg { - case pion.PeerConnectionStateNew: - break - case pion.PeerConnectionStateConnecting: - break - case pion.PeerConnectionStateConnected: - connState.Done(nil) - default: - connState.Done(errors.New("webrtc: " + msg.String())) - } - } - }) - - // Create offer - offer, err := prod.CreateOffer(client.api.medias) - if err != nil { - client.api.Close() - return nil, err - } - - // Send offer - client.api.sendOffer(offer) - sendOffer.Done(nil) - - if err = connState.Wait(); err != nil { - return nil, err - } - - if resolution != "" { - value, err := strconv.Atoi(resolution) - if err == nil { - client.api.sendResolution(value) - } - } - - return client, nil } func (c *Client) GetMedias() []*core.Media { From 43b7a662c10a1a38dc3c72b8cb127851f5cc5aab Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 12 May 2025 13:39:59 +0200 Subject: [PATCH 06/60] use streamType parameter --- pkg/tuya/api.go | 6 +++--- pkg/tuya/client.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index cef83af1..b66481fc 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -146,7 +146,7 @@ const ( defaultTimeout = 5 * time.Second ) -func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID string, secret string, useRTSP bool, useHLS bool) (*TuyaClient, error) { +func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID string, secret string, streamType string) (*TuyaClient, error) { client := &TuyaClient{ httpClient: &http.Client{Timeout: defaultTimeout}, mqtt: &TuyaMQTT{waiter: core.Waiter{}}, @@ -162,11 +162,11 @@ func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID stri return nil, fmt.Errorf("failed to initialize token: %w", err) } - if useRTSP { + if streamType == "rtsp" { if err := client.GetStreamUrl("rtsp"); err != nil { return nil, fmt.Errorf("failed to get RTSP URL: %w", err) } - } else if useHLS { + } else if streamType == "hls" { if err := client.GetStreamUrl("hls"); err != nil { return nil, fmt.Errorf("failed to get HLS URL: %w", err) } diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 3a11b595..4735c003 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -67,7 +67,7 @@ func Dial(rawURL string) (core.Producer, error) { } // Initialize Tuya API client - tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientID, secret, useRTSP, useHLS) + tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientID, secret, streamType) if err != nil { return nil, err } From 05c12b34e5c0d912eedda7fa1e190679d3e92363 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 12 May 2025 22:54:40 +0200 Subject: [PATCH 07/60] refactor --- pkg/tuya/api.go | 480 ++++++++++++++++++++++++++------------------- pkg/tuya/client.go | 51 ++--- pkg/tuya/mqtt.go | 68 +++---- 3 files changed, 339 insertions(+), 260 deletions(-) diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index b66481fc..4ee0bf8f 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -22,151 +22,165 @@ type TuyaClient struct { apiURL string rtspURL string hlsURL string - sessionID string - clientID string - deviceID string + sessionId string + clientId string + clientSecret string + deviceId string accessToken string refreshToken string - secret string expireTime int64 uid string - motoID string + motoId string auth string + skill *Skill iceServers []pionWebrtc.ICEServer medias []*core.Media + hasBackchannel bool } type Token struct { - UID string `json:"uid"` - AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` - ExpireTime int64 `json:"expire_time"` -} - -type AllocateRequest struct { - Type string `json:"type"` -} - -type AllocateResponse struct { - Success bool `json:"success"` - Result struct { - URL string `json:"url"` - } `json:"result"` + UID string `json:"uid"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpireTime int64 `json:"expire_time"` } type AudioAttributes struct { - CallMode []int `json:"call_mode"` // 1 = one way, 2 = two way - HardwareCapability []int `json:"hardware_capability"` // 1 = mic, 2 = speaker + CallMode []int `json:"call_mode"` // 1 = one way, 2 = two way + HardwareCapability []int `json:"hardware_capability"` // 1 = mic, 2 = speaker } type OpenApiICE struct { - Urls string `json:"urls"` - Username string `json:"username"` - Credential string `json:"credential"` - TTL int `json:"ttl"` + Urls string `json:"urls"` + Username string `json:"username"` + Credential string `json:"credential"` + TTL int `json:"ttl"` } type WebICE struct { - Urls string `json:"urls"` - Username string `json:"username,omitempty"` - Credential string `json:"credential,omitempty"` + Urls string `json:"urls"` + Username string `json:"username,omitempty"` + Credential string `json:"credential,omitempty"` } type P2PConfig struct { - Ices []OpenApiICE `json:"ices"` + Ices []OpenApiICE `json:"ices"` } type Skill struct { - WebRTC int `json:"webrtc"` + WebRTC int `json:"webrtc"` Audios []struct { - Channels int `json:"channels"` - DataBit int `json:"dataBit"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` + Channels int `json:"channels"` + DataBit int `json:"dataBit"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` } `json:"audios"` Videos []struct { - StreamType int `json:"streamType"` // streamType = 2 => H265 and streamType = 4 => H264 - ProfileId string `json:"profileId"` - Width int `json:"width"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` - Height int `json:"height"` + StreamType int `json:"streamType"` // streamType = 2 => main stream - streamType = 4 => sub stream + ProfileId string `json:"profileId"` + Width int `json:"width"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` + Height int `json:"height"` } `json:"videos"` } type WebRTConfig struct { - AudioAttributes AudioAttributes `json:"audio_attributes"` - Auth string `json:"auth"` - ID string `json:"id"` - MotoID string `json:"moto_id"` - P2PConfig P2PConfig `json:"p2p_config"` - Skill string `json:"skill"` - SupportsWebRTC bool `json:"supports_webrtc"` - VideoClaritiy int `json:"video_clarity"` -} - -type WebRTCConfigResponse struct { - Result WebRTConfig `json:"result"` -} - -type TokenResponse struct { - Result Token `json:"result"` -} - -type OpenIoTHubConfigRequest struct { - UID string `json:"uid"` - UniqueID string `json:"unique_id"` - LinkType string `json:"link_type"` - Topics string `json:"topics"` -} - -type OpenIoTHubConfigResponse struct { - Success bool `json:"success"` - Result OpenIoTHubConfig `json:"result"` + AudioAttributes AudioAttributes `json:"audio_attributes"` + Auth string `json:"auth"` + ID string `json:"id"` + MotoID string `json:"moto_id"` + P2PConfig P2PConfig `json:"p2p_config"` + ProtocolVersion string `json:"protocol_version"` + Skill string `json:"skill"` + SupportsWebRTCRecord bool `json:"supports_webrtc_record"` + SupportsWebRTC bool `json:"supports_webrtc"` + VedioClaritiy int `json:"vedio_clarity"` + VideoClaritiy int `json:"video_clarity"` + VideoClarities []int `json:"video_clarities"` } type OpenIoTHubConfig struct { - Url string `json:"url"` - ClientID string `json:"client_id"` - Username string `json:"username"` - Password string `json:"password"` - + Url string `json:"url"` + ClientID string `json:"client_id"` + Username string `json:"username"` + Password string `json:"password"` SinkTopic struct { IPC string `json:"ipc"` } `json:"sink_topic"` - SourceSink struct { IPC string `json:"ipc"` } `json:"source_topic"` + ExpireTime int `json:"expire_time"` +} - ExpireTime int `json:"expire_time"` +type WebRTCConfigResponse struct { + Success bool `json:"success"` + Result WebRTConfig `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` +} + +type TokenResponse struct { + Success bool `json:"success"` + Result Token `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` +} + +type AllocateRequest struct { + Type string `json:"type"` +} + +type AllocateResponse struct { + Success bool `json:"success"` + Result struct { + URL string `json:"url"` + } `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` +} + +type OpenIoTHubConfigRequest struct { + UID string `json:"uid"` + UniqueID string `json:"unique_id"` + LinkType string `json:"link_type"` + Topics string `json:"topics"` +} + +type OpenIoTHubConfigResponse struct { + Success bool `json:"success"` + Result OpenIoTHubConfig `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` } const ( defaultTimeout = 5 * time.Second ) -func NewTuyaClient(openAPIURL string, deviceID string, uid string, clientID string, secret string, streamType string) (*TuyaClient, error) { +func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId string, clientSecret string, streamMode string) (*TuyaClient, error) { client := &TuyaClient{ httpClient: &http.Client{Timeout: defaultTimeout}, mqtt: &TuyaMQTT{waiter: core.Waiter{}}, apiURL: openAPIURL, - sessionID: core.RandString(6, 62), - clientID: clientID, - deviceID: deviceID, - secret: secret, + sessionId: core.RandString(6, 62), + clientId: clientId, + deviceId: deviceId, + clientSecret: clientSecret, uid: uid, + hasBackchannel: false, } if err := client.InitToken(); err != nil { return nil, fmt.Errorf("failed to initialize token: %w", err) } - if streamType == "rtsp" { + if streamMode == "rtsp" { if err := client.GetStreamUrl("rtsp"); err != nil { return nil, fmt.Errorf("failed to get RTSP URL: %w", err) } - } else if streamType == "hls" { + } else if streamMode == "hls" { if err := client.GetStreamUrl("hls"); err != nil { return nil, fmt.Errorf("failed to get HLS URL: %w", err) } @@ -193,14 +207,14 @@ func(c *TuyaClient) Request(method string, url string, body any) ([]byte, error) if body != nil { jsonBody, err := json.Marshal(body) if err != nil { - return nil, fmt.Errorf("failed to marshal request body: %w", err) + return nil, err } bodyReader = bytes.NewReader(jsonBody) } req, err := http.NewRequest(method, url, bodyReader) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return nil, err } ts := time.Now().UnixNano() / 1000000 @@ -212,24 +226,24 @@ func(c *TuyaClient) Request(method string, url string, body any) ([]byte, error) req.Header.Set("Access-Control-Allow-Methods", "*") req.Header.Set("Access-Control-Allow-Headers", "*") req.Header.Set("mode", "no-cors") - req.Header.Set("client_id", c.clientID) + req.Header.Set("client_id", c.clientId) req.Header.Set("access_token", c.accessToken) req.Header.Set("sign", sign) req.Header.Set("t", strconv.FormatInt(ts, 10)) response, err := c.httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("failed to send request: %w", err) + return nil, err } defer response.Body.Close() res, err := io.ReadAll(response.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) + return nil, err } if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("request failed with status code %d: %s", response.StatusCode, string(res)) + return nil, err } return res, nil @@ -243,13 +257,17 @@ func(c *TuyaClient) InitToken() (err error) { body, err := c.Request("GET", url, nil) if err != nil { - return fmt.Errorf("failed to get token: %w", err) + return err } var tokenResponse TokenResponse err = json.Unmarshal(body, &tokenResponse) if err != nil { - return fmt.Errorf("failed to unmarshal token response: %w", err) + return err + } + + if !tokenResponse.Success { + return fmt.Errorf("error: %s", tokenResponse.Msg) } c.accessToken = tokenResponse.Result.AccessToken @@ -260,119 +278,139 @@ func(c *TuyaClient) InitToken() (err error) { } func(c *TuyaClient) InitDevice() (err error) { - url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.apiURL, c.uid, c.deviceID) + url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.apiURL, c.uid, c.deviceId) - body, err := c.Request("GET", url, nil) - if err != nil { - return fmt.Errorf("failed to get webrtc-configs: %w", err) - } - - var webRTCConfigResponse WebRTCConfigResponse - err = json.Unmarshal(body, &webRTCConfigResponse) - if err != nil { - return fmt.Errorf("failed to unmarshal webrtc-configs response: %w", err) - } - - c.motoID = webRTCConfigResponse.Result.MotoID - c.auth = webRTCConfigResponse.Result.Auth - - var skill Skill - err = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), &skill) + body, err := c.Request("GET", url, nil) if err != nil { - return fmt.Errorf("failed to unmarshal skill: %w", err) + return err } - var audioDirection string - if contains(webRTCConfigResponse.Result.AudioAttributes.CallMode, 2) && contains(webRTCConfigResponse.Result.AudioAttributes.HardwareCapability, 1) { - audioDirection = core.DirectionSendRecv - } else { - audioDirection = core.DirectionRecvonly + var webRTCConfigResponse WebRTCConfigResponse + err = json.Unmarshal(body, &webRTCConfigResponse) + if err != nil { + return err + } + + if !webRTCConfigResponse.Success { + return fmt.Errorf("error: %s", webRTCConfigResponse.Msg) } - - c.medias = make([]*core.Media, 0) - if len(skill.Audios) > 0 { - for _, audio := range skill.Audios { - c.medias = append(c.medias, &core.Media{ - Kind: core.KindAudio, - Direction: audioDirection, - Codecs: []*core.Codec{ - { - Name: "PCMU", - ClockRate: uint32(audio.SampleRate), - Channels: uint8(audio.Channels), - }, - }, - }) - } - } else { - c.medias = append(c.medias, &core.Media{ - Kind: core.KindAudio, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ + + c.motoId = webRTCConfigResponse.Result.MotoID + c.auth = webRTCConfigResponse.Result.Auth + + c.skill = &Skill{ + Audios: []struct { + Channels int `json:"channels"` + DataBit int `json:"dataBit"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` + }{}, + Videos: []struct { + StreamType int `json:"streamType"` + ProfileId string `json:"profileId"` + Width int `json:"width"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` + Height int `json:"height"` + }{}, + } + + if webRTCConfigResponse.Result.Skill != "" { + _ = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), c.skill) + } + + var audioDirection string + if contains(webRTCConfigResponse.Result.AudioAttributes.CallMode, 2) && + contains(webRTCConfigResponse.Result.AudioAttributes.HardwareCapability, 1) { + audioDirection = core.DirectionSendRecv + c.hasBackchannel = true + } else { + audioDirection = core.DirectionRecvonly + c.hasBackchannel = false + } + + c.medias = make([]*core.Media, 0) + + if len(c.skill.Audios) > 0 { + // Use the first Audio-Codec + audio := c.skill.Audios[0] + + c.medias = append(c.medias, &core.Media{ + Kind: core.KindAudio, + Direction: audioDirection, + Codecs: []*core.Codec{ + { + Name: "PCMU", + ClockRate: uint32(audio.SampleRate), + Channels: uint8(audio.Channels), + }, + }, + }) + } else { + // Use default values for Audio + c.medias = append(c.medias, &core.Media{ + Kind: core.KindAudio, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: "PCMU", + ClockRate: uint32(8000), + Channels: uint8(1), + }, + }, + }) + } + + if len(c.skill.Videos) > 0 { + // Use the first Video-Codec + video := c.skill.Videos[0] + + c.medias = append(c.medias, &core.Media{ + Kind: core.KindVideo, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: core.CodecH265, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }, { - Name: "PCMU", - ClockRate: uint32(8000), - Channels: uint8(1), - }, - }, - }) - } + Name: core.CodecH264, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }, + }, + }) + } else { + // Use default values for Video + c.medias = append(c.medias, &core.Media{ + Kind: core.KindVideo, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: core.CodecH264, + ClockRate: uint32(90000), + PayloadType: 96, + }, + }, + }) + } - if len(skill.Videos) > 0 { - // take only the first video codec - video := skill.Videos[0] - - var name string - switch video.CodecType { - case 4: - name = core.CodecH265 - case 2: - name = core.CodecH264 - default: - name = core.CodecH264 - } - - c.medias = append(c.medias, &core.Media{ - Kind: core.KindVideo, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: name, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, - }, - }, - }) - } else { - c.medias = append(c.medias, &core.Media{ - Kind: core.KindVideo, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: core.CodecH264, - ClockRate: uint32(90000), - PayloadType: 96, - }, - }, - }) - } + iceServersBytes, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) + if err != nil { + return err + } - iceServersBytes, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) - if err != nil { - return fmt.Errorf("failed to marshal ICE servers: %w", err) - } + c.iceServers, err = webrtc.UnmarshalICEServers([]byte(iceServersBytes)) + if err != nil { + return err + } - - c.iceServers, err = webrtc.UnmarshalICEServers([]byte(iceServersBytes)) - if err != nil { - return fmt.Errorf("failed to unmarshal ICE servers: %w", err) - } - - return nil + return nil } func(c *TuyaClient) GetStreamUrl(streamType string) (err error) { - url := fmt.Sprintf("https://%s/v1.0/devices/%s/stream/actions/allocate", c.apiURL, c.deviceID) + url := fmt.Sprintf("https://%s/v1.0/devices/%s/stream/actions/allocate", c.apiURL, c.deviceId) request := &AllocateRequest{ Type: streamType, @@ -380,26 +418,24 @@ func(c *TuyaClient) GetStreamUrl(streamType string) (err error) { body, err := c.Request("POST", url, request) if err != nil { - return fmt.Errorf("failed to get rtsp url: %w", err) + return err } var allosResponse AllocateResponse err = json.Unmarshal(body, &allosResponse) if err != nil { - return fmt.Errorf("failed to unmarshal stream response: %w", err) + return err } if !allosResponse.Success { - return fmt.Errorf("failed to get stream url: %s", string(body)) + return fmt.Errorf("error: %s", allosResponse.Msg) } switch streamType { case "rtsp": c.rtspURL = allosResponse.Result.URL - fmt.Printf("RTSP URL: %s\n", c.rtspURL) case "hls": c.hlsURL = allosResponse.Result.URL - fmt.Printf("HLS URL: %s\n", c.hlsURL) default: return fmt.Errorf("unsupported stream type: %s", streamType) } @@ -419,24 +455,66 @@ func(c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { body, err := c.Request("POST", url, request) if err != nil { - return nil, fmt.Errorf("failed to get OpenIoTHub config: %w", err) + return nil, err } var openIoTHubConfigResponse OpenIoTHubConfigResponse err = json.Unmarshal(body, &openIoTHubConfigResponse) if err != nil { - return nil, fmt.Errorf("failed to unmarshal OpenIoTHub config response: %w", err) + return nil, err } if !openIoTHubConfigResponse.Success { - return nil, fmt.Errorf("failed to get OpenIoTHub config: %s", string(body)) + return nil, fmt.Errorf("error: %s", openIoTHubConfigResponse.Msg) } return &openIoTHubConfigResponse.Result, nil } +// Search the streamType based on the selection "main" or "sub" +func (c *TuyaClient) getStreamType(streamChoice string) uint32 { + // Default streamType if nothing is found + defaultStreamType := uint32(1) + + if c.skill == nil || len(c.skill.Videos) == 0 { + return defaultStreamType + } + + // Find the highest and lowest resolution + var highestResType uint32 = defaultStreamType + var highestRes int = 0 + var lowestResType uint32 = defaultStreamType + var lowestRes int = 0 + + for _, video := range c.skill.Videos { + res := video.Width * video.Height + + // Highest Resolution + if res > highestRes { + highestRes = res + highestResType = uint32(video.StreamType) + } + + // Lower Resolution (or first if not set yet) + if lowestRes == 0 || res < lowestRes { + lowestRes = res + lowestResType = uint32(video.StreamType) + } + } + + // Return the streamType based on the selection + switch streamChoice { + case "main": + return highestResType + case "sub": + return lowestResType + default: + return defaultStreamType + } +} + func(c *TuyaClient) calBusinessSign(ts int64) string { - data := fmt.Sprintf("%s%s%s%d", c.clientID, c.accessToken, c.secret, ts) + data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts) val := md5.Sum([]byte(data)) res := fmt.Sprintf("%X", val) return res diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 4735c003..7b1406ea 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" "net/url" - "strconv" + "regexp" "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/pkg/core" @@ -14,9 +14,9 @@ import ( ) type Client struct { - api *TuyaClient - prod core.Producer - done chan struct{} + api *TuyaClient + prod core.Producer + done chan struct{} } const ( @@ -38,13 +38,13 @@ func Dial(rawURL string) (core.Producer, error) { query := u.Query() deviceID := query.Get("device_id") uid := query.Get("uid") - clientID := query.Get("client_id") - secret := query.Get("secret") - resolution := query.Get("resolution") + clientId := query.Get("client_id") + clientSecret := query.Get("client_secret") streamType := query.Get("type") - useRTSP := streamType == "rtsp" - useHLS := streamType == "hls" - useWebRTC := streamType == "webrtc" || streamType == "" + streamMode := query.Get("mode") + useRTSP := streamMode == "rtsp" + useHLS := streamMode == "hls" + useWebRTC := streamMode == "webrtc" || streamMode == "" // check if host is correct switch u.Hostname() { @@ -58,8 +58,12 @@ func Dial(rawURL string) (core.Producer, error) { return nil, fmt.Errorf("tuya: wrong host %s", u.Hostname()) } - if deviceID == "" || uid == "" || clientID == "" || secret == "" { - return nil, errors.New("tuya: wrong query") + if deviceID == "" || clientId == "" || clientSecret == "" { + return nil, errors.New("tuya: no device_id, client_id or client_secret") + } + + if useWebRTC && uid == "" { + return nil, errors.New("tuya: no uid") } if !useRTSP && !useHLS && !useWebRTC { @@ -67,7 +71,7 @@ func Dial(rawURL string) (core.Producer, error) { } // Initialize Tuya API client - tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientID, secret, streamType) + tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientId, clientSecret, streamType) if err != nil { return nil, err } @@ -157,7 +161,7 @@ func Dial(rawURL string) (core.Producer, error) { } client.api.mqtt.handleError = func(err error) { - fmt.Printf("Tuya error: %s\n", err.Error()) + // fmt.Printf("tuya: error: %s\n", err.Error()) client.Stop() } @@ -188,21 +192,18 @@ func Dial(rawURL string) (core.Producer, error) { return nil, err } + // horter sdp, remove a=extmap... line, device ONLY allow 8KB json payload + re := regexp.MustCompile(`\r\na=extmap[^\r\n]*`) + offer = re.ReplaceAllString(offer, "") + // Send offer - client.api.sendOffer(offer) + client.api.sendOffer(offer, tuyaAPI.getStreamType(streamType)) sendOffer.Done(nil) if err = connState.Wait(); err != nil { return nil, err } - if resolution != "" { - value, err := strconv.Atoi(resolution) - if err == nil { - client.api.sendResolution(value) - } - } - return client, nil } } @@ -216,11 +217,11 @@ func (c *Client) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, } func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiver) error { - if webrtcProd, ok := c.prod.(*webrtc.Conn); ok { - return webrtcProd.AddTrack(media, codec, track) + if prod, ok := c.prod.(*webrtc.Conn); ok { + return prod.AddTrack(media, codec, track) } - return fmt.Errorf("add track not supported") + return nil } func (c *Client) Start() error { diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index 3da27c08..fbd50021 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -24,64 +24,64 @@ type TuyaMQTT struct { } type MqttFrameHeader struct { - Type string `json:"type"` - From string `json:"from"` - To string `json:"to"` - SubDevID string `json:"sub_dev_id"` - SessionID string `json:"sessionid"` - MotoID string `json:"moto_id"` - TransactionID string `json:"tid"` + Type string `json:"type"` + From string `json:"from"` + To string `json:"to"` + SubDevID string `json:"sub_dev_id"` + SessionID string `json:"sessionid"` + MotoID string `json:"moto_id"` + TransactionID string `json:"tid"` } type MqttFrame struct { - Header MqttFrameHeader `json:"header"` - Message json.RawMessage `json:"msg"` + Header MqttFrameHeader `json:"header"` + Message json.RawMessage `json:"msg"` } type OfferFrame struct { - Mode string `json:"mode"` - Sdp string `json:"sdp"` - StreamType uint32 `json:"stream_type"` - Auth string `json:"auth"` + Mode string `json:"mode"` + Sdp string `json:"sdp"` + StreamType uint32 `json:"stream_type"` + Auth string `json:"auth"` } type AnswerFrame struct { - Mode string `json:"mode"` - Sdp string `json:"sdp"` + Mode string `json:"mode"` + Sdp string `json:"sdp"` } type CandidateFrame struct { - Mode string `json:"mode"` - Candidate string `json:"candidate"` + Mode string `json:"mode"` + Candidate string `json:"candidate"` } type ResolutionFrame struct { - Mode string `json:"mode"` - Value int `json:"value"` + Mode string `json:"mode"` + Value int `json:"value"` } type DisconnectFrame struct { - Mode string `json:"mode"` + Mode string `json:"mode"` } type MqttMessage struct { - Protocol int `json:"protocol"` - Pv string `json:"pv"` - T int64 `json:"t"` - Data MqttFrame `json:"data"` + Protocol int `json:"protocol"` + Pv string `json:"pv"` + T int64 `json:"t"` + Data MqttFrame `json:"data"` } func(c *TuyaClient) StartMQTT() error { hubConfig, err := c.LoadHubConfig() if err != nil { - return fmt.Errorf("failed to load hub config: %w", err) + return err } c.mqtt.publishTopic = hubConfig.SinkTopic.IPC c.mqtt.subscribeTopic = hubConfig.SourceSink.IPC - c.mqtt.publishTopic = strings.Replace(c.mqtt.publishTopic, "moto_id", c.motoID, 1) - c.mqtt.publishTopic = strings.Replace(c.mqtt.publishTopic, "{device_id}", c.deviceID, 1) + c.mqtt.publishTopic = strings.Replace(c.mqtt.publishTopic, "moto_id", c.motoId, 1) + c.mqtt.publishTopic = strings.Replace(c.mqtt.publishTopic, "{device_id}", c.deviceId, 1) parts := strings.Split(c.mqtt.subscribeTopic, "/") c.mqtt.uid = parts[3] @@ -96,7 +96,7 @@ func(c *TuyaClient) StartMQTT() error { c.mqtt.client = mqtt.NewClient(opts) if token := c.mqtt.client.Connect(); token.Wait() && token.Error() != nil { - return fmt.Errorf("failed to connect to MQTT broker: %w", token.Error()) + return token.Error() } if err := c.mqtt.waiter.Wait(); err != nil { @@ -129,7 +129,7 @@ func(c *TuyaClient) consume(client mqtt.Client, msg mqtt.Message) { return } - if rmqtt.Data.Header.SessionID != c.sessionID { + if rmqtt.Data.Header.SessionID != c.sessionId { return } @@ -202,11 +202,11 @@ func(c *TuyaMQTT) onError(err error) { } } -func (c *TuyaClient) sendOffer(sdp string) { +func (c *TuyaClient) sendOffer(sdp string, streamType uint32) { c.sendMqttMessage("offer", 302, "", OfferFrame{ Mode: "webrtc", Sdp: sdp, - StreamType: 1, + StreamType: streamType, Auth: c.auth, }) } @@ -251,9 +251,9 @@ func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transacti Header: MqttFrameHeader{ Type: messageType, From: c.mqtt.uid, - To: c.deviceID, - SessionID: c.sessionID, - MotoID: c.motoID, + To: c.deviceId, + SessionID: c.sessionId, + MotoID: c.motoId, TransactionID: transactionID, }, Message: jsonMessage, From 6d8d6a91ef89f887e142285123958880df67cd91 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 12 May 2025 22:55:55 +0200 Subject: [PATCH 08/60] format --- main.go | 2 +- pkg/hap/camera/accessory.go | 2 +- pkg/tapo/client.go | 2 +- pkg/tuya/api.go | 470 ++++++++++++++++++------------------ pkg/tuya/client.go | 26 +- pkg/tuya/mqtt.go | 98 ++++---- 6 files changed, 300 insertions(+), 300 deletions(-) diff --git a/main.go b/main.go index 3bbf632f..e2698331 100644 --- a/main.go +++ b/main.go @@ -89,7 +89,7 @@ func main() { roborock.Init() // roborock source homekit.Init() // homekit source ring.Init() // ring source - tuya.Init() // tuya source + tuya.Init() // tuya source nest.Init() // nest source bubble.Init() // bubble source expr.Init() // expr source diff --git a/pkg/hap/camera/accessory.go b/pkg/hap/camera/accessory.go index 42037d96..886b035d 100644 --- a/pkg/hap/camera/accessory.go +++ b/pkg/hap/camera/accessory.go @@ -62,7 +62,7 @@ func ServiceCameraRTPStreamManagement() *hap.Service { VideoAttrs: []VideoAttrs{ {Width: 1920, Height: 1080, Framerate: 30}, {Width: 1280, Height: 720, Framerate: 30}, // important for iPhones - {Width: 320, Height: 240, Framerate: 15}, // apple watch + {Width: 320, Height: 240, Framerate: 15}, // apple watch }, }, }, diff --git a/pkg/tapo/client.go b/pkg/tapo/client.go index c19267ff..5a9af501 100644 --- a/pkg/tapo/client.go +++ b/pkg/tapo/client.go @@ -292,7 +292,7 @@ func dial(req *http.Request, brand, username, password string) (net.Conn, *http. return nil, nil, err } _, _ = io.Copy(io.Discard, res.Body) // discard leftovers - _ = res.Body.Close() // ignore response body + _ = res.Body.Close() // ignore response body auth := res.Header.Get("WWW-Authenticate") diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index 4ee0bf8f..29152951 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -17,146 +17,146 @@ import ( ) type TuyaClient struct { - httpClient *http.Client - mqtt *TuyaMQTT - apiURL string - rtspURL string - hlsURL string - sessionId string - clientId string - clientSecret string - deviceId string - accessToken string - refreshToken string - expireTime int64 - uid string - motoId string - auth string - skill *Skill - iceServers []pionWebrtc.ICEServer - medias []*core.Media - hasBackchannel bool + httpClient *http.Client + mqtt *TuyaMQTT + apiURL string + rtspURL string + hlsURL string + sessionId string + clientId string + clientSecret string + deviceId string + accessToken string + refreshToken string + expireTime int64 + uid string + motoId string + auth string + skill *Skill + iceServers []pionWebrtc.ICEServer + medias []*core.Media + hasBackchannel bool } type Token struct { - UID string `json:"uid"` - AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` - ExpireTime int64 `json:"expire_time"` + UID string `json:"uid"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpireTime int64 `json:"expire_time"` } type AudioAttributes struct { - CallMode []int `json:"call_mode"` // 1 = one way, 2 = two way - HardwareCapability []int `json:"hardware_capability"` // 1 = mic, 2 = speaker + CallMode []int `json:"call_mode"` // 1 = one way, 2 = two way + HardwareCapability []int `json:"hardware_capability"` // 1 = mic, 2 = speaker } type OpenApiICE struct { - Urls string `json:"urls"` - Username string `json:"username"` - Credential string `json:"credential"` - TTL int `json:"ttl"` + Urls string `json:"urls"` + Username string `json:"username"` + Credential string `json:"credential"` + TTL int `json:"ttl"` } type WebICE struct { - Urls string `json:"urls"` - Username string `json:"username,omitempty"` - Credential string `json:"credential,omitempty"` + Urls string `json:"urls"` + Username string `json:"username,omitempty"` + Credential string `json:"credential,omitempty"` } type P2PConfig struct { - Ices []OpenApiICE `json:"ices"` + Ices []OpenApiICE `json:"ices"` } type Skill struct { - WebRTC int `json:"webrtc"` + WebRTC int `json:"webrtc"` Audios []struct { - Channels int `json:"channels"` - DataBit int `json:"dataBit"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` + Channels int `json:"channels"` + DataBit int `json:"dataBit"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` } `json:"audios"` Videos []struct { - StreamType int `json:"streamType"` // streamType = 2 => main stream - streamType = 4 => sub stream - ProfileId string `json:"profileId"` - Width int `json:"width"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` - Height int `json:"height"` + StreamType int `json:"streamType"` // streamType = 2 => main stream - streamType = 4 => sub stream + ProfileId string `json:"profileId"` + Width int `json:"width"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` + Height int `json:"height"` } `json:"videos"` } type WebRTConfig struct { - AudioAttributes AudioAttributes `json:"audio_attributes"` - Auth string `json:"auth"` - ID string `json:"id"` - MotoID string `json:"moto_id"` - P2PConfig P2PConfig `json:"p2p_config"` - ProtocolVersion string `json:"protocol_version"` - Skill string `json:"skill"` - SupportsWebRTCRecord bool `json:"supports_webrtc_record"` - SupportsWebRTC bool `json:"supports_webrtc"` - VedioClaritiy int `json:"vedio_clarity"` - VideoClaritiy int `json:"video_clarity"` - VideoClarities []int `json:"video_clarities"` + AudioAttributes AudioAttributes `json:"audio_attributes"` + Auth string `json:"auth"` + ID string `json:"id"` + MotoID string `json:"moto_id"` + P2PConfig P2PConfig `json:"p2p_config"` + ProtocolVersion string `json:"protocol_version"` + Skill string `json:"skill"` + SupportsWebRTCRecord bool `json:"supports_webrtc_record"` + SupportsWebRTC bool `json:"supports_webrtc"` + VedioClaritiy int `json:"vedio_clarity"` + VideoClaritiy int `json:"video_clarity"` + VideoClarities []int `json:"video_clarities"` } type OpenIoTHubConfig struct { - Url string `json:"url"` - ClientID string `json:"client_id"` - Username string `json:"username"` - Password string `json:"password"` + Url string `json:"url"` + ClientID string `json:"client_id"` + Username string `json:"username"` + Password string `json:"password"` SinkTopic struct { IPC string `json:"ipc"` } `json:"sink_topic"` SourceSink struct { IPC string `json:"ipc"` } `json:"source_topic"` - ExpireTime int `json:"expire_time"` + ExpireTime int `json:"expire_time"` } type WebRTCConfigResponse struct { - Success bool `json:"success"` - Result WebRTConfig `json:"result"` - Msg string `json:"msg,omitempty"` - Code int `json:"code,omitempty"` + Success bool `json:"success"` + Result WebRTConfig `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` } type TokenResponse struct { - Success bool `json:"success"` - Result Token `json:"result"` - Msg string `json:"msg,omitempty"` - Code int `json:"code,omitempty"` + Success bool `json:"success"` + Result Token `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` } type AllocateRequest struct { - Type string `json:"type"` + Type string `json:"type"` } type AllocateResponse struct { - Success bool `json:"success"` + Success bool `json:"success"` Result struct { URL string `json:"url"` } `json:"result"` - Msg string `json:"msg,omitempty"` - Code int `json:"code,omitempty"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` } type OpenIoTHubConfigRequest struct { - UID string `json:"uid"` - UniqueID string `json:"unique_id"` - LinkType string `json:"link_type"` - Topics string `json:"topics"` + UID string `json:"uid"` + UniqueID string `json:"unique_id"` + LinkType string `json:"link_type"` + Topics string `json:"topics"` } type OpenIoTHubConfigResponse struct { - Success bool `json:"success"` - Result OpenIoTHubConfig `json:"result"` - Msg string `json:"msg,omitempty"` - Code int `json:"code,omitempty"` + Success bool `json:"success"` + Result OpenIoTHubConfig `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` } const ( - defaultTimeout = 5 * time.Second + defaultTimeout = 5 * time.Second ) func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId string, clientSecret string, streamMode string) (*TuyaClient, error) { @@ -164,7 +164,7 @@ func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId stri httpClient: &http.Client{Timeout: defaultTimeout}, mqtt: &TuyaMQTT{waiter: core.Waiter{}}, apiURL: openAPIURL, - sessionId: core.RandString(6, 62), + sessionId: core.RandString(6, 62), clientId: clientId, deviceId: deviceId, clientSecret: clientSecret, @@ -202,7 +202,7 @@ func (c *TuyaClient) Close() { c.httpClient.CloseIdleConnections() } -func(c *TuyaClient) Request(method string, url string, body any) ([]byte, error) { +func (c *TuyaClient) Request(method string, url string, body any) ([]byte, error) { var bodyReader io.Reader if body != nil { jsonBody, err := json.Marshal(body) @@ -249,7 +249,7 @@ func(c *TuyaClient) Request(method string, url string, body any) ([]byte, error) return res, nil } -func(c *TuyaClient) InitToken() (err error) { +func (c *TuyaClient) InitToken() (err error) { url := fmt.Sprintf("https://%s/v1.0/token?grant_type=1", c.apiURL) c.accessToken = "" @@ -259,7 +259,7 @@ func(c *TuyaClient) InitToken() (err error) { if err != nil { return err } - + var tokenResponse TokenResponse err = json.Unmarshal(body, &tokenResponse) if err != nil { @@ -277,139 +277,139 @@ func(c *TuyaClient) InitToken() (err error) { return nil } -func(c *TuyaClient) InitDevice() (err error) { - url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.apiURL, c.uid, c.deviceId) +func (c *TuyaClient) InitDevice() (err error) { + url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.apiURL, c.uid, c.deviceId) - body, err := c.Request("GET", url, nil) - if err != nil { - return err - } + body, err := c.Request("GET", url, nil) + if err != nil { + return err + } - var webRTCConfigResponse WebRTCConfigResponse - err = json.Unmarshal(body, &webRTCConfigResponse) - if err != nil { - return err - } + var webRTCConfigResponse WebRTCConfigResponse + err = json.Unmarshal(body, &webRTCConfigResponse) + if err != nil { + return err + } if !webRTCConfigResponse.Success { return fmt.Errorf("error: %s", webRTCConfigResponse.Msg) } c.motoId = webRTCConfigResponse.Result.MotoID - c.auth = webRTCConfigResponse.Result.Auth + c.auth = webRTCConfigResponse.Result.Auth - c.skill = &Skill{ - Audios: []struct { - Channels int `json:"channels"` - DataBit int `json:"dataBit"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` - }{}, - Videos: []struct { - StreamType int `json:"streamType"` - ProfileId string `json:"profileId"` - Width int `json:"width"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` - Height int `json:"height"` - }{}, - } + c.skill = &Skill{ + Audios: []struct { + Channels int `json:"channels"` + DataBit int `json:"dataBit"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` + }{}, + Videos: []struct { + StreamType int `json:"streamType"` + ProfileId string `json:"profileId"` + Width int `json:"width"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` + Height int `json:"height"` + }{}, + } - if webRTCConfigResponse.Result.Skill != "" { - _ = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), c.skill) - } + if webRTCConfigResponse.Result.Skill != "" { + _ = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), c.skill) + } - var audioDirection string - if contains(webRTCConfigResponse.Result.AudioAttributes.CallMode, 2) && - contains(webRTCConfigResponse.Result.AudioAttributes.HardwareCapability, 1) { - audioDirection = core.DirectionSendRecv + var audioDirection string + if contains(webRTCConfigResponse.Result.AudioAttributes.CallMode, 2) && + contains(webRTCConfigResponse.Result.AudioAttributes.HardwareCapability, 1) { + audioDirection = core.DirectionSendRecv c.hasBackchannel = true - } else { - audioDirection = core.DirectionRecvonly + } else { + audioDirection = core.DirectionRecvonly c.hasBackchannel = false - } - - c.medias = make([]*core.Media, 0) - - if len(c.skill.Audios) > 0 { - // Use the first Audio-Codec - audio := c.skill.Audios[0] + } - c.medias = append(c.medias, &core.Media{ - Kind: core.KindAudio, - Direction: audioDirection, - Codecs: []*core.Codec{ - { - Name: "PCMU", - ClockRate: uint32(audio.SampleRate), - Channels: uint8(audio.Channels), - }, - }, - }) - } else { - // Use default values for Audio - c.medias = append(c.medias, &core.Media{ - Kind: core.KindAudio, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: "PCMU", - ClockRate: uint32(8000), - Channels: uint8(1), - }, - }, - }) - } + c.medias = make([]*core.Media, 0) - if len(c.skill.Videos) > 0 { - // Use the first Video-Codec - video := c.skill.Videos[0] + if len(c.skill.Audios) > 0 { + // Use the first Audio-Codec + audio := c.skill.Audios[0] - c.medias = append(c.medias, &core.Media{ - Kind: core.KindVideo, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: core.CodecH265, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, - }, + c.medias = append(c.medias, &core.Media{ + Kind: core.KindAudio, + Direction: audioDirection, + Codecs: []*core.Codec{ { - Name: core.CodecH264, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, - }, - }, - }) - } else { - // Use default values for Video - c.medias = append(c.medias, &core.Media{ - Kind: core.KindVideo, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: core.CodecH264, - ClockRate: uint32(90000), - PayloadType: 96, - }, - }, - }) - } + Name: "PCMU", + ClockRate: uint32(audio.SampleRate), + Channels: uint8(audio.Channels), + }, + }, + }) + } else { + // Use default values for Audio + c.medias = append(c.medias, &core.Media{ + Kind: core.KindAudio, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: "PCMU", + ClockRate: uint32(8000), + Channels: uint8(1), + }, + }, + }) + } - iceServersBytes, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) - if err != nil { - return err - } + if len(c.skill.Videos) > 0 { + // Use the first Video-Codec + video := c.skill.Videos[0] - c.iceServers, err = webrtc.UnmarshalICEServers([]byte(iceServersBytes)) - if err != nil { - return err - } + c.medias = append(c.medias, &core.Media{ + Kind: core.KindVideo, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: core.CodecH265, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }, + { + Name: core.CodecH264, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }, + }, + }) + } else { + // Use default values for Video + c.medias = append(c.medias, &core.Media{ + Kind: core.KindVideo, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: core.CodecH264, + ClockRate: uint32(90000), + PayloadType: 96, + }, + }, + }) + } - return nil + iceServersBytes, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) + if err != nil { + return err + } + + c.iceServers, err = webrtc.UnmarshalICEServers([]byte(iceServersBytes)) + if err != nil { + return err + } + + return nil } -func(c *TuyaClient) GetStreamUrl(streamType string) (err error) { +func (c *TuyaClient) GetStreamUrl(streamType string) (err error) { url := fmt.Sprintf("https://%s/v1.0/devices/%s/stream/actions/allocate", c.apiURL, c.deviceId) request := &AllocateRequest{ @@ -443,7 +443,7 @@ func(c *TuyaClient) GetStreamUrl(streamType string) (err error) { return nil } -func(c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { +func (c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { url := fmt.Sprintf("https://%s/v2.0/open-iot-hub/access/config", c.apiURL) request := &OpenIoTHubConfigRequest{ @@ -467,53 +467,53 @@ func(c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { if !openIoTHubConfigResponse.Success { return nil, fmt.Errorf("error: %s", openIoTHubConfigResponse.Msg) } - + return &openIoTHubConfigResponse.Result, nil } // Search the streamType based on the selection "main" or "sub" func (c *TuyaClient) getStreamType(streamChoice string) uint32 { // Default streamType if nothing is found - defaultStreamType := uint32(1) + defaultStreamType := uint32(1) + + if c.skill == nil || len(c.skill.Videos) == 0 { + return defaultStreamType + } - if c.skill == nil || len(c.skill.Videos) == 0 { - return defaultStreamType - } - // Find the highest and lowest resolution - var highestResType uint32 = defaultStreamType - var highestRes int = 0 - var lowestResType uint32 = defaultStreamType - var lowestRes int = 0 - - for _, video := range c.skill.Videos { - res := video.Width * video.Height - - // Highest Resolution - if res > highestRes { - highestRes = res - highestResType = uint32(video.StreamType) - } - + var highestResType uint32 = defaultStreamType + var highestRes int = 0 + var lowestResType uint32 = defaultStreamType + var lowestRes int = 0 + + for _, video := range c.skill.Videos { + res := video.Width * video.Height + + // Highest Resolution + if res > highestRes { + highestRes = res + highestResType = uint32(video.StreamType) + } + // Lower Resolution (or first if not set yet) - if lowestRes == 0 || res < lowestRes { - lowestRes = res - lowestResType = uint32(video.StreamType) - } - } - - // Return the streamType based on the selection - switch streamChoice { - case "main": - return highestResType - case "sub": - return lowestResType - default: - return defaultStreamType - } + if lowestRes == 0 || res < lowestRes { + lowestRes = res + lowestResType = uint32(video.StreamType) + } + } + + // Return the streamType based on the selection + switch streamChoice { + case "main": + return highestResType + case "sub": + return lowestResType + default: + return defaultStreamType + } } -func(c *TuyaClient) calBusinessSign(ts int64) string { +func (c *TuyaClient) calBusinessSign(ts int64) string { data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts) val := md5.Sum([]byte(data)) res := fmt.Sprintf("%X", val) @@ -527,4 +527,4 @@ func contains(slice []int, val int) bool { } } return false -} \ No newline at end of file +} diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 7b1406ea..ef9cbcc2 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -14,18 +14,18 @@ import ( ) type Client struct { - api *TuyaClient - prod core.Producer - done chan struct{} + api *TuyaClient + prod core.Producer + done chan struct{} } const ( - DefaultCnURL = "openapi.tuyacn.com" - DefaultWestUsURL = "openapi.tuyaus.com" - DefaultEastUsURL = "openapi-ueaz.tuyaus.com" + DefaultCnURL = "openapi.tuyacn.com" + DefaultWestUsURL = "openapi.tuyaus.com" + DefaultEastUsURL = "openapi-ueaz.tuyaus.com" DefaultCentralEuURL = "openapi.tuyaeu.com" - DefaultWestEuURL = "openapi-weaz.tuyaeu.com" - DefaultInURL = "openapi.tuyain.com" + DefaultWestEuURL = "openapi-weaz.tuyaeu.com" + DefaultInURL = "openapi.tuyain.com" ) func Dial(rawURL string) (core.Producer, error) { @@ -77,7 +77,7 @@ func Dial(rawURL string) (core.Producer, error) { } client := &Client{ - api: tuyaAPI, + api: tuyaAPI, done: make(chan struct{}), } @@ -93,7 +93,7 @@ func Dial(rawURL string) (core.Producer, error) { return streams.GetProducer(client.api.hlsURL) } else { conf := pion.Configuration{ - ICEServers: client.api.iceServers, + ICEServers: client.api.iceServers, ICETransportPolicy: pion.ICETransportPolicyAll, BundlePolicy: pion.BundlePolicyMaxBundle, } @@ -138,12 +138,12 @@ func Dial(rawURL string) (core.Producer, error) { client.Stop() return } - + if err = prod.SetAnswer(answer.Sdp); err != nil { client.Stop() return } - + prod.SDP = answer.Sdp } @@ -159,7 +159,7 @@ func Dial(rawURL string) (core.Producer, error) { client.api.mqtt.handleDisconnect = func() { client.Stop() } - + client.api.mqtt.handleError = func(err error) { // fmt.Printf("tuya: error: %s\n", err.Error()) client.Stop() diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index fbd50021..bf9badd7 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -11,67 +11,67 @@ import ( ) type TuyaMQTT struct { - client mqtt.Client - waiter core.Waiter - publishTopic string - subscribeTopic string - uid string - closed bool - handleAnswer func(answer AnswerFrame) - handleCandidate func(candidate CandidateFrame) - handleDisconnect func() - handleError func(err error) + client mqtt.Client + waiter core.Waiter + publishTopic string + subscribeTopic string + uid string + closed bool + handleAnswer func(answer AnswerFrame) + handleCandidate func(candidate CandidateFrame) + handleDisconnect func() + handleError func(err error) } type MqttFrameHeader struct { - Type string `json:"type"` - From string `json:"from"` - To string `json:"to"` - SubDevID string `json:"sub_dev_id"` - SessionID string `json:"sessionid"` - MotoID string `json:"moto_id"` - TransactionID string `json:"tid"` + Type string `json:"type"` + From string `json:"from"` + To string `json:"to"` + SubDevID string `json:"sub_dev_id"` + SessionID string `json:"sessionid"` + MotoID string `json:"moto_id"` + TransactionID string `json:"tid"` } type MqttFrame struct { - Header MqttFrameHeader `json:"header"` - Message json.RawMessage `json:"msg"` + Header MqttFrameHeader `json:"header"` + Message json.RawMessage `json:"msg"` } type OfferFrame struct { - Mode string `json:"mode"` - Sdp string `json:"sdp"` - StreamType uint32 `json:"stream_type"` - Auth string `json:"auth"` + Mode string `json:"mode"` + Sdp string `json:"sdp"` + StreamType uint32 `json:"stream_type"` + Auth string `json:"auth"` } type AnswerFrame struct { - Mode string `json:"mode"` - Sdp string `json:"sdp"` + Mode string `json:"mode"` + Sdp string `json:"sdp"` } type CandidateFrame struct { - Mode string `json:"mode"` - Candidate string `json:"candidate"` + Mode string `json:"mode"` + Candidate string `json:"candidate"` } type ResolutionFrame struct { - Mode string `json:"mode"` - Value int `json:"value"` + Mode string `json:"mode"` + Value int `json:"value"` } type DisconnectFrame struct { - Mode string `json:"mode"` + Mode string `json:"mode"` } type MqttMessage struct { - Protocol int `json:"protocol"` - Pv string `json:"pv"` - T int64 `json:"t"` - Data MqttFrame `json:"data"` + Protocol int `json:"protocol"` + Pv string `json:"pv"` + T int64 `json:"t"` + Data MqttFrame `json:"data"` } -func(c *TuyaClient) StartMQTT() error { +func (c *TuyaClient) StartMQTT() error { hubConfig, err := c.LoadHubConfig() if err != nil { return err @@ -106,14 +106,14 @@ func(c *TuyaClient) StartMQTT() error { return nil } -func(c *TuyaClient) StopMQTT() { +func (c *TuyaClient) StopMQTT() { if c.mqtt.client != nil { c.sendDisconnect() c.mqtt.client.Disconnect(1000) } } -func(c *TuyaClient) onConnect(client mqtt.Client) { +func (c *TuyaClient) onConnect(client mqtt.Client) { if token := client.Subscribe(c.mqtt.subscribeTopic, 1, c.consume); token.Wait() && token.Error() != nil { c.mqtt.waiter.Done(token.Error()) return @@ -122,7 +122,7 @@ func(c *TuyaClient) onConnect(client mqtt.Client) { c.mqtt.waiter.Done(nil) } -func(c *TuyaClient) consume(client mqtt.Client, msg mqtt.Message) { +func (c *TuyaClient) consume(client mqtt.Client, msg mqtt.Message) { var rmqtt MqttMessage if err := json.Unmarshal(msg.Payload(), &rmqtt); err != nil { c.mqtt.onError(fmt.Errorf("unmarshal mqtt message fail: %s, payload: %s", err.Error(), string(msg.Payload()))) @@ -143,7 +143,7 @@ func(c *TuyaClient) consume(client mqtt.Client, msg mqtt.Message) { } } -func(c *TuyaMQTT) onMqttAnswer(msg *MqttMessage) { +func (c *TuyaMQTT) onMqttAnswer(msg *MqttMessage) { var answerFrame AnswerFrame if err := json.Unmarshal(msg.Data.Message, &answerFrame); err != nil { c.onError(fmt.Errorf("unmarshal mqtt answer frame fail: %s, session: %s, frame: %s", @@ -152,11 +152,11 @@ func(c *TuyaMQTT) onMqttAnswer(msg *MqttMessage) { string(msg.Data.Message))) return } - + c.onAnswer(answerFrame) } -func(c *TuyaMQTT) onMqttCandidate(msg *MqttMessage) { +func (c *TuyaMQTT) onMqttCandidate(msg *MqttMessage) { var candidateFrame CandidateFrame if err := json.Unmarshal(msg.Data.Message, &candidateFrame); err != nil { c.onError(fmt.Errorf("unmarshal mqtt candidate frame fail: %s, session: %s, frame: %s", @@ -173,30 +173,30 @@ func(c *TuyaMQTT) onMqttCandidate(msg *MqttMessage) { c.onCandidate(candidateFrame) } -func(c *TuyaMQTT) onMqttDisconnect() { +func (c *TuyaMQTT) onMqttDisconnect() { c.closed = true c.onDisconnect() } -func(c *TuyaMQTT) onAnswer(answer AnswerFrame) { +func (c *TuyaMQTT) onAnswer(answer AnswerFrame) { if c.handleAnswer != nil { c.handleAnswer(answer) } } -func(c *TuyaMQTT) onCandidate(candidate CandidateFrame) { +func (c *TuyaMQTT) onCandidate(candidate CandidateFrame) { if c.handleCandidate != nil { c.handleCandidate(candidate) } } -func(c *TuyaMQTT) onDisconnect() { +func (c *TuyaMQTT) onDisconnect() { if c.handleDisconnect != nil { c.handleDisconnect() } } -func(c *TuyaMQTT) onError(err error) { +func (c *TuyaMQTT) onError(err error) { if c.handleError != nil { c.handleError(err) } @@ -220,12 +220,12 @@ func (c *TuyaClient) sendCandidate(candidate string) { func (c *TuyaClient) sendResolution(resolution int) { c.sendMqttMessage("resolution", 302, "", ResolutionFrame{ - Mode: "webrtc", - Value: resolution, + Mode: "webrtc", + Value: resolution, }) } -func(c *TuyaClient) sendDisconnect() { +func (c *TuyaClient) sendDisconnect() { c.sendMqttMessage("disconnect", 302, "", DisconnectFrame{ Mode: "webrtc", }) From bd2cbe20e04284eabff7a0053b1a7ca10837159c Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 13 May 2025 14:25:10 +0200 Subject: [PATCH 09/60] add useful links --- pkg/tuya/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 pkg/tuya/README.md diff --git a/pkg/tuya/README.md b/pkg/tuya/README.md new file mode 100644 index 00000000..f5cbc814 --- /dev/null +++ b/pkg/tuya/README.md @@ -0,0 +1,6 @@ +## Useful links + +- https://developer.tuya.com/en/docs/iot/webrtc?id=Kacsd4x2hl0se +- https://github.com/tuya/webrtc-demo-go +- https://github.com/bacco007/HomeAssistantConfig/blob/master/custom_components/xtend_tuya/multi_manager/tuya_iot/ipc/webrtc/xt_tuya_iot_webrtc_manager.py +- https://ipc-us.ismartlife.me/ \ No newline at end of file From 4f969d750aef486389a43b216f706ba347462e34 Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 13 May 2025 14:25:22 +0200 Subject: [PATCH 10/60] readme --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index 9712bbde..43e1c83b 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Ultimate camera streaming application with support for RTSP, WebRTC, HomeKit, FF * [Source: DVRIP](#source-dvrip) * [Source: Tapo](#source-tapo) * [Source: Kasa](#source-kasa) + * [Source: Tuya](#source-tuya) * [Source: GoPro](#source-gopro) * [Source: Ivideon](#source-ivideon) * [Source: Hass](#source-hass) @@ -565,6 +566,41 @@ streams: Tested: KD110, KC200, KC401, KC420WS, EC71. +#### Source: Tuya + +[Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. + +- Obtain `client_id`, `client_secret`, `uid` and `device_id` from [Tuya IoT Platform](https://iot.tuya.com/) +- Use `mode` parameter to select the stream type: + - `webrtc` - WebRTC stream (default) + - `rtsp` - RTSP stream _(if available)_ + - `hls` - HLS stream _(if available)_ +- Use `type` parameter to select the stream type: _(if available)_ + - `main` - Main stream (default) + - `sub` - Sub stream + +```yaml +streams: + # Tuya WebRTC stream + tuya_webrtc: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX + + # Tuya WebRTC stream (same as above) + tuya_webrtc_2: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&mode=webrtc + + # Tuya WebRTC stream (HD) + tuya_webrtc_hd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&type=main + + # Tuya WebRTC stream (SD) + tuya_webrtc_sd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&type=sub + + # Using RTSP when available (no "uid" required) + tuya_rtsp: tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=rtsp + + # Using HLS when available (no "uid" required) + tuya_hls: tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=hls +``` + + #### Source: GoPro *[New in v1.8.3](https://github.com/AlexxIT/go2rtc/releases/tag/v1.8.3)* From 6c255cd2f2684ecd4ee6d5f215992512f9ddb99b Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 13 May 2025 14:26:12 +0200 Subject: [PATCH 11/60] fix two way audio --- pkg/tuya/api.go | 19 +++++------- pkg/tuya/client.go | 66 +++++++++++++++++++++++++----------------- pkg/webrtc/conn.go | 22 +++++++------- pkg/webrtc/consumer.go | 2 +- 4 files changed, 59 insertions(+), 50 deletions(-) diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index 29152951..2f7cd194 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -319,15 +319,8 @@ func (c *TuyaClient) InitDevice() (err error) { _ = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), c.skill) } - var audioDirection string - if contains(webRTCConfigResponse.Result.AudioAttributes.CallMode, 2) && - contains(webRTCConfigResponse.Result.AudioAttributes.HardwareCapability, 1) { - audioDirection = core.DirectionSendRecv - c.hasBackchannel = true - } else { - audioDirection = core.DirectionRecvonly - c.hasBackchannel = false - } + c.hasBackchannel = contains(webRTCConfigResponse.Result.AudioAttributes.CallMode, 2) && + contains(webRTCConfigResponse.Result.AudioAttributes.HardwareCapability, 1) c.medias = make([]*core.Media, 0) @@ -335,9 +328,14 @@ func (c *TuyaClient) InitDevice() (err error) { // Use the first Audio-Codec audio := c.skill.Audios[0] + direction := core.DirectionRecvonly + if c.hasBackchannel { + direction = core.DirectionSendRecv + } + c.medias = append(c.medias, &core.Media{ Kind: core.KindAudio, - Direction: audioDirection, + Direction: direction, Codecs: []*core.Codec{ { Name: "PCMU", @@ -471,7 +469,6 @@ func (c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { return &openIoTHubConfigResponse.Result, nil } -// Search the streamType based on the selection "main" or "sub" func (c *TuyaClient) getStreamType(streamChoice string) uint32 { // Default streamType if nothing is found defaultStreamType := uint32(1) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index ef9cbcc2..586a8bbd 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -1,7 +1,6 @@ package tuya import ( - "encoding/json" "errors" "fmt" "net/url" @@ -10,12 +9,13 @@ import ( "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/webrtc" + "github.com/pion/rtp" pion "github.com/pion/webrtc/v4" ) type Client struct { api *TuyaClient - prod core.Producer + conn *webrtc.Conn done chan struct{} } @@ -95,7 +95,7 @@ func Dial(rawURL string) (core.Producer, error) { conf := pion.Configuration{ ICEServers: client.api.iceServers, ICETransportPolicy: pion.ICETransportPolicyAll, - BundlePolicy: pion.BundlePolicyMaxBundle, + BundlePolicy: pion.BundlePolicyBalanced, } api, err := webrtc.NewAPI() @@ -119,16 +119,16 @@ func Dial(rawURL string) (core.Producer, error) { // waiter will wait PC error or WS error or nil (connection OK) var connState core.Waiter - prod := webrtc.NewConn(pc) - prod.FormatName = "tuya/webrtc" - prod.Mode = core.ModeActiveProducer - prod.Protocol = "mqtt" - prod.URL = rawURL - - client.prod = prod + client.conn = webrtc.NewConn(pc) + client.conn.FormatName = "tuya/webrtc" + client.conn.Mode = core.ModeActiveProducer + client.conn.Protocol = "mqtt" + client.conn.URL = rawURL // Set up MQTT handlers client.api.mqtt.handleAnswer = func(answer AnswerFrame) { + // fmt.Printf("tuya: answer: %s\n", answer.Sdp) + desc := pion.SessionDescription{ Type: pion.SDPTypePranswer, SDP: answer.Sdp, @@ -139,17 +139,17 @@ func Dial(rawURL string) (core.Producer, error) { return } - if err = prod.SetAnswer(answer.Sdp); err != nil { + if err = client.conn.SetAnswer(answer.Sdp); err != nil { client.Stop() return } - prod.SDP = answer.Sdp + client.conn.SDP = answer.Sdp } client.api.mqtt.handleCandidate = func(candidate CandidateFrame) { if candidate.Candidate != "" { - prod.AddCandidate(candidate.Candidate) + client.conn.AddCandidate(candidate.Candidate) if err != nil { client.Stop() } @@ -165,7 +165,7 @@ func Dial(rawURL string) (core.Producer, error) { client.Stop() } - prod.Listen(func(msg any) { + client.conn.Listen(func(msg any) { switch msg := msg.(type) { case *pion.ICECandidate: _ = sendOffer.Wait() @@ -186,13 +186,14 @@ func Dial(rawURL string) (core.Producer, error) { }) // Create offer - offer, err := prod.CreateOffer(client.api.medias) + offer, err := client.conn.CreateOffer(client.api.medias) if err != nil { client.api.Close() return nil, err } // horter sdp, remove a=extmap... line, device ONLY allow 8KB json payload + // https://github.com/tuya/webrtc-demo-go/blob/04575054f18ccccb6bc9d82939dd46d449544e20/static/js/main.js#L224 re := regexp.MustCompile(`\r\na=extmap[^\r\n]*`) offer = re.ReplaceAllString(offer, "") @@ -209,23 +210,38 @@ func Dial(rawURL string) (core.Producer, error) { } func (c *Client) GetMedias() []*core.Media { - return c.prod.GetMedias() + return c.conn.GetMedias() } func (c *Client) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, error) { - return c.prod.GetTrack(media, codec) + return c.conn.GetTrack(media, codec) } func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiver) error { - if prod, ok := c.prod.(*webrtc.Conn); ok { - return prod.AddTrack(media, codec, track) + // RepackG711 will not work, so add default logic without repacking + + payloadType := codec.PayloadType + + localTrack := c.conn.GetSenderTrack(media.ID) + if localTrack == nil { + return errors.New("webrtc: can't get track") } + sender := core.NewSender(media, codec) + sender.Handler = func(packet *rtp.Packet) { + c.conn.Send += packet.MarshalSize() + //important to send with remote PayloadType + _ = localTrack.WriteRTP(payloadType, packet) + } + + sender.HandleRTP(track) + c.conn.Senders = append(c.conn.Senders, sender) + return nil } func (c *Client) Start() error { - return c.prod.Start() + return c.conn.Start() } func (c *Client) Stop() error { @@ -236,8 +252,8 @@ func (c *Client) Stop() error { close(c.done) } - if c.prod != nil { - _ = c.prod.Stop() + if c.conn != nil { + _ = c.conn.Stop() } if c.api != nil { @@ -248,9 +264,5 @@ func (c *Client) Stop() error { } func (c *Client) MarshalJSON() ([]byte, error) { - if webrtcProd, ok := c.prod.(*webrtc.Conn); ok { - return webrtcProd.MarshalJSON() - } - - return json.Marshal(c.prod) + return c.conn.MarshalJSON() } diff --git a/pkg/webrtc/conn.go b/pkg/webrtc/conn.go index 092b05c8..f853bf43 100644 --- a/pkg/webrtc/conn.go +++ b/pkg/webrtc/conn.go @@ -161,16 +161,7 @@ func (c *Conn) AddCandidate(candidate string) error { return c.pc.AddICECandidate(webrtc.ICECandidateInit{Candidate: candidate}) } -func (c *Conn) getTranseiver(mid string) *webrtc.RTPTransceiver { - for _, tr := range c.pc.GetTransceivers() { - if tr.Mid() == mid { - return tr - } - } - return nil -} - -func (c *Conn) getSenderTrack(mid string) *Track { +func (c *Conn) GetSenderTrack(mid string) *Track { if tr := c.getTranseiver(mid); tr != nil { if s := tr.Sender(); s != nil { if t := s.Track().(*Track); t != nil { @@ -181,6 +172,15 @@ func (c *Conn) getSenderTrack(mid string) *Track { return nil } +func (c *Conn) getTranseiver(mid string) *webrtc.RTPTransceiver { + for _, tr := range c.pc.GetTransceivers() { + if tr.Mid() == mid { + return tr + } + } + return nil +} + func (c *Conn) getMediaCodec(remote *webrtc.TrackRemote) (*core.Media, *core.Codec) { for _, tr := range c.pc.GetTransceivers() { // search Transeiver for this TrackRemote @@ -209,7 +209,7 @@ func (c *Conn) getMediaCodec(remote *webrtc.TrackRemote) (*core.Media, *core.Cod // check GetTrack panic(core.Caller()) - return nil, nil + // return nil, nil } func sanitizeIP6(host string) string { diff --git a/pkg/webrtc/consumer.go b/pkg/webrtc/consumer.go index ebc3a008..767394df 100644 --- a/pkg/webrtc/consumer.go +++ b/pkg/webrtc/consumer.go @@ -32,7 +32,7 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv panic(core.Caller()) } - localTrack := c.getSenderTrack(media.ID) + localTrack := c.GetSenderTrack(media.ID) if localTrack == nil { return errors.New("webrtc: can't get track") } From 5ec942cb5ea46d0f005e969fc99f1611394ae135 Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 13 May 2025 15:09:27 +0200 Subject: [PATCH 12/60] fix stream mode --- pkg/tuya/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 586a8bbd..8a0cc221 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -71,7 +71,7 @@ func Dial(rawURL string) (core.Producer, error) { } // Initialize Tuya API client - tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientId, clientSecret, streamType) + tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientId, clientSecret, streamMode) if err != nil { return nil, err } From e7bd3d401f8473771866ec8d2f068fb083732586 Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 13 May 2025 22:25:35 +0200 Subject: [PATCH 13/60] wip h265 datachannel --- pkg/tuya/api.go | 96 +++++++++++++++++++++++++++---------------- pkg/tuya/client.go | 4 +- pkg/tuya/mqtt.go | 100 ++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 154 insertions(+), 46 deletions(-) diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index 2f7cd194..13e8265e 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -76,10 +76,10 @@ type Skill struct { SampleRate int `json:"sampleRate"` } `json:"audios"` Videos []struct { - StreamType int `json:"streamType"` // streamType = 2 => main stream - streamType = 4 => sub stream + StreamType int `json:"streamType"` // 2 = main stream, 4 = sub stream ProfileId string `json:"profileId"` Width int `json:"width"` - CodecType int `json:"codecType"` + CodecType int `json:"codecType"` // 2 = H264, 4 = H265 SampleRate int `json:"sampleRate"` Height int `json:"height"` } `json:"videos"` @@ -325,24 +325,24 @@ func (c *TuyaClient) InitDevice() (err error) { c.medias = make([]*core.Media, 0) if len(c.skill.Audios) > 0 { - // Use the first Audio-Codec - audio := c.skill.Audios[0] - direction := core.DirectionRecvonly if c.hasBackchannel { direction = core.DirectionSendRecv } + codecs := make([]*core.Codec, 0) + for _, audio := range c.skill.Audios { + codecs = append(codecs, &core.Codec{ + Name: getAudioCodec(audio.CodecType), + ClockRate: uint32(audio.SampleRate), + Channels: uint8(audio.Channels), + }) + } + c.medias = append(c.medias, &core.Media{ Kind: core.KindAudio, Direction: direction, - Codecs: []*core.Codec{ - { - Name: "PCMU", - ClockRate: uint32(audio.SampleRate), - Channels: uint8(audio.Channels), - }, - }, + Codecs: codecs, }) } else { // Use default values for Audio @@ -351,7 +351,7 @@ func (c *TuyaClient) InitDevice() (err error) { Direction: core.DirectionRecvonly, Codecs: []*core.Codec{ { - Name: "PCMU", + Name: core.CodecPCMU, ClockRate: uint32(8000), Channels: uint8(1), }, @@ -360,24 +360,27 @@ func (c *TuyaClient) InitDevice() (err error) { } if len(c.skill.Videos) > 0 { - // Use the first Video-Codec - video := c.skill.Videos[0] + codecs := make([]*core.Codec, 0) + for _, video := range c.skill.Videos { + if video.CodecType == 2 { + codecs = append(codecs, &core.Codec{ + Name: core.CodecH264, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }) + } else if video.CodecType == 4 { + codecs = append(codecs, &core.Codec{ + Name: core.CodecH265, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }) + } + } c.medias = append(c.medias, &core.Media{ Kind: core.KindVideo, Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: core.CodecH265, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, - }, - { - Name: core.CodecH264, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, - }, - }, + Codecs: codecs, }) } else { // Use default values for Video @@ -469,19 +472,19 @@ func (c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { return &openIoTHubConfigResponse.Result, nil } -func (c *TuyaClient) getStreamType(streamChoice string) uint32 { +func (c *TuyaClient) getStreamType(streamChoice string) int { // Default streamType if nothing is found - defaultStreamType := uint32(1) + defaultStreamType := 1 if c.skill == nil || len(c.skill.Videos) == 0 { return defaultStreamType } // Find the highest and lowest resolution - var highestResType uint32 = defaultStreamType - var highestRes int = 0 - var lowestResType uint32 = defaultStreamType - var lowestRes int = 0 + var highestResType = defaultStreamType + var highestRes = 0 + var lowestResType = defaultStreamType + var lowestRes = 0 for _, video := range c.skill.Videos { res := video.Width * video.Height @@ -489,13 +492,13 @@ func (c *TuyaClient) getStreamType(streamChoice string) uint32 { // Highest Resolution if res > highestRes { highestRes = res - highestResType = uint32(video.StreamType) + highestResType = video.StreamType } // Lower Resolution (or first if not set yet) if lowestRes == 0 || res < lowestRes { lowestRes = res - lowestResType = uint32(video.StreamType) + lowestResType = video.StreamType } } @@ -510,6 +513,29 @@ func (c *TuyaClient) getStreamType(streamChoice string) uint32 { } } +func getAudioCodec(codecType int) string { + switch codecType { + // case 100: + // return "ADPCM" + case 101: + return core.CodecPCM + case 102, 103, 104: + return core.CodecAAC + case 105: + return core.CodecPCMU + case 106: + return core.CodecPCMA + // case 107: + // return "G726-32" + // case 108: + // return "SPEEX" + case 109: + return core.CodecMP3 + default: + return core.CodecPCMU + } +} + func (c *TuyaClient) calBusinessSign(ts int64) string { data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts) val := md5.Sum([]byte(data)) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 8a0cc221..3fb7bf07 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -95,7 +95,7 @@ func Dial(rawURL string) (core.Producer, error) { conf := pion.Configuration{ ICEServers: client.api.iceServers, ICETransportPolicy: pion.ICETransportPolicyAll, - BundlePolicy: pion.BundlePolicyBalanced, + BundlePolicy: pion.BundlePolicyMaxBundle, } api, err := webrtc.NewAPI() @@ -148,6 +148,8 @@ func Dial(rawURL string) (core.Producer, error) { } client.api.mqtt.handleCandidate = func(candidate CandidateFrame) { + // fmt.Printf("tuya: candidate: %s\n", candidate.Candidate) + if candidate.Candidate != "" { client.conn.AddCandidate(candidate.Candidate) if err != nil { diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index bf9badd7..eb1c719e 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -39,10 +39,11 @@ type MqttFrame struct { } type OfferFrame struct { - Mode string `json:"mode"` - Sdp string `json:"sdp"` - StreamType uint32 `json:"stream_type"` - Auth string `json:"auth"` + Mode string `json:"mode"` + Sdp string `json:"sdp"` + StreamType int `json:"stream_type"` + Auth string `json:"auth"` + DatachannelEnable bool `json:"datachannel_enable"` } type AnswerFrame struct { @@ -57,7 +58,12 @@ type CandidateFrame struct { type ResolutionFrame struct { Mode string `json:"mode"` - Value int `json:"value"` + Value int `json:"value"` // 0: HD, 1: SD +} + +type SpeakerFrame struct { + Mode string `json:"mode"` + Value int `json:"value"` // 0: off, 1: on } type DisconnectFrame struct { @@ -202,12 +208,61 @@ func (c *TuyaMQTT) onError(err error) { } } -func (c *TuyaClient) sendOffer(sdp string, streamType uint32) { +func (c *TuyaClient) sendOffer(sdp string, streamType int) { + // H265 is currently not supported because Tuya does not send H265 data, and therefore also no audio over the normal WebRTC connection. + // The WebRTC connection is used only for sending audio back to the device (backchannel). + // Tuya expects a separate WebRTC DataChannel for H265 data and sends the H265 video and audio data packaged as fMP4 data back. + // These must then be processed separately (WIP - Work In Progress) + + // Example Answer (H265/PCMU with backchannel): + + /* + v=0 + o=- 1747174385 1 IN IP4 127.0.0.1 + s=- + t=0 0 + a=group:BUNDLE 0 1 + a=msid-semantic: WMS UMSklk + m=audio 9 UDP/TLS/RTP/SAVPF 0 + c=IN IP4 0.0.0.0 + a=rtcp:9 IN IP4 0.0.0.0 + a=ice-ufrag:zuRr + a=ice-pwd:EDeWXz847P810fyDyKxbmTdX + a=ice-options:trickle + a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14 + a=setup:active + a=mid:0 + a=sendrecv + a=msid:UMSklk NiNNboEn1rJWoQYtpguoKr1GBwpvPST + a=rtcp-mux + a=rtpmap:0 PCMU/8000 + a=ssrc:832759612 cname:bfa87264438073154dhdek + m=video 9 UDP/TLS/RTP/SAVPF 0 + c=IN IP4 0.0.0.0 + a=rtcp:9 IN IP4 0.0.0.0 + a=ice-ufrag:zuRr + a=ice-pwd:EDeWXz847P810fyDyKxbmTdX + a=ice-options:trickle + a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14 + a=setup:active + a=mid:1 + a=sendonly + a=msid:UMSklk l9o6icIVb7n7vDdp0KhocYnsijhd774 + a=rtcp-mux + a=rtpmap:0 /0 + a=rtcp-fb:0 ccm fir + a=rtcp-fb:0 nack + a=rtcp-fb:0 nack pli + a=fmtp:0 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id= + a=ssrc:0 cname:bfa87264438073154dhdek + */ + c.sendMqttMessage("offer", 302, "", OfferFrame{ - Mode: "webrtc", - Sdp: sdp, - StreamType: streamType, - Auth: c.auth, + Mode: "webrtc", + Sdp: sdp, + StreamType: streamType, + Auth: c.auth, + DatachannelEnable: c.isHEVC(streamType), }) } @@ -219,12 +274,23 @@ func (c *TuyaClient) sendCandidate(candidate string) { } func (c *TuyaClient) sendResolution(resolution int) { + if !c.isClaritySupported(resolution) { + return + } + c.sendMqttMessage("resolution", 302, "", ResolutionFrame{ Mode: "webrtc", Value: resolution, }) } +func (c *TuyaClient) sendSpeaker(speaker int) { + c.sendMqttMessage("speaker", 302, "", SpeakerFrame{ + Mode: "webrtc", + Value: speaker, + }) +} + func (c *TuyaClient) sendDisconnect() { c.sendMqttMessage("disconnect", 302, "", DisconnectFrame{ Mode: "webrtc", @@ -271,3 +337,17 @@ func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transacti c.mqtt.onError(fmt.Errorf("mqtt publish fail: %s, topic: %s", token.Error().Error(), c.mqtt.publishTopic)) } } + +func (c *TuyaClient) isHEVC(streamType int) bool { + for _, video := range c.skill.Videos { + if video.StreamType == streamType { + return video.CodecType == 4 + } + } + + return false +} + +func (c *TuyaClient) isClaritySupported(webrtcValue int) bool { + return (webrtcValue & (1 << 5)) != 0 +} From 499dc103901914a4d3fbddd293e836cb448cf026 Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 13 May 2025 22:41:07 +0200 Subject: [PATCH 14/60] comments --- pkg/tuya/mqtt.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index eb1c719e..88f3a199 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -209,11 +209,18 @@ func (c *TuyaMQTT) onError(err error) { } func (c *TuyaClient) sendOffer(sdp string, streamType int) { + // Note: // H265 is currently not supported because Tuya does not send H265 data, and therefore also no audio over the normal WebRTC connection. // The WebRTC connection is used only for sending audio back to the device (backchannel). // Tuya expects a separate WebRTC DataChannel for H265 data and sends the H265 video and audio data packaged as fMP4 data back. // These must then be processed separately (WIP - Work In Progress) + // Note 2: + // Even if we don't receive any data, the peer connection is correctly established and connected + + // Note 3: + // It seems that if even one stream is HEVC, we also need to use the datachannel for the substream, even if that substream is using H264. + // Example Answer (H265/PCMU with backchannel): /* From 524cdb7176e1cc1e665ce137029664a5958bdb1d Mon Sep 17 00:00:00 2001 From: seydx Date: Thu, 15 May 2025 10:42:34 +0200 Subject: [PATCH 15/60] demuxer: support hvc and pcm --- pkg/iso/reader.go | 14 +++- pkg/mp4/demuxer.go | 195 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 206 insertions(+), 3 deletions(-) diff --git a/pkg/iso/reader.go b/pkg/iso/reader.go index 175e2563..501a4eac 100644 --- a/pkg/iso/reader.go +++ b/pkg/iso/reader.go @@ -86,7 +86,7 @@ func DecodeAtom(b []byte) (any, error) { return DecodeAtom(data[1+3+4:]) } - case "avc1", "hev1": + case "avc1", "hev1", "hvc1": b = data[6+2+2+2+4+4+4+2+2+4+4+4+2+32+2+2:] atom, err := DecodeAtom(b) if err != nil { @@ -141,7 +141,17 @@ func DecodeAtom(b []byte) (any, error) { return atom, nil case MoofTrafTfdt: - return &AtomTfdt{DecodeTime: binary.BigEndian.Uint64(data[4:])}, nil + // Check version to determine field size + version := data[0] // First byte is version + if version == 0 { + // Version 0 uses 32-bit time + decodeTime := uint64(binary.BigEndian.Uint32(data[4:])) + return &AtomTfdt{DecodeTime: decodeTime}, nil + } else { + // Version 1 uses 64-bit time + decodeTime := binary.BigEndian.Uint64(data[4:]) + return &AtomTfdt{DecodeTime: decodeTime}, nil + } case MoofTrafTrun: rd := bits.NewReader(data) diff --git a/pkg/mp4/demuxer.go b/pkg/mp4/demuxer.go index 25c8c70e..67da93de 100644 --- a/pkg/mp4/demuxer.go +++ b/pkg/mp4/demuxer.go @@ -1,9 +1,12 @@ package mp4 import ( + "fmt" + "github.com/AlexxIT/go2rtc/pkg/aac" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/h264" + "github.com/AlexxIT/go2rtc/pkg/h265" "github.com/AlexxIT/go2rtc/pkg/iso" "github.com/pion/rtp" ) @@ -13,6 +16,16 @@ type Demuxer struct { timeScales map[uint32]float32 } +type TrackPackets struct { + TrackID uint32 + Packets []*core.Packet +} + +type TrackData struct { + DecodeTime uint32 + Trun *iso.AtomTrun +} + func (d *Demuxer) Probe(init []byte) (medias []*core.Media) { var trackID, timeScale uint32 @@ -34,11 +47,23 @@ func (d *Demuxer) Probe(init []byte) (medias []*core.Media) { switch atom.Name { case "avc1": codec = h264.ConfigToCodec(atom.Config) + case "hvc1", "hev1": + codec = h265.ConfigToCodec(atom.Config) } case *iso.AtomAudio: switch atom.Name { case "mp4a": - codec = aac.ConfigToCodec(atom.Config) + // G.711 PCMU audio detection for 8kHz mono (Tuya...) + if atom.SampleRate == 8000 && atom.Channels == 1 { + codec = &core.Codec{ + Name: core.CodecPCMU, + ClockRate: 8000, + Channels: 1, + PayloadType: 0, + } + } else { + codec = aac.ConfigToCodec(atom.Config) + } } } @@ -47,6 +72,7 @@ func (d *Demuxer) Probe(init []byte) (medias []*core.Media) { d.timeScales[trackID] = float32(codec.ClockRate) / float32(timeScale) medias = append(medias, &core.Media{ + ID: fmt.Sprintf("trackID=%d", trackID), Kind: codec.Kind(), Direction: core.DirectionRecvonly, Codecs: []*core.Codec{codec}, @@ -114,3 +140,170 @@ func (d *Demuxer) Demux(data2 []byte) (trackID uint32, packets []*core.Packet) { return } + +// DemuxAll returns packets from all tracks found in the fragment +func (d *Demuxer) DemuxAll(data []byte) []TrackPackets { + atoms, err := iso.DecodeAtoms(data) + if err != nil { + return nil + } + + // Map to store track-specific data + trackData := make(map[uint32]TrackData) + var mdat []byte + + // First pass: collect all track data + for _, atom := range atoms { + switch atom := atom.(type) { + case *iso.AtomMdat: + mdat = atom.Data + } + } + + // Temporary variables to track current track ID while parsing + var currentTrackID uint32 + + // Second pass: process traf boxes + for _, atom := range atoms { + switch atom := atom.(type) { + case *iso.AtomTfhd: + currentTrackID = atom.TrackID + + // Initialize track data if not exists + if _, ok := trackData[currentTrackID]; !ok { + trackData[currentTrackID] = TrackData{} + } + + case *iso.AtomTfdt: + if currentTrackID != 0 { + td := trackData[currentTrackID] + td.DecodeTime = uint32(atom.DecodeTime) + trackData[currentTrackID] = td + } + + case *iso.AtomTrun: + if currentTrackID != 0 { + td := trackData[currentTrackID] + td.Trun = atom + trackData[currentTrackID] = td + } + } + } + + // Process all tracks and collect results + var results []TrackPackets + + for tid, td := range trackData { + if td.Trun == nil || mdat == nil || len(td.Trun.SamplesSize) == 0 { + continue + } + + codec := d.codecs[tid] + if codec == nil { + continue + } + + timeScale := d.timeScales[tid] + + var packets []*core.Packet + switch codec.Kind() { + case "video": + packets = createVideoPackets(td.Trun, mdat, td.DecodeTime, timeScale) + case "audio": + packets = createAudioPackets(td.Trun, mdat, td.DecodeTime, timeScale, codec) + } + + if len(packets) > 0 { + results = append(results, TrackPackets{ + TrackID: tid, + Packets: packets, + }) + } + } + + return results +} + +// Creates video packets (H.264/H.265) +func createVideoPackets(trun *iso.AtomTrun, mdat []byte, decodeTime uint32, timeScale float32) []*core.Packet { + n := len(trun.SamplesSize) + hasDurations := len(trun.SamplesDuration) > 0 + + packets := make([]*core.Packet, n) + offset := uint32(0) + ts := decodeTime + + for i := 0; i < n; i++ { + // Get duration from array or use default + var duration uint32 + if hasDurations && i < len(trun.SamplesDuration) { + duration = trun.SamplesDuration[i] + } else { + duration = 1000 // Default for video + } + + size := trun.SamplesSize[i] + + if offset+size > uint32(len(mdat)) { + return packets[:i] + } + + timestamp := uint32(float32(ts) * timeScale) + packets[i] = &rtp.Packet{ + Header: rtp.Header{Timestamp: timestamp}, + Payload: mdat[offset : offset+size], + } + + offset += size + ts += duration + } + + return packets +} + +// Creates audio packets (G.711, AAC, etc.) +func createAudioPackets(trun *iso.AtomTrun, mdat []byte, decodeTime uint32, timeScale float32, codec *core.Codec) []*core.Packet { + n := len(trun.SamplesSize) + hasDurations := len(trun.SamplesDuration) > 0 + + packets := make([]*core.Packet, n) + offset := uint32(0) + ts := decodeTime + isPCM := codec.Name == core.CodecPCMU || codec.Name == core.CodecPCMA || codec.Name == core.CodecPCM || codec.Name == core.CodecPCML + + for i := 0; i < n; i++ { + size := trun.SamplesSize[i] + + // Calculate duration based on codec + var duration uint32 + if hasDurations && i < len(trun.SamplesDuration) { + duration = trun.SamplesDuration[i] + } else if isPCM { + duration = size + } else { + duration = 1024 + } + + if offset+size > uint32(len(mdat)) { + return packets[:i] + } + + // Calculate timestamp based on codec + var timestamp uint32 + if isPCM { + timestamp = ts + } else { + timestamp = uint32(float32(ts) * timeScale) + } + + packets[i] = &rtp.Packet{ + Header: rtp.Header{Timestamp: timestamp}, + Payload: mdat[offset : offset+size], + } + + offset += size + ts += duration + } + + return packets +} From 3d222136f95c4de1f219f4db1c7a95ba57720198 Mon Sep 17 00:00:00 2001 From: seydx Date: Thu, 15 May 2025 10:45:04 +0200 Subject: [PATCH 16/60] support h265 --- pkg/tuya/api.go | 70 ++++++----- pkg/tuya/client.go | 45 +++++-- pkg/tuya/dc.go | 253 ++++++++++++++++++++++++++++++++++++++++ pkg/tuya/frameBuffer.go | 86 ++++++++++++++ pkg/tuya/mqtt.go | 83 +++---------- 5 files changed, 428 insertions(+), 109 deletions(-) create mode 100644 pkg/tuya/dc.go create mode 100644 pkg/tuya/frameBuffer.go diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index 13e8265e..6ffa3dd7 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -67,22 +67,26 @@ type P2PConfig struct { Ices []OpenApiICE `json:"ices"` } +type AudioSkill struct { + Channels int `json:"channels"` + DataBit int `json:"dataBit"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` +} + +type VideoSkill struct { + StreamType int `json:"streamType"` // 2 = main stream, 4 = sub stream + ProfileId string `json:"profileId,omitempty"` + CodecType int `json:"codecType"` // 2 = H264, 4 = H265 + Width int `json:"width"` + Height int `json:"height"` + SampleRate int `json:"sampleRate"` +} + type Skill struct { - WebRTC int `json:"webrtc"` - Audios []struct { - Channels int `json:"channels"` - DataBit int `json:"dataBit"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` - } `json:"audios"` - Videos []struct { - StreamType int `json:"streamType"` // 2 = main stream, 4 = sub stream - ProfileId string `json:"profileId"` - Width int `json:"width"` - CodecType int `json:"codecType"` // 2 = H264, 4 = H265 - SampleRate int `json:"sampleRate"` - Height int `json:"height"` - } `json:"videos"` + WebRTC int `json:"webrtc"` + Audios []AudioSkill `json:"audios"` + Videos []VideoSkill `json:"videos"` } type WebRTConfig struct { @@ -299,20 +303,9 @@ func (c *TuyaClient) InitDevice() (err error) { c.auth = webRTCConfigResponse.Result.Auth c.skill = &Skill{ - Audios: []struct { - Channels int `json:"channels"` - DataBit int `json:"dataBit"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` - }{}, - Videos: []struct { - StreamType int `json:"streamType"` - ProfileId string `json:"profileId"` - Width int `json:"width"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` - Height int `json:"height"` - }{}, + WebRTC: 3, // basic webrtc + Audios: make([]AudioSkill, 0), + Videos: make([]VideoSkill, 0), } if webRTCConfigResponse.Result.Skill != "" { @@ -393,6 +386,11 @@ func (c *TuyaClient) InitDevice() (err error) { ClockRate: uint32(90000), PayloadType: 96, }, + { + Name: core.CodecH265, + ClockRate: uint32(90000), + PayloadType: 96, + }, }, }) } @@ -536,6 +534,20 @@ func getAudioCodec(codecType int) string { } } +func (c *TuyaClient) isHEVC(streamType int) bool { + for _, video := range c.skill.Videos { + if video.StreamType == streamType { + return video.CodecType == 4 + } + } + + return false +} + +func (c *TuyaClient) isClaritySupported(webrtcValue int) bool { + return (webrtcValue & (1 << 5)) != 0 +} + func (c *TuyaClient) calBusinessSign(ts int64) string { data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts) val := md5.Sum([]byte(data)) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 3fb7bf07..37697310 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -14,9 +14,10 @@ import ( ) type Client struct { - api *TuyaClient - conn *webrtc.Conn - done chan struct{} + api *TuyaClient + conn *webrtc.Conn + dcConn *DCConn + done chan struct{} } const ( @@ -92,6 +93,8 @@ func Dial(rawURL string) (core.Producer, error) { } return streams.GetProducer(client.api.hlsURL) } else { + isHEVC := client.api.isHEVC(client.api.getStreamType(streamType)) + conf := pion.Configuration{ ICEServers: client.api.iceServers, ICETransportPolicy: pion.ICETransportPolicyAll, @@ -116,7 +119,7 @@ func Dial(rawURL string) (core.Producer, error) { // protect from blocking on errors defer sendOffer.Done(nil) - // waiter will wait PC error or WS error or nil (connection OK) + // waiter will wait PC error var connState core.Waiter client.conn = webrtc.NewConn(pc) @@ -167,6 +170,15 @@ func Dial(rawURL string) (core.Producer, error) { client.Stop() } + // Set up data channel for HEVC + if isHEVC { + client.dcConn, err = NewDCConn(pc, client) + if err != nil { + client.api.Close() + return nil, err + } + } + client.conn.Listen(func(msg any) { switch msg := msg.(type) { case *pion.ICECandidate: @@ -182,6 +194,7 @@ func Dial(rawURL string) (core.Producer, error) { case pion.PeerConnectionStateConnected: connState.Done(nil) default: + client.Stop() connState.Done(errors.New("webrtc: " + msg.String())) } } @@ -200,9 +213,18 @@ func Dial(rawURL string) (core.Producer, error) { offer = re.ReplaceAllString(offer, "") // Send offer - client.api.sendOffer(offer, tuyaAPI.getStreamType(streamType)) + client.api.sendOffer(offer, streamType) sendOffer.Done(nil) + if client.dcConn != nil { + if err = client.dcConn.connected.Wait(); err != nil { + client.Stop() + return nil, err + } + + return client.dcConn, nil + } + if err = connState.Wait(); err != nil { return nil, err } @@ -223,12 +245,7 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece // RepackG711 will not work, so add default logic without repacking payloadType := codec.PayloadType - localTrack := c.conn.GetSenderTrack(media.ID) - if localTrack == nil { - return errors.New("webrtc: can't get track") - } - sender := core.NewSender(media, codec) sender.Handler = func(packet *rtp.Packet) { c.conn.Send += packet.MarshalSize() @@ -243,6 +260,10 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece } func (c *Client) Start() error { + if c.dcConn != nil { + c.dcConn.Start() + } + return c.conn.Start() } @@ -258,6 +279,10 @@ func (c *Client) Stop() error { _ = c.conn.Stop() } + if c.dcConn != nil { + _ = c.dcConn.Stop() + } + if c.api != nil { c.api.Close() } diff --git a/pkg/tuya/dc.go b/pkg/tuya/dc.go new file mode 100644 index 00000000..2ccd1313 --- /dev/null +++ b/pkg/tuya/dc.go @@ -0,0 +1,253 @@ +package tuya + +import ( + "encoding/json" + "errors" + + "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/mp4" + "github.com/pion/rtp" + pion "github.com/pion/webrtc/v4" +) + +type DCConn struct { + core.Connection + + client *Client + dc *pion.DataChannel + dem *mp4.Demuxer + queue *FrameBufferQueue + msgs chan pion.DataChannelMessage + connected core.Waiter + closed core.Waiter + initialized bool +} + +type DataChannelMessage struct { + Type string `json:"type"` + Msg string `json:"msg"` +} + +func NewDCConn(pc *pion.PeerConnection, c *Client) (*DCConn, error) { + maxRetransmits := uint16(5) + ordered := true + dc, err := pc.CreateDataChannel("fmp4Stream", &pion.DataChannelInit{ + MaxRetransmits: &maxRetransmits, + Ordered: &ordered, + }) + + if err != nil { + return nil, err + } + + conn := &DCConn{ + Connection: core.Connection{ + ID: core.NewID(), + FormatName: "webrtc/fmp4", + Transport: dc, + }, + client: c, + dc: dc, + dem: &mp4.Demuxer{}, + queue: NewFrameBufferQueue(), + msgs: make(chan pion.DataChannelMessage, 10), // Saw max 4 messages in a row, 10 should be enough + initialized: false, + } + + dc.OnMessage(func(msg pion.DataChannelMessage) { + conn.msgs <- msg + }) + + dc.OnError(func(err error) { + conn.connected.Done(err) + }) + + dc.OnClose(func() { + close(conn.msgs) + conn.connected.Done(errors.New("datachannel: closed")) + }) + + go conn.initializationLoop() + + return conn, nil +} + +func (c *DCConn) initializationLoop() { + for msg := range c.msgs { + if c.initialized { + return + } + + err := c.probe(msg) + if err != nil { + c.connected.Done(err) + return + } + + if c.initialized { + c.connected.Done(nil) + return + } + } +} + +func (c *DCConn) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, error) { + if media.Direction == core.DirectionSendRecv || media.Direction == core.DirectionSendonly { + return c.client.GetTrack(media, codec) + } + + for _, receiver := range c.Receivers { + if receiver.Codec == codec { + return receiver, nil + } + } + receiver := core.NewReceiver(media, codec) + c.Receivers = append(c.Receivers, receiver) + return receiver, nil +} + +func (c *DCConn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiver) error { + payloadType := codec.PayloadType + localTrack := c.client.conn.GetSenderTrack(media.ID) + sender := core.NewSender(media, codec) + sender.Handler = func(packet *rtp.Packet) { + c.Send += packet.MarshalSize() + //important to send with remote PayloadType + _ = localTrack.WriteRTP(payloadType, packet) + } + + sender.HandleRTP(track) + c.Senders = append(c.Senders, sender) + + return nil +} + +func (c *DCConn) Start() error { + receivers := make(map[uint32]*core.Receiver) + for _, receiver := range c.Receivers { + trackID := c.dem.GetTrackID(receiver.Codec) + receivers[trackID] = receiver + } + + ch := make(chan []byte, 10) + defer close(ch) + + go func() { + for data := range ch { + allTracks := c.dem.DemuxAll(data) + for _, trackData := range allTracks { + trackID := trackData.TrackID + packets := trackData.Packets + receiver := receivers[trackID] + if receiver == nil { + continue + } + + for _, packet := range packets { + receiver.WriteRTP(packet) + } + } + } + }() + + go func() { + for msg := range c.msgs { + if len(msg.Data) >= 4 { + segmentNum := int(msg.Data[1]) + fragmentCount := int(msg.Data[2]) + fragmentSeq := int(msg.Data[3]) + mp4Data := msg.Data[4:] + + c.queue.AddFragment(segmentNum, fragmentCount, fragmentSeq, mp4Data) + + if c.queue.IsSegmentComplete(segmentNum, fragmentCount) { + b := c.queue.GetCombinedBuffer(segmentNum) + c.Recv += len(b) + ch <- b + } + } + } + }() + + c.closed.Wait() + return nil +} + +func (c *DCConn) sendMessageToDataChannel(message string) error { + if c.dc != nil { + return c.dc.SendText(message) + } + + return nil +} + +func (c *DCConn) probe(msg pion.DataChannelMessage) (err error) { + if msg.IsString { + var message DataChannelMessage + if err = json.Unmarshal(msg.Data, &message); err != nil { + return err + } + + switch message.Type { + case "codec": + response, _ := json.Marshal(DataChannelMessage{ + Type: "start", + Msg: "fmp4", + }) + + err = c.sendMessageToDataChannel(string(response)) + if err != nil { + return err + } + + case "recv": + response, _ := json.Marshal(DataChannelMessage{ + Type: "complete", + Msg: "", + }) + + err = c.sendMessageToDataChannel(string(response)) + if err != nil { + return err + } + } + + } else { + if len(msg.Data) >= 4 { + messageType := msg.Data[0] + segmentNum := int(msg.Data[1]) + fragmentCount := int(msg.Data[2]) + fragmentSeq := int(msg.Data[3]) + mp4Data := msg.Data[4:] + + // initialization segment + if messageType == 0 && segmentNum == 1 && fragmentCount == 1 && fragmentSeq == 1 { + medias := c.dem.Probe(mp4Data) + c.Medias = append(c.Medias, medias...) + + // Add backchannel + webrtcMedias := c.client.GetMedias() + for _, media := range webrtcMedias { + if media.Kind == core.KindAudio { + if media.Direction == core.DirectionSendRecv || media.Direction == core.DirectionSendonly { + c.Medias = append(c.Medias, media) + } + } + } + + c.initialized = true + } + } + } + + return nil +} + +func (c *DCConn) Stop() error { + if c.dc != nil && c.dc.ReadyState() == pion.DataChannelStateOpen { + _ = c.dc.Close() + } + + c.closed.Done(nil) + return nil +} diff --git a/pkg/tuya/frameBuffer.go b/pkg/tuya/frameBuffer.go new file mode 100644 index 00000000..bbcb4ff5 --- /dev/null +++ b/pkg/tuya/frameBuffer.go @@ -0,0 +1,86 @@ +package tuya + +import ( + "sort" + "sync" +) + +type FrameBufferQueue struct { + segments map[int]map[int][]byte // segNum -> fragSeq -> data + mu sync.Mutex +} + +func NewFrameBufferQueue() *FrameBufferQueue { + return &FrameBufferQueue{ + segments: make(map[int]map[int][]byte), + } +} + +func (q *FrameBufferQueue) AddFragment(segmentNum, fragmentCount, fragmentSeq int, data []byte) { + q.mu.Lock() + defer q.mu.Unlock() + + if _, ok := q.segments[segmentNum]; !ok { + q.segments[segmentNum] = make(map[int][]byte) + } + + q.segments[segmentNum][fragmentSeq] = data +} + +func (q *FrameBufferQueue) IsSegmentComplete(segmentNum, fragmentCount int) bool { + q.mu.Lock() + defer q.mu.Unlock() + + if frags, ok := q.segments[segmentNum]; ok { + // Make sure we have the right number of fragments + if len(frags) != fragmentCount { + return false + } + + // Check if we have all sequences from 1 to fragmentCount + for i := 1; i <= fragmentCount; i++ { + if _, ok := frags[i]; !ok { + return false + } + } + + return true + } + + return false +} + +func (q *FrameBufferQueue) GetCombinedBuffer(segNum int) []byte { + q.mu.Lock() + defer q.mu.Unlock() + + if frags, ok := q.segments[segNum]; ok { + // Sort fragments by sequence number + var keys []int + for k := range frags { + keys = append(keys, k) + } + sort.Ints(keys) + + // Calculate total size for pre-allocation + totalSize := 0 + for _, k := range keys { + totalSize += len(frags[k]) + } + + // Pre-allocate buffer for better performance + combined := make([]byte, 0, totalSize) + + // Combine fragments in sequence order + for _, k := range keys { + combined = append(combined, frags[k]...) + } + + // Remove this segment to free memory + delete(q.segments, segNum) + + return combined + } + + return nil +} diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index 88f3a199..9eb25b49 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -208,68 +208,25 @@ func (c *TuyaMQTT) onError(err error) { } } -func (c *TuyaClient) sendOffer(sdp string, streamType int) { - // Note: - // H265 is currently not supported because Tuya does not send H265 data, and therefore also no audio over the normal WebRTC connection. - // The WebRTC connection is used only for sending audio back to the device (backchannel). - // Tuya expects a separate WebRTC DataChannel for H265 data and sends the H265 video and audio data packaged as fMP4 data back. - // These must then be processed separately (WIP - Work In Progress) +func (c *TuyaClient) sendOffer(sdp string, streamType string) { + fixedStreamType := c.getStreamType(streamType) + isHEVC := c.isHEVC(fixedStreamType) - // Note 2: - // Even if we don't receive any data, the peer connection is correctly established and connected - - // Note 3: - // It seems that if even one stream is HEVC, we also need to use the datachannel for the substream, even if that substream is using H264. - - // Example Answer (H265/PCMU with backchannel): - - /* - v=0 - o=- 1747174385 1 IN IP4 127.0.0.1 - s=- - t=0 0 - a=group:BUNDLE 0 1 - a=msid-semantic: WMS UMSklk - m=audio 9 UDP/TLS/RTP/SAVPF 0 - c=IN IP4 0.0.0.0 - a=rtcp:9 IN IP4 0.0.0.0 - a=ice-ufrag:zuRr - a=ice-pwd:EDeWXz847P810fyDyKxbmTdX - a=ice-options:trickle - a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14 - a=setup:active - a=mid:0 - a=sendrecv - a=msid:UMSklk NiNNboEn1rJWoQYtpguoKr1GBwpvPST - a=rtcp-mux - a=rtpmap:0 PCMU/8000 - a=ssrc:832759612 cname:bfa87264438073154dhdek - m=video 9 UDP/TLS/RTP/SAVPF 0 - c=IN IP4 0.0.0.0 - a=rtcp:9 IN IP4 0.0.0.0 - a=ice-ufrag:zuRr - a=ice-pwd:EDeWXz847P810fyDyKxbmTdX - a=ice-options:trickle - a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14 - a=setup:active - a=mid:1 - a=sendonly - a=msid:UMSklk l9o6icIVb7n7vDdp0KhocYnsijhd774 - a=rtcp-mux - a=rtpmap:0 /0 - a=rtcp-fb:0 ccm fir - a=rtcp-fb:0 nack - a=rtcp-fb:0 nack pli - a=fmtp:0 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id= - a=ssrc:0 cname:bfa87264438073154dhdek - */ + if isHEVC { + // On HEVC we use streamType 0 for main stream and 1 for sub stream + if streamType == "main" { + fixedStreamType = 0 + } else { + fixedStreamType = 1 + } + } c.sendMqttMessage("offer", 302, "", OfferFrame{ Mode: "webrtc", Sdp: sdp, - StreamType: streamType, + StreamType: fixedStreamType, Auth: c.auth, - DatachannelEnable: c.isHEVC(streamType), + DatachannelEnable: isHEVC, }) } @@ -344,17 +301,3 @@ func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transacti c.mqtt.onError(fmt.Errorf("mqtt publish fail: %s, topic: %s", token.Error().Error(), c.mqtt.publishTopic)) } } - -func (c *TuyaClient) isHEVC(streamType int) bool { - for _, video := range c.skill.Videos { - if video.StreamType == streamType { - return video.CodecType == 4 - } - } - - return false -} - -func (c *TuyaClient) isClaritySupported(webrtcValue int) bool { - return (webrtcValue & (1 << 5)) != 0 -} From 27fe2622ec6c2bd20e141f61dbf6be76ee6973ab Mon Sep 17 00:00:00 2001 From: seydx Date: Sat, 17 May 2025 11:40:27 +0200 Subject: [PATCH 17/60] revert demuxer --- pkg/iso/reader.go | 14 +--- pkg/mp4/demuxer.go | 195 +-------------------------------------------- 2 files changed, 3 insertions(+), 206 deletions(-) diff --git a/pkg/iso/reader.go b/pkg/iso/reader.go index 501a4eac..175e2563 100644 --- a/pkg/iso/reader.go +++ b/pkg/iso/reader.go @@ -86,7 +86,7 @@ func DecodeAtom(b []byte) (any, error) { return DecodeAtom(data[1+3+4:]) } - case "avc1", "hev1", "hvc1": + case "avc1", "hev1": b = data[6+2+2+2+4+4+4+2+2+4+4+4+2+32+2+2:] atom, err := DecodeAtom(b) if err != nil { @@ -141,17 +141,7 @@ func DecodeAtom(b []byte) (any, error) { return atom, nil case MoofTrafTfdt: - // Check version to determine field size - version := data[0] // First byte is version - if version == 0 { - // Version 0 uses 32-bit time - decodeTime := uint64(binary.BigEndian.Uint32(data[4:])) - return &AtomTfdt{DecodeTime: decodeTime}, nil - } else { - // Version 1 uses 64-bit time - decodeTime := binary.BigEndian.Uint64(data[4:]) - return &AtomTfdt{DecodeTime: decodeTime}, nil - } + return &AtomTfdt{DecodeTime: binary.BigEndian.Uint64(data[4:])}, nil case MoofTrafTrun: rd := bits.NewReader(data) diff --git a/pkg/mp4/demuxer.go b/pkg/mp4/demuxer.go index 67da93de..25c8c70e 100644 --- a/pkg/mp4/demuxer.go +++ b/pkg/mp4/demuxer.go @@ -1,12 +1,9 @@ package mp4 import ( - "fmt" - "github.com/AlexxIT/go2rtc/pkg/aac" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/h264" - "github.com/AlexxIT/go2rtc/pkg/h265" "github.com/AlexxIT/go2rtc/pkg/iso" "github.com/pion/rtp" ) @@ -16,16 +13,6 @@ type Demuxer struct { timeScales map[uint32]float32 } -type TrackPackets struct { - TrackID uint32 - Packets []*core.Packet -} - -type TrackData struct { - DecodeTime uint32 - Trun *iso.AtomTrun -} - func (d *Demuxer) Probe(init []byte) (medias []*core.Media) { var trackID, timeScale uint32 @@ -47,23 +34,11 @@ func (d *Demuxer) Probe(init []byte) (medias []*core.Media) { switch atom.Name { case "avc1": codec = h264.ConfigToCodec(atom.Config) - case "hvc1", "hev1": - codec = h265.ConfigToCodec(atom.Config) } case *iso.AtomAudio: switch atom.Name { case "mp4a": - // G.711 PCMU audio detection for 8kHz mono (Tuya...) - if atom.SampleRate == 8000 && atom.Channels == 1 { - codec = &core.Codec{ - Name: core.CodecPCMU, - ClockRate: 8000, - Channels: 1, - PayloadType: 0, - } - } else { - codec = aac.ConfigToCodec(atom.Config) - } + codec = aac.ConfigToCodec(atom.Config) } } @@ -72,7 +47,6 @@ func (d *Demuxer) Probe(init []byte) (medias []*core.Media) { d.timeScales[trackID] = float32(codec.ClockRate) / float32(timeScale) medias = append(medias, &core.Media{ - ID: fmt.Sprintf("trackID=%d", trackID), Kind: codec.Kind(), Direction: core.DirectionRecvonly, Codecs: []*core.Codec{codec}, @@ -140,170 +114,3 @@ func (d *Demuxer) Demux(data2 []byte) (trackID uint32, packets []*core.Packet) { return } - -// DemuxAll returns packets from all tracks found in the fragment -func (d *Demuxer) DemuxAll(data []byte) []TrackPackets { - atoms, err := iso.DecodeAtoms(data) - if err != nil { - return nil - } - - // Map to store track-specific data - trackData := make(map[uint32]TrackData) - var mdat []byte - - // First pass: collect all track data - for _, atom := range atoms { - switch atom := atom.(type) { - case *iso.AtomMdat: - mdat = atom.Data - } - } - - // Temporary variables to track current track ID while parsing - var currentTrackID uint32 - - // Second pass: process traf boxes - for _, atom := range atoms { - switch atom := atom.(type) { - case *iso.AtomTfhd: - currentTrackID = atom.TrackID - - // Initialize track data if not exists - if _, ok := trackData[currentTrackID]; !ok { - trackData[currentTrackID] = TrackData{} - } - - case *iso.AtomTfdt: - if currentTrackID != 0 { - td := trackData[currentTrackID] - td.DecodeTime = uint32(atom.DecodeTime) - trackData[currentTrackID] = td - } - - case *iso.AtomTrun: - if currentTrackID != 0 { - td := trackData[currentTrackID] - td.Trun = atom - trackData[currentTrackID] = td - } - } - } - - // Process all tracks and collect results - var results []TrackPackets - - for tid, td := range trackData { - if td.Trun == nil || mdat == nil || len(td.Trun.SamplesSize) == 0 { - continue - } - - codec := d.codecs[tid] - if codec == nil { - continue - } - - timeScale := d.timeScales[tid] - - var packets []*core.Packet - switch codec.Kind() { - case "video": - packets = createVideoPackets(td.Trun, mdat, td.DecodeTime, timeScale) - case "audio": - packets = createAudioPackets(td.Trun, mdat, td.DecodeTime, timeScale, codec) - } - - if len(packets) > 0 { - results = append(results, TrackPackets{ - TrackID: tid, - Packets: packets, - }) - } - } - - return results -} - -// Creates video packets (H.264/H.265) -func createVideoPackets(trun *iso.AtomTrun, mdat []byte, decodeTime uint32, timeScale float32) []*core.Packet { - n := len(trun.SamplesSize) - hasDurations := len(trun.SamplesDuration) > 0 - - packets := make([]*core.Packet, n) - offset := uint32(0) - ts := decodeTime - - for i := 0; i < n; i++ { - // Get duration from array or use default - var duration uint32 - if hasDurations && i < len(trun.SamplesDuration) { - duration = trun.SamplesDuration[i] - } else { - duration = 1000 // Default for video - } - - size := trun.SamplesSize[i] - - if offset+size > uint32(len(mdat)) { - return packets[:i] - } - - timestamp := uint32(float32(ts) * timeScale) - packets[i] = &rtp.Packet{ - Header: rtp.Header{Timestamp: timestamp}, - Payload: mdat[offset : offset+size], - } - - offset += size - ts += duration - } - - return packets -} - -// Creates audio packets (G.711, AAC, etc.) -func createAudioPackets(trun *iso.AtomTrun, mdat []byte, decodeTime uint32, timeScale float32, codec *core.Codec) []*core.Packet { - n := len(trun.SamplesSize) - hasDurations := len(trun.SamplesDuration) > 0 - - packets := make([]*core.Packet, n) - offset := uint32(0) - ts := decodeTime - isPCM := codec.Name == core.CodecPCMU || codec.Name == core.CodecPCMA || codec.Name == core.CodecPCM || codec.Name == core.CodecPCML - - for i := 0; i < n; i++ { - size := trun.SamplesSize[i] - - // Calculate duration based on codec - var duration uint32 - if hasDurations && i < len(trun.SamplesDuration) { - duration = trun.SamplesDuration[i] - } else if isPCM { - duration = size - } else { - duration = 1024 - } - - if offset+size > uint32(len(mdat)) { - return packets[:i] - } - - // Calculate timestamp based on codec - var timestamp uint32 - if isPCM { - timestamp = ts - } else { - timestamp = uint32(float32(ts) * timeScale) - } - - packets[i] = &rtp.Packet{ - Header: rtp.Header{Timestamp: timestamp}, - Payload: mdat[offset : offset+size], - } - - offset += size - ts += duration - } - - return packets -} From 16a812c8b8f1558cba8608f7530092483decb5eb Mon Sep 17 00:00:00 2001 From: seydx Date: Sat, 17 May 2025 11:42:06 +0200 Subject: [PATCH 18/60] revert webrtc --- pkg/webrtc/conn.go | 24 ++++++++++++------------ pkg/webrtc/consumer.go | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/webrtc/conn.go b/pkg/webrtc/conn.go index f853bf43..063831f0 100644 --- a/pkg/webrtc/conn.go +++ b/pkg/webrtc/conn.go @@ -161,21 +161,21 @@ func (c *Conn) AddCandidate(candidate string) error { return c.pc.AddICECandidate(webrtc.ICECandidateInit{Candidate: candidate}) } -func (c *Conn) GetSenderTrack(mid string) *Track { - if tr := c.getTranseiver(mid); tr != nil { - if s := tr.Sender(); s != nil { - if t := s.Track().(*Track); t != nil { - return t - } +func (c *Conn) getTranseiver(mid string) *webrtc.RTPTransceiver { + for _, tr := range c.pc.GetTransceivers() { + if tr.Mid() == mid { + return tr } } return nil } -func (c *Conn) getTranseiver(mid string) *webrtc.RTPTransceiver { - for _, tr := range c.pc.GetTransceivers() { - if tr.Mid() == mid { - return tr +func (c *Conn) getSenderTrack(mid string) *Track { + if tr := c.getTranseiver(mid); tr != nil { + if s := tr.Sender(); s != nil { + if t := s.Track().(*Track); t != nil { + return t + } } } return nil @@ -209,7 +209,7 @@ func (c *Conn) getMediaCodec(remote *webrtc.TrackRemote) (*core.Media, *core.Cod // check GetTrack panic(core.Caller()) - // return nil, nil + return nil, nil } func sanitizeIP6(host string) string { @@ -217,4 +217,4 @@ func sanitizeIP6(host string) string { return "[" + host + "]" } return host -} +} \ No newline at end of file diff --git a/pkg/webrtc/consumer.go b/pkg/webrtc/consumer.go index 767394df..1efb1507 100644 --- a/pkg/webrtc/consumer.go +++ b/pkg/webrtc/consumer.go @@ -32,7 +32,7 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv panic(core.Caller()) } - localTrack := c.GetSenderTrack(media.ID) + localTrack := c.getSenderTrack(media.ID) if localTrack == nil { return errors.New("webrtc: can't get track") } @@ -87,4 +87,4 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv c.Senders = append(c.Senders, sender) return nil -} +} \ No newline at end of file From a9bcb46f387bd8fe1bc46dde556cf710a264f821 Mon Sep 17 00:00:00 2001 From: seydx Date: Sat, 17 May 2025 14:25:18 +0200 Subject: [PATCH 19/60] refactor --- pkg/tuya/api.go | 447 ++++++++++++++-------------------------- pkg/tuya/client.go | 277 ++++++++++++++++++++----- pkg/tuya/dc.go | 253 ----------------------- pkg/tuya/frameBuffer.go | 86 -------- pkg/tuya/mqtt.go | 58 +++--- pkg/webrtc/conn.go | 2 +- pkg/webrtc/consumer.go | 2 +- 7 files changed, 415 insertions(+), 710 deletions(-) delete mode 100644 pkg/tuya/dc.go delete mode 100644 pkg/tuya/frameBuffer.go diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index 6ffa3dd7..d08a68dc 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -17,25 +17,23 @@ import ( ) type TuyaClient struct { - httpClient *http.Client - mqtt *TuyaMQTT - apiURL string - rtspURL string - hlsURL string - sessionId string - clientId string - clientSecret string - deviceId string - accessToken string - refreshToken string - expireTime int64 - uid string - motoId string - auth string - skill *Skill - iceServers []pionWebrtc.ICEServer - medias []*core.Media - hasBackchannel bool + httpClient *http.Client + mqtt *TuyaMQTT + apiURL string + rtspURL string + hlsURL string + sessionId string + clientId string + clientSecret string + deviceId string + accessToken string + refreshToken string + expireTime int64 + uid string + motoId string + auth string + skill *Skill + iceServers []pionWebrtc.ICEServer } type Token struct { @@ -159,21 +157,16 @@ type OpenIoTHubConfigResponse struct { Code int `json:"code,omitempty"` } -const ( - defaultTimeout = 5 * time.Second -) - -func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId string, clientSecret string, streamMode string) (*TuyaClient, error) { +func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId string, clientSecret string, streamMode string, streamRole string) (*TuyaClient, error) { client := &TuyaClient{ - httpClient: &http.Client{Timeout: defaultTimeout}, - mqtt: &TuyaMQTT{waiter: core.Waiter{}}, - apiURL: openAPIURL, - sessionId: core.RandString(6, 62), - clientId: clientId, - deviceId: deviceId, - clientSecret: clientSecret, - uid: uid, - hasBackchannel: false, + httpClient: &http.Client{Timeout: 5 * time.Second}, + mqtt: &TuyaMQTT{waiter: core.Waiter{}}, + apiURL: openAPIURL, + sessionId: core.RandString(6, 62), + clientId: clientId, + deviceId: deviceId, + clientSecret: clientSecret, + uid: uid, } if err := client.InitToken(); err != nil { @@ -189,7 +182,7 @@ func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId stri return nil, fmt.Errorf("failed to get HLS URL: %w", err) } } else { - if err := client.InitDevice(); err != nil { + if err := client.InitDevice(streamRole); err != nil { return nil, fmt.Errorf("failed to initialize device: %w", err) } @@ -206,6 +199,135 @@ func (c *TuyaClient) Close() { c.httpClient.CloseIdleConnections() } +func (c *TuyaClient) InitToken() (err error) { + url := fmt.Sprintf("https://%s/v1.0/token?grant_type=1", c.apiURL) + + c.accessToken = "" + c.refreshToken = "" + + body, err := c.Request("GET", url, nil) + if err != nil { + return err + } + + var tokenResponse TokenResponse + err = json.Unmarshal(body, &tokenResponse) + if err != nil { + return err + } + + if !tokenResponse.Success { + return fmt.Errorf(tokenResponse.Msg) + } + + c.accessToken = tokenResponse.Result.AccessToken + c.refreshToken = tokenResponse.Result.RefreshToken + c.expireTime = tokenResponse.Result.ExpireTime + + return nil +} + +func (c *TuyaClient) InitDevice(streamRole string) (err error) { + url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.apiURL, c.uid, c.deviceId) + + body, err := c.Request("GET", url, nil) + if err != nil { + return err + } + + var webRTCConfigResponse WebRTCConfigResponse + err = json.Unmarshal(body, &webRTCConfigResponse) + if err != nil { + return err + } + + if !webRTCConfigResponse.Success { + return fmt.Errorf(webRTCConfigResponse.Msg) + } + + err = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), &c.skill) + if err != nil { + return err + } + + iceServers, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) + if err != nil { + return err + } + + c.iceServers, err = webrtc.UnmarshalICEServers(iceServers) + if err != nil { + return err + } + + c.motoId = webRTCConfigResponse.Result.MotoID + c.auth = webRTCConfigResponse.Result.Auth + + return nil +} + +func (c *TuyaClient) GetStreamUrl(streamType string) (err error) { + url := fmt.Sprintf("https://%s/v1.0/devices/%s/stream/actions/allocate", c.apiURL, c.deviceId) + + request := &AllocateRequest{ + Type: streamType, + } + + body, err := c.Request("POST", url, request) + if err != nil { + return err + } + + var allosResponse AllocateResponse + err = json.Unmarshal(body, &allosResponse) + if err != nil { + return err + } + + if !allosResponse.Success { + return fmt.Errorf(allosResponse.Msg) + } + + switch streamType { + case "rtsp": + c.rtspURL = allosResponse.Result.URL + case "hls": + c.hlsURL = allosResponse.Result.URL + default: + return fmt.Errorf("unsupported stream type: %s", streamType) + } + + return nil +} + +func (c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { + url := fmt.Sprintf("https://%s/v2.0/open-iot-hub/access/config", c.apiURL) + + request := &OpenIoTHubConfigRequest{ + UID: c.uid, + UniqueID: uuid.New().String(), + LinkType: "mqtt", + Topics: "ipc", + } + + body, err := c.Request("POST", url, request) + if err != nil { + return nil, err + } + + var openIoTHubConfigResponse OpenIoTHubConfigResponse + err = json.Unmarshal(body, &openIoTHubConfigResponse) + if err != nil { + return nil, err + } + + if !openIoTHubConfigResponse.Success { + return nil, fmt.Errorf(openIoTHubConfigResponse.Msg) + } + + return &openIoTHubConfigResponse.Result, nil +} + func (c *TuyaClient) Request(method string, url string, body any) ([]byte, error) { var bodyReader io.Reader if body != nil { @@ -253,224 +375,7 @@ func (c *TuyaClient) Request(method string, url string, body any) ([]byte, error return res, nil } -func (c *TuyaClient) InitToken() (err error) { - url := fmt.Sprintf("https://%s/v1.0/token?grant_type=1", c.apiURL) - - c.accessToken = "" - c.refreshToken = "" - - body, err := c.Request("GET", url, nil) - if err != nil { - return err - } - - var tokenResponse TokenResponse - err = json.Unmarshal(body, &tokenResponse) - if err != nil { - return err - } - - if !tokenResponse.Success { - return fmt.Errorf("error: %s", tokenResponse.Msg) - } - - c.accessToken = tokenResponse.Result.AccessToken - c.refreshToken = tokenResponse.Result.RefreshToken - c.expireTime = tokenResponse.Result.ExpireTime - - return nil -} - -func (c *TuyaClient) InitDevice() (err error) { - url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.apiURL, c.uid, c.deviceId) - - body, err := c.Request("GET", url, nil) - if err != nil { - return err - } - - var webRTCConfigResponse WebRTCConfigResponse - err = json.Unmarshal(body, &webRTCConfigResponse) - if err != nil { - return err - } - - if !webRTCConfigResponse.Success { - return fmt.Errorf("error: %s", webRTCConfigResponse.Msg) - } - - c.motoId = webRTCConfigResponse.Result.MotoID - c.auth = webRTCConfigResponse.Result.Auth - - c.skill = &Skill{ - WebRTC: 3, // basic webrtc - Audios: make([]AudioSkill, 0), - Videos: make([]VideoSkill, 0), - } - - if webRTCConfigResponse.Result.Skill != "" { - _ = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), c.skill) - } - - c.hasBackchannel = contains(webRTCConfigResponse.Result.AudioAttributes.CallMode, 2) && - contains(webRTCConfigResponse.Result.AudioAttributes.HardwareCapability, 1) - - c.medias = make([]*core.Media, 0) - - if len(c.skill.Audios) > 0 { - direction := core.DirectionRecvonly - if c.hasBackchannel { - direction = core.DirectionSendRecv - } - - codecs := make([]*core.Codec, 0) - for _, audio := range c.skill.Audios { - codecs = append(codecs, &core.Codec{ - Name: getAudioCodec(audio.CodecType), - ClockRate: uint32(audio.SampleRate), - Channels: uint8(audio.Channels), - }) - } - - c.medias = append(c.medias, &core.Media{ - Kind: core.KindAudio, - Direction: direction, - Codecs: codecs, - }) - } else { - // Use default values for Audio - c.medias = append(c.medias, &core.Media{ - Kind: core.KindAudio, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: core.CodecPCMU, - ClockRate: uint32(8000), - Channels: uint8(1), - }, - }, - }) - } - - if len(c.skill.Videos) > 0 { - codecs := make([]*core.Codec, 0) - for _, video := range c.skill.Videos { - if video.CodecType == 2 { - codecs = append(codecs, &core.Codec{ - Name: core.CodecH264, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, - }) - } else if video.CodecType == 4 { - codecs = append(codecs, &core.Codec{ - Name: core.CodecH265, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, - }) - } - } - - c.medias = append(c.medias, &core.Media{ - Kind: core.KindVideo, - Direction: core.DirectionRecvonly, - Codecs: codecs, - }) - } else { - // Use default values for Video - c.medias = append(c.medias, &core.Media{ - Kind: core.KindVideo, - Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: core.CodecH264, - ClockRate: uint32(90000), - PayloadType: 96, - }, - { - Name: core.CodecH265, - ClockRate: uint32(90000), - PayloadType: 96, - }, - }, - }) - } - - iceServersBytes, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) - if err != nil { - return err - } - - c.iceServers, err = webrtc.UnmarshalICEServers([]byte(iceServersBytes)) - if err != nil { - return err - } - - return nil -} - -func (c *TuyaClient) GetStreamUrl(streamType string) (err error) { - url := fmt.Sprintf("https://%s/v1.0/devices/%s/stream/actions/allocate", c.apiURL, c.deviceId) - - request := &AllocateRequest{ - Type: streamType, - } - - body, err := c.Request("POST", url, request) - if err != nil { - return err - } - - var allosResponse AllocateResponse - err = json.Unmarshal(body, &allosResponse) - if err != nil { - return err - } - - if !allosResponse.Success { - return fmt.Errorf("error: %s", allosResponse.Msg) - } - - switch streamType { - case "rtsp": - c.rtspURL = allosResponse.Result.URL - case "hls": - c.hlsURL = allosResponse.Result.URL - default: - return fmt.Errorf("unsupported stream type: %s", streamType) - } - - return nil -} - -func (c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { - url := fmt.Sprintf("https://%s/v2.0/open-iot-hub/access/config", c.apiURL) - - request := &OpenIoTHubConfigRequest{ - UID: c.uid, - UniqueID: uuid.New().String(), - LinkType: "mqtt", - Topics: "ipc", - } - - body, err := c.Request("POST", url, request) - if err != nil { - return nil, err - } - - var openIoTHubConfigResponse OpenIoTHubConfigResponse - err = json.Unmarshal(body, &openIoTHubConfigResponse) - if err != nil { - return nil, err - } - - if !openIoTHubConfigResponse.Success { - return nil, fmt.Errorf("error: %s", openIoTHubConfigResponse.Msg) - } - - return &openIoTHubConfigResponse.Result, nil -} - -func (c *TuyaClient) getStreamType(streamChoice string) int { +func (c *TuyaClient) getStreamType(streamRole string) int { // Default streamType if nothing is found defaultStreamType := 1 @@ -501,7 +406,7 @@ func (c *TuyaClient) getStreamType(streamChoice string) int { } // Return the streamType based on the selection - switch streamChoice { + switch streamRole { case "main": return highestResType case "sub": @@ -511,29 +416,6 @@ func (c *TuyaClient) getStreamType(streamChoice string) int { } } -func getAudioCodec(codecType int) string { - switch codecType { - // case 100: - // return "ADPCM" - case 101: - return core.CodecPCM - case 102, 103, 104: - return core.CodecAAC - case 105: - return core.CodecPCMU - case 106: - return core.CodecPCMA - // case 107: - // return "G726-32" - // case 108: - // return "SPEEX" - case 109: - return core.CodecMP3 - default: - return core.CodecPCMU - } -} - func (c *TuyaClient) isHEVC(streamType int) bool { for _, video := range c.skill.Videos { if video.StreamType == streamType { @@ -544,22 +426,9 @@ func (c *TuyaClient) isHEVC(streamType int) bool { return false } -func (c *TuyaClient) isClaritySupported(webrtcValue int) bool { - return (webrtcValue & (1 << 5)) != 0 -} - func (c *TuyaClient) calBusinessSign(ts int64) string { data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts) val := md5.Sum([]byte(data)) res := fmt.Sprintf("%X", val) return res } - -func contains(slice []int, val int) bool { - for _, item := range slice { - if item == val { - return true - } - } - return false -} diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 37697310..ecf4d480 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -1,6 +1,7 @@ package tuya import ( + "encoding/json" "errors" "fmt" "net/url" @@ -14,10 +15,30 @@ import ( ) type Client struct { - api *TuyaClient - conn *webrtc.Conn - dcConn *DCConn - done chan struct{} + api *TuyaClient + conn *webrtc.Conn + pc *pion.PeerConnection + dc *pion.DataChannel + videoSSRC uint32 + audioSSRC uint32 + isHEVC bool + connected core.Waiter + closed bool + handlers map[uint32]func(*rtp.Packet) +} + +type DataChannelMessage struct { + Type string `json:"type"` + Msg string `json:"msg"` +} + +type RecvMessage struct { + Video struct { + SSRC uint32 `json:"ssrc"` + } `json:"video"` + Audio struct { + SSRC uint32 `json:"ssrc"` + } `json:"audio"` } const ( @@ -30,7 +51,6 @@ const ( ) func Dial(rawURL string) (core.Producer, error) { - // Parse URL and validate basic params u, err := url.Parse(rawURL) if err != nil { return nil, err @@ -41,8 +61,13 @@ func Dial(rawURL string) (core.Producer, error) { uid := query.Get("uid") clientId := query.Get("client_id") clientSecret := query.Get("client_secret") - streamType := query.Get("type") + streamRole := query.Get("role") streamMode := query.Get("mode") + + if streamRole == "" || (streamRole != "main" && streamRole != "sub") { + streamRole = "main" + } + useRTSP := streamMode == "rtsp" useHLS := streamMode == "hls" useWebRTC := streamMode == "webrtc" || streamMode == "" @@ -72,14 +97,14 @@ func Dial(rawURL string) (core.Producer, error) { } // Initialize Tuya API client - tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientId, clientSecret, streamMode) + tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientId, clientSecret, streamMode, streamRole) if err != nil { - return nil, err + return nil, fmt.Errorf("tuya: %w", err) } client := &Client{ - api: tuyaAPI, - done: make(chan struct{}), + api: tuyaAPI, + handlers: make(map[uint32]func(*rtp.Packet)), } if useRTSP { @@ -93,8 +118,9 @@ func Dial(rawURL string) (core.Producer, error) { } return streams.GetProducer(client.api.hlsURL) } else { - isHEVC := client.api.isHEVC(client.api.getStreamType(streamType)) + client.isHEVC = client.api.isHEVC(client.api.getStreamType(streamRole)) + // Create a new PeerConnection conf := pion.Configuration{ ICEServers: client.api.iceServers, ICETransportPolicy: pion.ICETransportPolicyAll, @@ -107,7 +133,7 @@ func Dial(rawURL string) (core.Producer, error) { return nil, err } - pc, err := api.NewPeerConnection(conf) + client.pc, err = api.NewPeerConnection(conf) if err != nil { client.api.Close() return nil, err @@ -119,10 +145,8 @@ func Dial(rawURL string) (core.Producer, error) { // protect from blocking on errors defer sendOffer.Done(nil) - // waiter will wait PC error - var connState core.Waiter - - client.conn = webrtc.NewConn(pc) + // Create new WebRTC connection + client.conn = webrtc.NewConn(client.pc) client.conn.FormatName = "tuya/webrtc" client.conn.Mode = core.ModeActiveProducer client.conn.Protocol = "mqtt" @@ -137,8 +161,8 @@ func Dial(rawURL string) (core.Producer, error) { SDP: answer.Sdp, } - if err = pc.SetRemoteDescription(desc); err != nil { - client.Stop() + if err = client.pc.SetRemoteDescription(desc); err != nil { + client.connected.Done(err) return } @@ -147,7 +171,18 @@ func Dial(rawURL string) (core.Producer, error) { return } - client.conn.SDP = answer.Sdp + if client.isHEVC { + // Tuya answers always with H264 codec, replace with HEVC + for _, media := range client.conn.Medias { + if media.Kind == core.KindVideo { + for _, codec := range media.Codecs { + if codec.Name == core.CodecH264 { + codec.Name = core.CodecH265 + } + } + } + } + } } client.api.mqtt.handleCandidate = func(candidate CandidateFrame) { @@ -170,15 +205,57 @@ func Dial(rawURL string) (core.Producer, error) { client.Stop() } - // Set up data channel for HEVC - if isHEVC { - client.dcConn, err = NewDCConn(pc, client) - if err != nil { - client.api.Close() - return nil, err - } + // On HEVC, use DataChannel to receive video/audio + if client.isHEVC { + // Create a new DataChannel + maxRetransmits := uint16(5) + ordered := true + client.dc, err = client.pc.CreateDataChannel("fmp4Stream", &pion.DataChannelInit{ + MaxRetransmits: &maxRetransmits, + Ordered: &ordered, + }) + + // Set up data channel handler + client.dc.OnMessage(func(msg pion.DataChannelMessage) { + if msg.IsString { + client.probe(msg) + } else { + packet := &rtp.Packet{} + if err := packet.Unmarshal(msg.Data); err != nil { + return + } + + if handler, ok := client.handlers[packet.SSRC]; ok { + handler(packet) + } + } + }) + + client.dc.OnError(func(err error) { + // fmt.Printf("tuya: datachannel error: %s\n", err.Error()) + client.connected.Done(err) + }) + + client.dc.OnClose(func() { + // fmt.Println("tuya: datachannel closed") + client.connected.Done(errors.New("datachannel: closed")) + }) + + client.dc.OnOpen(func() { + // fmt.Println("tuya: datachannel opened") + + codecRequest, _ := json.Marshal(DataChannelMessage{ + Type: "codec", + Msg: "", + }) + + if err := client.sendMessageToDataChannel(codecRequest); err != nil { + client.connected.Done(fmt.Errorf("failed to send codec request: %w", err)) + } + }) } + // Set up pc handler client.conn.Listen(func(msg any) { switch msg := msg.(type) { case *pion.ICECandidate: @@ -192,16 +269,25 @@ func Dial(rawURL string) (core.Producer, error) { case pion.PeerConnectionStateConnecting: break case pion.PeerConnectionStateConnected: - connState.Done(nil) + // On HEVC, wait for DataChannel to be opened and camera to send codec info + if !client.isHEVC { + client.connected.Done(nil) + } default: client.Stop() - connState.Done(errors.New("webrtc: " + msg.String())) + client.connected.Done(errors.New("webrtc: " + msg.String())) } } }) + // Audio first, otherwise tuya will send corrupt sdp + medias := []*core.Media{ + {Kind: core.KindAudio, Direction: core.DirectionSendRecv}, + {Kind: core.KindVideo, Direction: core.DirectionRecvonly}, + } + // Create offer - offer, err := client.conn.CreateOffer(client.api.medias) + offer, err := client.conn.CreateOffer(medias) if err != nil { client.api.Close() return nil, err @@ -213,20 +299,12 @@ func Dial(rawURL string) (core.Producer, error) { offer = re.ReplaceAllString(offer, "") // Send offer - client.api.sendOffer(offer, streamType) + client.api.sendOffer(offer, streamRole) sendOffer.Done(nil) - if client.dcConn != nil { - if err = client.dcConn.connected.Wait(); err != nil { - client.Stop() - return nil, err - } - - return client.dcConn, nil - } - - if err = connState.Wait(); err != nil { - return nil, err + // Wait for connection + if err = client.connected.Wait(); err != nil { + return nil, fmt.Errorf("tuya: %w", err) } return client, nil @@ -242,10 +320,15 @@ func (c *Client) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, } func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiver) error { - // RepackG711 will not work, so add default logic without repacking + // Manually handle backchannel, because repacking audio through go2rtc does not work + + localTrack := c.getSender() + if localTrack == nil { + return errors.New("webrtc: can't get track") + } payloadType := codec.PayloadType - localTrack := c.conn.GetSenderTrack(media.ID) + sender := core.NewSender(media, codec) sender.Handler = func(packet *rtp.Packet) { c.conn.Send += packet.MarshalSize() @@ -260,29 +343,47 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece } func (c *Client) Start() error { - if c.dcConn != nil { - c.dcConn.Start() + if len(c.conn.Receivers) == 0 { + return errors.New("tuya: no receivers") + } + + var video, audio *core.Receiver + for _, receiver := range c.conn.Receivers { + if receiver.Codec.IsVideo() { + video = receiver + } else if receiver.Codec.IsAudio() { + audio = receiver + } + } + + c.handlers[c.videoSSRC] = func(packet *rtp.Packet) { + if video != nil { + video.WriteRTP(packet) + } + } + + c.handlers[c.audioSSRC] = func(packet *rtp.Packet) { + if audio != nil { + audio.WriteRTP(packet) + } } return c.conn.Start() } func (c *Client) Stop() error { - select { - case <-c.done: + if c.closed { return nil - default: - close(c.done) + } + + for ssrc := range c.handlers { + delete(c.handlers, ssrc) } if c.conn != nil { _ = c.conn.Stop() } - if c.dcConn != nil { - _ = c.dcConn.Stop() - } - if c.api != nil { c.api.Close() } @@ -293,3 +394,73 @@ func (c *Client) Stop() error { func (c *Client) MarshalJSON() ([]byte, error) { return c.conn.MarshalJSON() } + +func (c *Client) probe(msg pion.DataChannelMessage) { + // fmt.Printf("[tuya] Received string message: %s\n", string(msg.Data)) + + var message DataChannelMessage + if err := json.Unmarshal([]byte(msg.Data), &message); err != nil { + c.connected.Done(fmt.Errorf("failed to parse datachannel message: %w", err)) + } + + switch message.Type { + case "codec": + // fmt.Printf("[tuya] Codec info from camera: %s\n", message.Msg) + + frameRequest, _ := json.Marshal(DataChannelMessage{ + Type: "start", + Msg: "frame", + }) + + err := c.sendMessageToDataChannel(frameRequest) + if err != nil { + c.connected.Done(fmt.Errorf("failed to send frame request: %w", err)) + } + + case "recv": + var recvMessage RecvMessage + if err := json.Unmarshal([]byte(message.Msg), &recvMessage); err != nil { + c.connected.Done(fmt.Errorf("failed to parse recv message: %w", err)) + return + } + + c.videoSSRC = recvMessage.Video.SSRC + c.audioSSRC = recvMessage.Audio.SSRC + + completeMsg, _ := json.Marshal(DataChannelMessage{ + Type: "complete", + Msg: "", + }) + + err := c.sendMessageToDataChannel(completeMsg) + if err != nil { + c.connected.Done(fmt.Errorf("failed to send complete message: %w", err)) + } + + c.connected.Done(nil) + } +} + +func (c *Client) sendMessageToDataChannel(message []byte) error { + if c.dc != nil { + // fmt.Printf("[tuya] sending message to data channel: %s\n", message) + return c.dc.Send(message) + } + + return nil +} + +func (c *Client) getSender() *webrtc.Track { + for _, tr := range c.pc.GetTransceivers() { + if tr.Kind() == pion.RTPCodecTypeAudio { + if tr.Kind() == pion.RTPCodecType(pion.RTPTransceiverDirectionSendonly) || tr.Kind() == pion.RTPCodecType(pion.RTPTransceiverDirectionSendrecv) { + if s := tr.Sender(); s != nil { + if t := s.Track().(*webrtc.Track); t != nil { + return t + } + } + } + } + } + return nil +} diff --git a/pkg/tuya/dc.go b/pkg/tuya/dc.go deleted file mode 100644 index 2ccd1313..00000000 --- a/pkg/tuya/dc.go +++ /dev/null @@ -1,253 +0,0 @@ -package tuya - -import ( - "encoding/json" - "errors" - - "github.com/AlexxIT/go2rtc/pkg/core" - "github.com/AlexxIT/go2rtc/pkg/mp4" - "github.com/pion/rtp" - pion "github.com/pion/webrtc/v4" -) - -type DCConn struct { - core.Connection - - client *Client - dc *pion.DataChannel - dem *mp4.Demuxer - queue *FrameBufferQueue - msgs chan pion.DataChannelMessage - connected core.Waiter - closed core.Waiter - initialized bool -} - -type DataChannelMessage struct { - Type string `json:"type"` - Msg string `json:"msg"` -} - -func NewDCConn(pc *pion.PeerConnection, c *Client) (*DCConn, error) { - maxRetransmits := uint16(5) - ordered := true - dc, err := pc.CreateDataChannel("fmp4Stream", &pion.DataChannelInit{ - MaxRetransmits: &maxRetransmits, - Ordered: &ordered, - }) - - if err != nil { - return nil, err - } - - conn := &DCConn{ - Connection: core.Connection{ - ID: core.NewID(), - FormatName: "webrtc/fmp4", - Transport: dc, - }, - client: c, - dc: dc, - dem: &mp4.Demuxer{}, - queue: NewFrameBufferQueue(), - msgs: make(chan pion.DataChannelMessage, 10), // Saw max 4 messages in a row, 10 should be enough - initialized: false, - } - - dc.OnMessage(func(msg pion.DataChannelMessage) { - conn.msgs <- msg - }) - - dc.OnError(func(err error) { - conn.connected.Done(err) - }) - - dc.OnClose(func() { - close(conn.msgs) - conn.connected.Done(errors.New("datachannel: closed")) - }) - - go conn.initializationLoop() - - return conn, nil -} - -func (c *DCConn) initializationLoop() { - for msg := range c.msgs { - if c.initialized { - return - } - - err := c.probe(msg) - if err != nil { - c.connected.Done(err) - return - } - - if c.initialized { - c.connected.Done(nil) - return - } - } -} - -func (c *DCConn) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, error) { - if media.Direction == core.DirectionSendRecv || media.Direction == core.DirectionSendonly { - return c.client.GetTrack(media, codec) - } - - for _, receiver := range c.Receivers { - if receiver.Codec == codec { - return receiver, nil - } - } - receiver := core.NewReceiver(media, codec) - c.Receivers = append(c.Receivers, receiver) - return receiver, nil -} - -func (c *DCConn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiver) error { - payloadType := codec.PayloadType - localTrack := c.client.conn.GetSenderTrack(media.ID) - sender := core.NewSender(media, codec) - sender.Handler = func(packet *rtp.Packet) { - c.Send += packet.MarshalSize() - //important to send with remote PayloadType - _ = localTrack.WriteRTP(payloadType, packet) - } - - sender.HandleRTP(track) - c.Senders = append(c.Senders, sender) - - return nil -} - -func (c *DCConn) Start() error { - receivers := make(map[uint32]*core.Receiver) - for _, receiver := range c.Receivers { - trackID := c.dem.GetTrackID(receiver.Codec) - receivers[trackID] = receiver - } - - ch := make(chan []byte, 10) - defer close(ch) - - go func() { - for data := range ch { - allTracks := c.dem.DemuxAll(data) - for _, trackData := range allTracks { - trackID := trackData.TrackID - packets := trackData.Packets - receiver := receivers[trackID] - if receiver == nil { - continue - } - - for _, packet := range packets { - receiver.WriteRTP(packet) - } - } - } - }() - - go func() { - for msg := range c.msgs { - if len(msg.Data) >= 4 { - segmentNum := int(msg.Data[1]) - fragmentCount := int(msg.Data[2]) - fragmentSeq := int(msg.Data[3]) - mp4Data := msg.Data[4:] - - c.queue.AddFragment(segmentNum, fragmentCount, fragmentSeq, mp4Data) - - if c.queue.IsSegmentComplete(segmentNum, fragmentCount) { - b := c.queue.GetCombinedBuffer(segmentNum) - c.Recv += len(b) - ch <- b - } - } - } - }() - - c.closed.Wait() - return nil -} - -func (c *DCConn) sendMessageToDataChannel(message string) error { - if c.dc != nil { - return c.dc.SendText(message) - } - - return nil -} - -func (c *DCConn) probe(msg pion.DataChannelMessage) (err error) { - if msg.IsString { - var message DataChannelMessage - if err = json.Unmarshal(msg.Data, &message); err != nil { - return err - } - - switch message.Type { - case "codec": - response, _ := json.Marshal(DataChannelMessage{ - Type: "start", - Msg: "fmp4", - }) - - err = c.sendMessageToDataChannel(string(response)) - if err != nil { - return err - } - - case "recv": - response, _ := json.Marshal(DataChannelMessage{ - Type: "complete", - Msg: "", - }) - - err = c.sendMessageToDataChannel(string(response)) - if err != nil { - return err - } - } - - } else { - if len(msg.Data) >= 4 { - messageType := msg.Data[0] - segmentNum := int(msg.Data[1]) - fragmentCount := int(msg.Data[2]) - fragmentSeq := int(msg.Data[3]) - mp4Data := msg.Data[4:] - - // initialization segment - if messageType == 0 && segmentNum == 1 && fragmentCount == 1 && fragmentSeq == 1 { - medias := c.dem.Probe(mp4Data) - c.Medias = append(c.Medias, medias...) - - // Add backchannel - webrtcMedias := c.client.GetMedias() - for _, media := range webrtcMedias { - if media.Kind == core.KindAudio { - if media.Direction == core.DirectionSendRecv || media.Direction == core.DirectionSendonly { - c.Medias = append(c.Medias, media) - } - } - } - - c.initialized = true - } - } - } - - return nil -} - -func (c *DCConn) Stop() error { - if c.dc != nil && c.dc.ReadyState() == pion.DataChannelStateOpen { - _ = c.dc.Close() - } - - c.closed.Done(nil) - return nil -} diff --git a/pkg/tuya/frameBuffer.go b/pkg/tuya/frameBuffer.go deleted file mode 100644 index bbcb4ff5..00000000 --- a/pkg/tuya/frameBuffer.go +++ /dev/null @@ -1,86 +0,0 @@ -package tuya - -import ( - "sort" - "sync" -) - -type FrameBufferQueue struct { - segments map[int]map[int][]byte // segNum -> fragSeq -> data - mu sync.Mutex -} - -func NewFrameBufferQueue() *FrameBufferQueue { - return &FrameBufferQueue{ - segments: make(map[int]map[int][]byte), - } -} - -func (q *FrameBufferQueue) AddFragment(segmentNum, fragmentCount, fragmentSeq int, data []byte) { - q.mu.Lock() - defer q.mu.Unlock() - - if _, ok := q.segments[segmentNum]; !ok { - q.segments[segmentNum] = make(map[int][]byte) - } - - q.segments[segmentNum][fragmentSeq] = data -} - -func (q *FrameBufferQueue) IsSegmentComplete(segmentNum, fragmentCount int) bool { - q.mu.Lock() - defer q.mu.Unlock() - - if frags, ok := q.segments[segmentNum]; ok { - // Make sure we have the right number of fragments - if len(frags) != fragmentCount { - return false - } - - // Check if we have all sequences from 1 to fragmentCount - for i := 1; i <= fragmentCount; i++ { - if _, ok := frags[i]; !ok { - return false - } - } - - return true - } - - return false -} - -func (q *FrameBufferQueue) GetCombinedBuffer(segNum int) []byte { - q.mu.Lock() - defer q.mu.Unlock() - - if frags, ok := q.segments[segNum]; ok { - // Sort fragments by sequence number - var keys []int - for k := range frags { - keys = append(keys, k) - } - sort.Ints(keys) - - // Calculate total size for pre-allocation - totalSize := 0 - for _, k := range keys { - totalSize += len(frags[k]) - } - - // Pre-allocate buffer for better performance - combined := make([]byte, 0, totalSize) - - // Combine fragments in sequence order - for _, k := range keys { - combined = append(combined, frags[k]...) - } - - // Remove this segment to free memory - delete(q.segments, segNum) - - return combined - } - - return nil -} diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index 9eb25b49..ea69f60e 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -131,7 +131,7 @@ func (c *TuyaClient) onConnect(client mqtt.Client) { func (c *TuyaClient) consume(client mqtt.Client, msg mqtt.Message) { var rmqtt MqttMessage if err := json.Unmarshal(msg.Payload(), &rmqtt); err != nil { - c.mqtt.onError(fmt.Errorf("unmarshal mqtt message fail: %s, payload: %s", err.Error(), string(msg.Payload()))) + c.mqtt.onError(err) return } @@ -152,10 +152,7 @@ func (c *TuyaClient) consume(client mqtt.Client, msg mqtt.Message) { func (c *TuyaMQTT) onMqttAnswer(msg *MqttMessage) { var answerFrame AnswerFrame if err := json.Unmarshal(msg.Data.Message, &answerFrame); err != nil { - c.onError(fmt.Errorf("unmarshal mqtt answer frame fail: %s, session: %s, frame: %s", - err.Error(), - msg.Data.Header.SessionID, - string(msg.Data.Message))) + c.onError(err) return } @@ -165,10 +162,7 @@ func (c *TuyaMQTT) onMqttAnswer(msg *MqttMessage) { func (c *TuyaMQTT) onMqttCandidate(msg *MqttMessage) { var candidateFrame CandidateFrame if err := json.Unmarshal(msg.Data.Message, &candidateFrame); err != nil { - c.onError(fmt.Errorf("unmarshal mqtt candidate frame fail: %s, session: %s, frame: %s", - err.Error(), - msg.Data.Header.SessionID, - string(msg.Data.Message))) + c.onError(err) return } @@ -208,37 +202,48 @@ func (c *TuyaMQTT) onError(err error) { } } -func (c *TuyaClient) sendOffer(sdp string, streamType string) { - fixedStreamType := c.getStreamType(streamType) - isHEVC := c.isHEVC(fixedStreamType) +func (c *TuyaClient) sendOffer(sdp string, streamRole string) { + streamType := c.getStreamType(streamRole) + isHEVC := c.isHEVC(streamType) if isHEVC { // On HEVC we use streamType 0 for main stream and 1 for sub stream - if streamType == "main" { - fixedStreamType = 0 + if streamRole == "main" { + streamType = 0 } else { - fixedStreamType = 1 + streamType = 1 } } - c.sendMqttMessage("offer", 302, "", OfferFrame{ + err := c.sendMqttMessage("offer", 302, "", OfferFrame{ Mode: "webrtc", Sdp: sdp, - StreamType: fixedStreamType, + StreamType: streamType, Auth: c.auth, DatachannelEnable: isHEVC, }) + + if err != nil { + c.mqtt.onError(err) + return + } } func (c *TuyaClient) sendCandidate(candidate string) { - c.sendMqttMessage("candidate", 302, "", CandidateFrame{ + err := c.sendMqttMessage("candidate", 302, "", CandidateFrame{ Mode: "webrtc", Candidate: candidate, }) + + if err != nil { + c.mqtt.onError(err) + return + } } func (c *TuyaClient) sendResolution(resolution int) { - if !c.isClaritySupported(resolution) { + isClaritySupperted := (c.skill.WebRTC & (1 << 5)) != 0 + if !isClaritySupperted { return } @@ -261,16 +266,14 @@ func (c *TuyaClient) sendDisconnect() { }) } -func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transactionID string, data interface{}) { +func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transactionID string, data interface{}) error { if c.mqtt.closed { - c.mqtt.onError(fmt.Errorf("mqtt client is closed, send mqtt message fail")) - return + return fmt.Errorf("mqtt client is closed, send mqtt message fail") } jsonMessage, err := json.Marshal(data) if err != nil { - c.mqtt.onError(fmt.Errorf("marshal mqtt message fail: %s", err.Error())) - return + return err } msg := &MqttMessage{ @@ -292,12 +295,13 @@ func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transacti payload, err := json.Marshal(msg) if err != nil { - c.mqtt.onError(fmt.Errorf("marshal mqtt message fail: %s", err.Error())) - return + return err } token := c.mqtt.client.Publish(c.mqtt.publishTopic, 1, false, payload) if token.Wait() && token.Error() != nil { - c.mqtt.onError(fmt.Errorf("mqtt publish fail: %s, topic: %s", token.Error().Error(), c.mqtt.publishTopic)) + return token.Error() } + + return nil } diff --git a/pkg/webrtc/conn.go b/pkg/webrtc/conn.go index 063831f0..092b05c8 100644 --- a/pkg/webrtc/conn.go +++ b/pkg/webrtc/conn.go @@ -217,4 +217,4 @@ func sanitizeIP6(host string) string { return "[" + host + "]" } return host -} \ No newline at end of file +} diff --git a/pkg/webrtc/consumer.go b/pkg/webrtc/consumer.go index 1efb1507..ebc3a008 100644 --- a/pkg/webrtc/consumer.go +++ b/pkg/webrtc/consumer.go @@ -87,4 +87,4 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv c.Senders = append(c.Senders, sender) return nil -} \ No newline at end of file +} From e7044a93f62b2b4004f18b6de45b890d61032275 Mon Sep 17 00:00:00 2001 From: seydx Date: Sat, 17 May 2025 15:23:54 +0200 Subject: [PATCH 20/60] fix video/audio and minor improvements --- pkg/tuya/api.go | 73 ++++++++++++++++++++++++++++++++++++++++++++++ pkg/tuya/client.go | 43 ++++++++++++++++++++------- pkg/tuya/mqtt.go | 56 +++++++++++++++-------------------- 3 files changed, 128 insertions(+), 44 deletions(-) diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index d08a68dc..3842ed02 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -375,6 +375,79 @@ func (c *TuyaClient) Request(method string, url string, body any) ([]byte, error return res, nil } +func (c *TuyaClient) getVideoCodecs() []*core.Codec { + if len(c.skill.Videos) > 0 { + codecs := make([]*core.Codec, 0) + + for _, video := range c.skill.Videos { + name := core.CodecH264 + if c.isHEVC(video.StreamType) { + name = core.CodecH265 + } + + codec := &core.Codec{ + Name: name, + ClockRate: uint32(video.SampleRate), + } + + codecs = append(codecs, codec) + } + + if len(codecs) > 0 { + return codecs + } + } + + return nil +} + +func (c *TuyaClient) getAudioCodecs() []*core.Codec { + if len(c.skill.Audios) > 0 { + codecs := make([]*core.Codec, 0) + + for _, audio := range c.skill.Audios { + name := getAudioCodecName(&audio) + + codec := &core.Codec{ + Name: name, + ClockRate: uint32(audio.SampleRate), + Channels: uint8(audio.Channels), + } + codecs = append(codecs, codec) + } + + if len(codecs) > 0 { + return codecs + } + } + + return nil +} + +// https://protect-us.ismartlife.me/ +func getAudioCodecName(audioSkill *AudioSkill) string { + switch audioSkill.CodecType { + // case 100: + // return "ADPCM" + case 101: + return core.CodecPCML + case 102, 103, 104: + return core.CodecAAC + case 105: + return core.CodecPCMU + case 106: + return core.CodecPCMA + // case 107: + // return "G726-32" + // case 108: + // return "SPEEX" + case 109: + return core.CodecMP3 + default: + return core.CodecPCML + } +} + func (c *TuyaClient) getStreamType(streamRole string) int { // Default streamType if nothing is found defaultStreamType := 1 diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index ecf4d480..bebdacd5 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -129,13 +129,13 @@ func Dial(rawURL string) (core.Producer, error) { api, err := webrtc.NewAPI() if err != nil { - client.api.Close() + client.Stop() return nil, err } client.pc, err = api.NewPeerConnection(conf) if err != nil { - client.api.Close() + client.Stop() return nil, err } @@ -167,18 +167,27 @@ func Dial(rawURL string) (core.Producer, error) { } if err = client.conn.SetAnswer(answer.Sdp); err != nil { - client.Stop() + client.connected.Done(err) return } if client.isHEVC { - // Tuya answers always with H264 codec, replace with HEVC + // Tuya seems to answers always with H264 and PCMU/8000 and PCMA/8000 codecs, replace with real codecs + for _, media := range client.conn.Medias { if media.Kind == core.KindVideo { - for _, codec := range media.Codecs { - if codec.Name == core.CodecH264 { - codec.Name = core.CodecH265 - } + codecs := client.api.getVideoCodecs() + if codecs != nil { + media.Codecs = codecs + } + } + } + + for _, media := range client.conn.Medias { + if media.Kind == core.KindAudio { + codecs := client.api.getAudioCodecs() + if codecs != nil { + media.Codecs = codecs } } } @@ -197,6 +206,7 @@ func Dial(rawURL string) (core.Producer, error) { } client.api.mqtt.handleDisconnect = func() { + // fmt.Println("tuya: disconnect") client.Stop() } @@ -222,6 +232,7 @@ func Dial(rawURL string) (core.Producer, error) { } else { packet := &rtp.Packet{} if err := packet.Unmarshal(msg.Data); err != nil { + // skip return } @@ -260,7 +271,9 @@ func Dial(rawURL string) (core.Producer, error) { switch msg := msg.(type) { case *pion.ICECandidate: _ = sendOffer.Wait() - client.api.sendCandidate("a=" + msg.ToJSON().Candidate) + if err := client.api.sendCandidate("a=" + msg.ToJSON().Candidate); err != nil { + client.connected.Done(err) + } case pion.PeerConnectionState: switch msg { @@ -289,7 +302,7 @@ func Dial(rawURL string) (core.Producer, error) { // Create offer offer, err := client.conn.CreateOffer(medias) if err != nil { - client.api.Close() + client.Stop() return nil, err } @@ -299,7 +312,11 @@ func Dial(rawURL string) (core.Producer, error) { offer = re.ReplaceAllString(offer, "") // Send offer - client.api.sendOffer(offer, streamRole) + if err := client.api.sendOffer(offer, streamRole); err != nil { + client.Stop() + return nil, fmt.Errorf("tuya: %w", err) + } + sendOffer.Done(nil) // Wait for connection @@ -327,6 +344,8 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece return errors.New("webrtc: can't get track") } + _ = c.api.sendSpeaker(1) + payloadType := codec.PayloadType sender := core.NewSender(media, codec) @@ -376,6 +395,8 @@ func (c *Client) Stop() error { return nil } + c.closed = true + for ssrc := range c.handlers { delete(c.handlers, ssrc) } diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index ea69f60e..ebc27894 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -56,10 +56,10 @@ type CandidateFrame struct { Candidate string `json:"candidate"` } -type ResolutionFrame struct { - Mode string `json:"mode"` - Value int `json:"value"` // 0: HD, 1: SD -} +// type ResolutionFrame struct { +// Mode string `json:"mode"` +// Value int `json:"value"` // 0: HD, 1: SD +// } type SpeakerFrame struct { Mode string `json:"mode"` @@ -114,7 +114,7 @@ func (c *TuyaClient) StartMQTT() error { func (c *TuyaClient) StopMQTT() { if c.mqtt.client != nil { - c.sendDisconnect() + _ = c.sendDisconnect() c.mqtt.client.Disconnect(1000) } } @@ -202,7 +202,7 @@ func (c *TuyaMQTT) onError(err error) { } } -func (c *TuyaClient) sendOffer(sdp string, streamRole string) { +func (c *TuyaClient) sendOffer(sdp string, streamRole string) error { streamType := c.getStreamType(streamRole) isHEVC := c.isHEVC(streamType) @@ -215,53 +215,43 @@ func (c *TuyaClient) sendOffer(sdp string, streamRole string) { } } - err := c.sendMqttMessage("offer", 302, "", OfferFrame{ + return c.sendMqttMessage("offer", 302, "", OfferFrame{ Mode: "webrtc", Sdp: sdp, StreamType: streamType, Auth: c.auth, DatachannelEnable: isHEVC, }) - - if err != nil { - c.mqtt.onError(err) - return - } } -func (c *TuyaClient) sendCandidate(candidate string) { - err := c.sendMqttMessage("candidate", 302, "", CandidateFrame{ +func (c *TuyaClient) sendCandidate(candidate string) error { + return c.sendMqttMessage("candidate", 302, "", CandidateFrame{ Mode: "webrtc", Candidate: candidate, }) - - if err != nil { - c.mqtt.onError(err) - return - } } -func (c *TuyaClient) sendResolution(resolution int) { - isClaritySupperted := (c.skill.WebRTC & (1 << 5)) != 0 - if !isClaritySupperted { - return - } +// func (c *TuyaClient) sendResolution(resolution int) error { +// isClaritySupperted := (c.skill.WebRTC & (1 << 5)) != 0 +// if !isClaritySupperted { +// return nil +// } - c.sendMqttMessage("resolution", 302, "", ResolutionFrame{ - Mode: "webrtc", - Value: resolution, - }) -} +// return c.sendMqttMessage("resolution", 302, "", ResolutionFrame{ +// Mode: "webrtc", +// Value: resolution, +// }) +// } -func (c *TuyaClient) sendSpeaker(speaker int) { - c.sendMqttMessage("speaker", 302, "", SpeakerFrame{ +func (c *TuyaClient) sendSpeaker(speaker int) error { + return c.sendMqttMessage("speaker", 302, "", SpeakerFrame{ Mode: "webrtc", Value: speaker, }) } -func (c *TuyaClient) sendDisconnect() { - c.sendMqttMessage("disconnect", 302, "", DisconnectFrame{ +func (c *TuyaClient) sendDisconnect() error { + return c.sendMqttMessage("disconnect", 302, "", DisconnectFrame{ Mode: "webrtc", }) } From 8a8fb66eeb3de0dffb31c9bb5a1eb6d4b3b66345 Mon Sep 17 00:00:00 2001 From: seydx <34152761+seydx@users.noreply.github.com> Date: Sat, 17 May 2025 15:55:43 +0200 Subject: [PATCH 21/60] Update README.md Co-authored-by: Felipe Santos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 43e1c83b..7982cbf2 100644 --- a/README.md +++ b/README.md @@ -570,7 +570,7 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. [Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. -- Obtain `client_id`, `client_secret`, `uid` and `device_id` from [Tuya IoT Platform](https://iot.tuya.com/) +- Obtain `device_id`, `client_id`, `client_secret`, and `uid` (if using `mode=webrtc`) from [Tuya IoT Platform](https://iot.tuya.com/). [Here's a guide](https://xzetsubou.github.io/hass-localtuya/cloud_api/). - Use `mode` parameter to select the stream type: - `webrtc` - WebRTC stream (default) - `rtsp` - RTSP stream _(if available)_ From 691f6d9cddc273a900fa738e1cf95e097ec8aa84 Mon Sep 17 00:00:00 2001 From: seydx Date: Sat, 17 May 2025 15:57:06 +0200 Subject: [PATCH 22/60] update README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7982cbf2..ed2b96d2 100644 --- a/README.md +++ b/README.md @@ -575,7 +575,7 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. - `webrtc` - WebRTC stream (default) - `rtsp` - RTSP stream _(if available)_ - `hls` - HLS stream _(if available)_ -- Use `type` parameter to select the stream type: _(if available)_ +- Use `role` parameter to select the stream: - `main` - Main stream (default) - `sub` - Sub stream @@ -588,10 +588,10 @@ streams: tuya_webrtc_2: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&mode=webrtc # Tuya WebRTC stream (HD) - tuya_webrtc_hd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&type=main + tuya_webrtc_hd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&role=main # Tuya WebRTC stream (SD) - tuya_webrtc_sd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&type=sub + tuya_webrtc_sd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&role=sub # Using RTSP when available (no "uid" required) tuya_rtsp: tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=rtsp From f045f3fccd0cbc28b64a9f43328bdc4f022472bf Mon Sep 17 00:00:00 2001 From: seydx Date: Sat, 17 May 2025 18:19:59 +0200 Subject: [PATCH 23/60] dont expose raw url in stream info --- pkg/tuya/client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index bebdacd5..c67f8c47 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -150,7 +150,6 @@ func Dial(rawURL string) (core.Producer, error) { client.conn.FormatName = "tuya/webrtc" client.conn.Mode = core.ModeActiveProducer client.conn.Protocol = "mqtt" - client.conn.URL = rawURL // Set up MQTT handlers client.api.mqtt.handleAnswer = func(answer AnswerFrame) { From 1cc8b373dec7769318065e9c849e5a66f2ae1b57 Mon Sep 17 00:00:00 2001 From: seydx Date: Sun, 18 May 2025 05:11:41 +0200 Subject: [PATCH 24/60] change query --- README.md | 10 +++++----- pkg/tuya/README.md | 3 ++- pkg/tuya/api.go | 16 ++++++++-------- pkg/tuya/client.go | 12 ++++++------ pkg/tuya/mqtt.go | 8 ++++---- 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ed2b96d2..05d940fd 100644 --- a/README.md +++ b/README.md @@ -575,9 +575,9 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. - `webrtc` - WebRTC stream (default) - `rtsp` - RTSP stream _(if available)_ - `hls` - HLS stream _(if available)_ -- Use `role` parameter to select the stream: - - `main` - Main stream (default) - - `sub` - Sub stream +- Use `resolution` parameter to select the stream: + - `hd` - HD stream (default) + - `sd` - SD stream ```yaml streams: @@ -588,10 +588,10 @@ streams: tuya_webrtc_2: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&mode=webrtc # Tuya WebRTC stream (HD) - tuya_webrtc_hd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&role=main + tuya_webrtc_hd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=hd # Tuya WebRTC stream (SD) - tuya_webrtc_sd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&role=sub + tuya_webrtc_sd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=sd # Using RTSP when available (no "uid" required) tuya_rtsp: tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=rtsp diff --git a/pkg/tuya/README.md b/pkg/tuya/README.md index f5cbc814..137bd155 100644 --- a/pkg/tuya/README.md +++ b/pkg/tuya/README.md @@ -3,4 +3,5 @@ - https://developer.tuya.com/en/docs/iot/webrtc?id=Kacsd4x2hl0se - https://github.com/tuya/webrtc-demo-go - https://github.com/bacco007/HomeAssistantConfig/blob/master/custom_components/xtend_tuya/multi_manager/tuya_iot/ipc/webrtc/xt_tuya_iot_webrtc_manager.py -- https://ipc-us.ismartlife.me/ \ No newline at end of file +- https://ipc-us.ismartlife.me/ +- https://protect-us.ismartlife.me/ \ No newline at end of file diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index 3842ed02..3fb72c62 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -73,7 +73,7 @@ type AudioSkill struct { } type VideoSkill struct { - StreamType int `json:"streamType"` // 2 = main stream, 4 = sub stream + StreamType int `json:"streamType"` // 2 = main stream (hd), 4 = sub stream (sd) ProfileId string `json:"profileId,omitempty"` CodecType int `json:"codecType"` // 2 = H264, 4 = H265 Width int `json:"width"` @@ -157,7 +157,7 @@ type OpenIoTHubConfigResponse struct { Code int `json:"code,omitempty"` } -func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId string, clientSecret string, streamMode string, streamRole string) (*TuyaClient, error) { +func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId string, clientSecret string, streamMode string) (*TuyaClient, error) { client := &TuyaClient{ httpClient: &http.Client{Timeout: 5 * time.Second}, mqtt: &TuyaMQTT{waiter: core.Waiter{}}, @@ -182,7 +182,7 @@ func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId stri return nil, fmt.Errorf("failed to get HLS URL: %w", err) } } else { - if err := client.InitDevice(streamRole); err != nil { + if err := client.InitDevice(); err != nil { return nil, fmt.Errorf("failed to initialize device: %w", err) } @@ -227,7 +227,7 @@ func (c *TuyaClient) InitToken() (err error) { return nil } -func (c *TuyaClient) InitDevice(streamRole string) (err error) { +func (c *TuyaClient) InitDevice() (err error) { url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.apiURL, c.uid, c.deviceId) body, err := c.Request("GET", url, nil) @@ -448,7 +448,7 @@ func getAudioCodecName(audioSkill *AudioSkill) string { } } -func (c *TuyaClient) getStreamType(streamRole string) int { +func (c *TuyaClient) getStreamType(streamResolution string) int { // Default streamType if nothing is found defaultStreamType := 1 @@ -479,10 +479,10 @@ func (c *TuyaClient) getStreamType(streamRole string) int { } // Return the streamType based on the selection - switch streamRole { - case "main": + switch streamResolution { + case "hd": return highestResType - case "sub": + case "sd": return lowestResType default: return defaultStreamType diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index c67f8c47..5a865690 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -61,11 +61,11 @@ func Dial(rawURL string) (core.Producer, error) { uid := query.Get("uid") clientId := query.Get("client_id") clientSecret := query.Get("client_secret") - streamRole := query.Get("role") + streamResolution := query.Get("resolution") streamMode := query.Get("mode") - if streamRole == "" || (streamRole != "main" && streamRole != "sub") { - streamRole = "main" + if streamResolution == "" || (streamResolution != "hd" && streamResolution != "sd") { + streamResolution = "hd" } useRTSP := streamMode == "rtsp" @@ -97,7 +97,7 @@ func Dial(rawURL string) (core.Producer, error) { } // Initialize Tuya API client - tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientId, clientSecret, streamMode, streamRole) + tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientId, clientSecret, streamMode) if err != nil { return nil, fmt.Errorf("tuya: %w", err) } @@ -118,7 +118,7 @@ func Dial(rawURL string) (core.Producer, error) { } return streams.GetProducer(client.api.hlsURL) } else { - client.isHEVC = client.api.isHEVC(client.api.getStreamType(streamRole)) + client.isHEVC = client.api.isHEVC(client.api.getStreamType(streamResolution)) // Create a new PeerConnection conf := pion.Configuration{ @@ -311,7 +311,7 @@ func Dial(rawURL string) (core.Producer, error) { offer = re.ReplaceAllString(offer, "") // Send offer - if err := client.api.sendOffer(offer, streamRole); err != nil { + if err := client.api.sendOffer(offer, streamResolution); err != nil { client.Stop() return nil, fmt.Errorf("tuya: %w", err) } diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index ebc27894..6fd3d4a0 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -202,13 +202,13 @@ func (c *TuyaMQTT) onError(err error) { } } -func (c *TuyaClient) sendOffer(sdp string, streamRole string) error { - streamType := c.getStreamType(streamRole) +func (c *TuyaClient) sendOffer(sdp string, streamResolution string) error { + streamType := c.getStreamType(streamResolution) isHEVC := c.isHEVC(streamType) if isHEVC { - // On HEVC we use streamType 0 for main stream and 1 for sub stream - if streamRole == "main" { + // On HEVC we use streamType 0 for main stream (hd) and 1 for sub stream (sd) + if streamResolution == "hd" { streamType = 0 } else { streamType = 1 From 67dfc942a0ab8f854feaa56391b5ed3e087e6a4d Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 19 May 2025 15:41:37 +0200 Subject: [PATCH 25/60] update useful links --- pkg/tuya/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/tuya/README.md b/pkg/tuya/README.md index 137bd155..cc213a66 100644 --- a/pkg/tuya/README.md +++ b/pkg/tuya/README.md @@ -4,4 +4,5 @@ - https://github.com/tuya/webrtc-demo-go - https://github.com/bacco007/HomeAssistantConfig/blob/master/custom_components/xtend_tuya/multi_manager/tuya_iot/ipc/webrtc/xt_tuya_iot_webrtc_manager.py - https://ipc-us.ismartlife.me/ -- https://protect-us.ismartlife.me/ \ No newline at end of file +- https://protect-us.ismartlife.me/ +- https://github.com/tuya/tuya-device-sharing-sdk \ No newline at end of file From 998c85d6f5598f08d0e23c241756c8ef8c700ae6 Mon Sep 17 00:00:00 2001 From: seydx Date: Thu, 22 May 2025 00:05:49 +0200 Subject: [PATCH 26/60] - support adding cameras via interface - support qr code auth - support resolution change - support h265 - refactor code --- README.md | 59 +++-- internal/tuya/tuya.go | 201 ++++++++++++++++ pkg/tuya/README.md | 5 +- pkg/tuya/api.go | 507 --------------------------------------- pkg/tuya/client.go | 513 +++++++++++++++++++++------------------- pkg/tuya/cloud_api.go | 312 ++++++++++++++++++++++++ pkg/tuya/crypto.go | 134 +++++++++++ pkg/tuya/helper.go | 72 ++++++ pkg/tuya/interface.go | 259 ++++++++++++++++++++ pkg/tuya/mqtt.go | 267 +++++++++++---------- pkg/tuya/sharing_api.go | 473 ++++++++++++++++++++++++++++++++++++ www/add.html | 105 ++++++++ 12 files changed, 2003 insertions(+), 904 deletions(-) delete mode 100644 pkg/tuya/api.go create mode 100644 pkg/tuya/cloud_api.go create mode 100644 pkg/tuya/crypto.go create mode 100644 pkg/tuya/helper.go create mode 100644 pkg/tuya/interface.go create mode 100644 pkg/tuya/sharing_api.go diff --git a/README.md b/README.md index 05d940fd..00b9cc45 100644 --- a/README.md +++ b/README.md @@ -568,39 +568,60 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. #### Source: Tuya -[Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. +[Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. Go2rtc supports `Cloud API` and `Open API`. +The `Cloud API` requires setting up a cloud project in the Tuya Developer Platform to retrieve the required credentials. The `Open API` does not require a cloud project and the cameras can be added through the interface via QR code (user code required), but it does not support webrtc mode and two way audio. + +**Cloud API**: - Obtain `device_id`, `client_id`, `client_secret`, and `uid` (if using `mode=webrtc`) from [Tuya IoT Platform](https://iot.tuya.com/). [Here's a guide](https://xzetsubou.github.io/hass-localtuya/cloud_api/). -- Use `mode` parameter to select the stream type: - - `webrtc` - WebRTC stream (default) - - `rtsp` - RTSP stream _(if available)_ - - `hls` - HLS stream _(if available)_ -- Use `resolution` parameter to select the stream: + +**Open API**: +- To get your user code, open the Tuya Smart app or Smart Life app and go to `Profile` > `Settings` > `Account and Security` > `User Code` +- Open the Go2rtc interface and go to `Add` > `Tuya` and enter your `User Code` in the `User Code` field. Click on `Generate QR Code` and scan it with the Tuya Smart app or Smart Life app. After scanning, click on `Login`. All cameras in your home (not shared ones) will be listed in the Go2rtc interface. Copy/Paste stream URLs to your `go2rtc.yaml` file. + +**Configuring the stream:** +- Use `mode` parameter to select the stream type (not all cameras support all modes): + - `webrtc` - WebRTC stream _(default for `Cloud API`)_ + - `rtsp` - RTSP stream _(default for `Open API`)_ + - `hls` - HLS stream + - `flv` - FLV stream _(only available for `Open API`)_ + - `rtmp` - RTMP stream _(only available for `Open API`)_ + +- Use `resolution` parameter to select the stream (only available for `Cloud API` and not all cameras support `hd` stream): - `hd` - HD stream (default) - `sd` - SD stream ```yaml streams: - # Tuya WebRTC stream - tuya_webrtc: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX + # Cloud API: WebRTC stream + tuya_webrtc: + - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX - # Tuya WebRTC stream (same as above) - tuya_webrtc_2: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&mode=webrtc + # Cloud API: WebRTC stream (same as above) + tuya_webrtc_2: + - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&mode=webrtc - # Tuya WebRTC stream (HD) - tuya_webrtc_hd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=hd + # Cloud API: WebRTC stream (HD) + tuya_webrtc_hd: + - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=hd - # Tuya WebRTC stream (SD) - tuya_webrtc_sd: tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=sd + # Cloud API: WebRTC stream (SD) + tuya_webrtc_sd: + - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=sd - # Using RTSP when available (no "uid" required) - tuya_rtsp: tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=rtsp + # Cloud API: RTSP stream when available (no "uid" required) + tuya_rtsp: + - tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=rtsp - # Using HLS when available (no "uid" required) - tuya_hls: tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=hls + # Cloud API: HLS stream when available (no "uid" required) + tuya_hls: + - tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=hls + + # Open API: RTSP stream + tuya_openapi: + - tuya://apigw.tuyaeu.com?device_id=XXX&terminal_id=XXX&token=XXX&uid=XXX ``` - #### Source: GoPro *[New in v1.8.3](https://github.com/AlexxIT/go2rtc/releases/tag/v1.8.3)* diff --git a/internal/tuya/tuya.go b/internal/tuya/tuya.go index c3b34e4a..b5457253 100644 --- a/internal/tuya/tuya.go +++ b/internal/tuya/tuya.go @@ -1,13 +1,214 @@ package tuya import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/AlexxIT/go2rtc/internal/api" "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/tuya" ) +var users = make(map[string]tuya.LoginResponse) + func Init() { streams.HandleFunc("tuya", func(source string) (core.Producer, error) { return tuya.Dial(source) }) + + api.HandleFunc("api/tuya", apiTuya) +} + +func apiTuya(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + userCode := query.Get("user_code") + token := query.Get("token") + + if userCode == "" { + http.Error(w, "user_code is required", http.StatusBadRequest) + return + } + + var auth *tuya.LoginResponse + if loginResponse, ok := users[userCode]; ok { + expireTime := loginResponse.Timestamp + loginResponse.Result.ExpireTime + + if expireTime > time.Now().Unix() { + auth = &loginResponse + } else { + delete(users, userCode) + token = "" + } + } + + if auth == nil && token == "" { + qrCode, err := getQRCode(userCode) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // response qrCode + json.NewEncoder(w).Encode(map[string]interface{}{ + "qrCode": qrCode, + }) + + return + } + + if auth == nil && token != "" { + authResponse, err := login(userCode, token) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + auth = authResponse + } + + if auth == nil { + http.Error(w, "failed to get auth", http.StatusInternalServerError) + return + } + + tokenInfo := tuya.TokenInfo{ + AccessToken: auth.Result.AccessToken, + ExpireTime: auth.Timestamp + auth.Result.ExpireTime, + RefreshToken: auth.Result.RefreshToken, + } + + tokenInfoBase64, err := tuya.ToBase64(&tokenInfo) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + tuyaAPI, err := tuya.NewTuyaOpenApiClient( + strings.Replace(auth.Result.Endpoint, "https://", "", 1), + auth.Result.UID, + "", + auth.Result.TerminalID, + tokenInfo, + "", + ) + + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + devices, err := tuyaAPI.GetAllDevices() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var items []*api.Source + for _, device := range devices { + cleanQuery := url.Values{} + cleanQuery.Set("uid", auth.Result.UID) + cleanQuery.Set("token", tokenInfoBase64) + cleanQuery.Set("terminal_id", auth.Result.TerminalID) + cleanQuery.Set("device_id", device.ID) + + endpoint := strings.Replace(auth.Result.Endpoint, "https://", "tuya://", 1) + url := fmt.Sprintf("%s?%s", endpoint, cleanQuery.Encode()) + + items = append(items, &api.Source{ + Name: device.Name, + URL: url, + }) + } + + api.ResponseSources(w, items) +} + +func login(userCode string, qrCode string) (*tuya.LoginResponse, error) { + url := fmt.Sprintf("https://%s/v1.0/m/life/home-assistant/qrcode/tokens/%s?clientid=%s&usercode=%s", tuya.TUYA_HOST, qrCode, tuya.TUYA_CLIENT_ID, userCode) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + httpClient := &http.Client{ + Timeout: 10 * time.Second, + } + + response, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer response.Body.Close() + + res, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get QR code: %s", string(res)) + } + + var loginResponse tuya.LoginResponse + err = json.Unmarshal(res, &loginResponse) + if err != nil { + return nil, err + } + + if !loginResponse.Success { + return nil, fmt.Errorf("failed to login: %s", loginResponse.Msg) + } + + users[userCode] = loginResponse + + return &loginResponse, nil +} + +func getQRCode(userCode string) (string, error) { + url := fmt.Sprintf("https://%s/v1.0/m/life/home-assistant/qrcode/tokens?clientid=%s&schema=%s&usercode=%s", tuya.TUYA_HOST, tuya.TUYA_CLIENT_ID, tuya.TUYA_SCHEMA, userCode) + + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return "", err + } + + req.Header.Set("Content-Type", "text/plain") + + httpClient := &http.Client{ + Timeout: 10 * time.Second, + } + + response, err := httpClient.Do(req) + if err != nil { + return "", err + } + defer response.Body.Close() + + res, err := io.ReadAll(response.Body) + if err != nil { + return "", err + } + + if response.StatusCode != http.StatusOK { + return "", err + } + + var qrResponse tuya.QRResponse + err = json.Unmarshal(res, &qrResponse) + if err != nil { + return "", err + } + + if !qrResponse.Success { + return "", fmt.Errorf("failed to get QR code: %s", qrResponse.Msg) + } + + return qrResponse.Result.Code, nil } diff --git a/pkg/tuya/README.md b/pkg/tuya/README.md index cc213a66..f1936404 100644 --- a/pkg/tuya/README.md +++ b/pkg/tuya/README.md @@ -3,6 +3,7 @@ - https://developer.tuya.com/en/docs/iot/webrtc?id=Kacsd4x2hl0se - https://github.com/tuya/webrtc-demo-go - https://github.com/bacco007/HomeAssistantConfig/blob/master/custom_components/xtend_tuya/multi_manager/tuya_iot/ipc/webrtc/xt_tuya_iot_webrtc_manager.py +- https://github.com/tuya/tuya-device-sharing-sdk +- https://github.com/make-all/tuya-local/blob/main/custom_components/tuya_local/cloud.py - https://ipc-us.ismartlife.me/ -- https://protect-us.ismartlife.me/ -- https://github.com/tuya/tuya-device-sharing-sdk \ No newline at end of file +- https://protect-us.ismartlife.me/ \ No newline at end of file diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go deleted file mode 100644 index 3fb72c62..00000000 --- a/pkg/tuya/api.go +++ /dev/null @@ -1,507 +0,0 @@ -package tuya - -import ( - "bytes" - "crypto/md5" - "encoding/json" - "fmt" - "io" - "net/http" - "strconv" - "time" - - "github.com/AlexxIT/go2rtc/pkg/core" - "github.com/AlexxIT/go2rtc/pkg/webrtc" - "github.com/google/uuid" - pionWebrtc "github.com/pion/webrtc/v4" -) - -type TuyaClient struct { - httpClient *http.Client - mqtt *TuyaMQTT - apiURL string - rtspURL string - hlsURL string - sessionId string - clientId string - clientSecret string - deviceId string - accessToken string - refreshToken string - expireTime int64 - uid string - motoId string - auth string - skill *Skill - iceServers []pionWebrtc.ICEServer -} - -type Token struct { - UID string `json:"uid"` - AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` - ExpireTime int64 `json:"expire_time"` -} - -type AudioAttributes struct { - CallMode []int `json:"call_mode"` // 1 = one way, 2 = two way - HardwareCapability []int `json:"hardware_capability"` // 1 = mic, 2 = speaker -} - -type OpenApiICE struct { - Urls string `json:"urls"` - Username string `json:"username"` - Credential string `json:"credential"` - TTL int `json:"ttl"` -} - -type WebICE struct { - Urls string `json:"urls"` - Username string `json:"username,omitempty"` - Credential string `json:"credential,omitempty"` -} - -type P2PConfig struct { - Ices []OpenApiICE `json:"ices"` -} - -type AudioSkill struct { - Channels int `json:"channels"` - DataBit int `json:"dataBit"` - CodecType int `json:"codecType"` - SampleRate int `json:"sampleRate"` -} - -type VideoSkill struct { - StreamType int `json:"streamType"` // 2 = main stream (hd), 4 = sub stream (sd) - ProfileId string `json:"profileId,omitempty"` - CodecType int `json:"codecType"` // 2 = H264, 4 = H265 - Width int `json:"width"` - Height int `json:"height"` - SampleRate int `json:"sampleRate"` -} - -type Skill struct { - WebRTC int `json:"webrtc"` - Audios []AudioSkill `json:"audios"` - Videos []VideoSkill `json:"videos"` -} - -type WebRTConfig struct { - AudioAttributes AudioAttributes `json:"audio_attributes"` - Auth string `json:"auth"` - ID string `json:"id"` - MotoID string `json:"moto_id"` - P2PConfig P2PConfig `json:"p2p_config"` - ProtocolVersion string `json:"protocol_version"` - Skill string `json:"skill"` - SupportsWebRTCRecord bool `json:"supports_webrtc_record"` - SupportsWebRTC bool `json:"supports_webrtc"` - VedioClaritiy int `json:"vedio_clarity"` - VideoClaritiy int `json:"video_clarity"` - VideoClarities []int `json:"video_clarities"` -} - -type OpenIoTHubConfig struct { - Url string `json:"url"` - ClientID string `json:"client_id"` - Username string `json:"username"` - Password string `json:"password"` - SinkTopic struct { - IPC string `json:"ipc"` - } `json:"sink_topic"` - SourceSink struct { - IPC string `json:"ipc"` - } `json:"source_topic"` - ExpireTime int `json:"expire_time"` -} - -type WebRTCConfigResponse struct { - Success bool `json:"success"` - Result WebRTConfig `json:"result"` - Msg string `json:"msg,omitempty"` - Code int `json:"code,omitempty"` -} - -type TokenResponse struct { - Success bool `json:"success"` - Result Token `json:"result"` - Msg string `json:"msg,omitempty"` - Code int `json:"code,omitempty"` -} - -type AllocateRequest struct { - Type string `json:"type"` -} - -type AllocateResponse struct { - Success bool `json:"success"` - Result struct { - URL string `json:"url"` - } `json:"result"` - Msg string `json:"msg,omitempty"` - Code int `json:"code,omitempty"` -} - -type OpenIoTHubConfigRequest struct { - UID string `json:"uid"` - UniqueID string `json:"unique_id"` - LinkType string `json:"link_type"` - Topics string `json:"topics"` -} - -type OpenIoTHubConfigResponse struct { - Success bool `json:"success"` - Result OpenIoTHubConfig `json:"result"` - Msg string `json:"msg,omitempty"` - Code int `json:"code,omitempty"` -} - -func NewTuyaClient(openAPIURL string, deviceId string, uid string, clientId string, clientSecret string, streamMode string) (*TuyaClient, error) { - client := &TuyaClient{ - httpClient: &http.Client{Timeout: 5 * time.Second}, - mqtt: &TuyaMQTT{waiter: core.Waiter{}}, - apiURL: openAPIURL, - sessionId: core.RandString(6, 62), - clientId: clientId, - deviceId: deviceId, - clientSecret: clientSecret, - uid: uid, - } - - if err := client.InitToken(); err != nil { - return nil, fmt.Errorf("failed to initialize token: %w", err) - } - - if streamMode == "rtsp" { - if err := client.GetStreamUrl("rtsp"); err != nil { - return nil, fmt.Errorf("failed to get RTSP URL: %w", err) - } - } else if streamMode == "hls" { - if err := client.GetStreamUrl("hls"); err != nil { - return nil, fmt.Errorf("failed to get HLS URL: %w", err) - } - } else { - if err := client.InitDevice(); err != nil { - return nil, fmt.Errorf("failed to initialize device: %w", err) - } - - if err := client.StartMQTT(); err != nil { - return nil, fmt.Errorf("failed to start MQTT: %w", err) - } - } - - return client, nil -} - -func (c *TuyaClient) Close() { - c.StopMQTT() - c.httpClient.CloseIdleConnections() -} - -func (c *TuyaClient) InitToken() (err error) { - url := fmt.Sprintf("https://%s/v1.0/token?grant_type=1", c.apiURL) - - c.accessToken = "" - c.refreshToken = "" - - body, err := c.Request("GET", url, nil) - if err != nil { - return err - } - - var tokenResponse TokenResponse - err = json.Unmarshal(body, &tokenResponse) - if err != nil { - return err - } - - if !tokenResponse.Success { - return fmt.Errorf(tokenResponse.Msg) - } - - c.accessToken = tokenResponse.Result.AccessToken - c.refreshToken = tokenResponse.Result.RefreshToken - c.expireTime = tokenResponse.Result.ExpireTime - - return nil -} - -func (c *TuyaClient) InitDevice() (err error) { - url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.apiURL, c.uid, c.deviceId) - - body, err := c.Request("GET", url, nil) - if err != nil { - return err - } - - var webRTCConfigResponse WebRTCConfigResponse - err = json.Unmarshal(body, &webRTCConfigResponse) - if err != nil { - return err - } - - if !webRTCConfigResponse.Success { - return fmt.Errorf(webRTCConfigResponse.Msg) - } - - err = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), &c.skill) - if err != nil { - return err - } - - iceServers, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) - if err != nil { - return err - } - - c.iceServers, err = webrtc.UnmarshalICEServers(iceServers) - if err != nil { - return err - } - - c.motoId = webRTCConfigResponse.Result.MotoID - c.auth = webRTCConfigResponse.Result.Auth - - return nil -} - -func (c *TuyaClient) GetStreamUrl(streamType string) (err error) { - url := fmt.Sprintf("https://%s/v1.0/devices/%s/stream/actions/allocate", c.apiURL, c.deviceId) - - request := &AllocateRequest{ - Type: streamType, - } - - body, err := c.Request("POST", url, request) - if err != nil { - return err - } - - var allosResponse AllocateResponse - err = json.Unmarshal(body, &allosResponse) - if err != nil { - return err - } - - if !allosResponse.Success { - return fmt.Errorf(allosResponse.Msg) - } - - switch streamType { - case "rtsp": - c.rtspURL = allosResponse.Result.URL - case "hls": - c.hlsURL = allosResponse.Result.URL - default: - return fmt.Errorf("unsupported stream type: %s", streamType) - } - - return nil -} - -func (c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { - url := fmt.Sprintf("https://%s/v2.0/open-iot-hub/access/config", c.apiURL) - - request := &OpenIoTHubConfigRequest{ - UID: c.uid, - UniqueID: uuid.New().String(), - LinkType: "mqtt", - Topics: "ipc", - } - - body, err := c.Request("POST", url, request) - if err != nil { - return nil, err - } - - var openIoTHubConfigResponse OpenIoTHubConfigResponse - err = json.Unmarshal(body, &openIoTHubConfigResponse) - if err != nil { - return nil, err - } - - if !openIoTHubConfigResponse.Success { - return nil, fmt.Errorf(openIoTHubConfigResponse.Msg) - } - - return &openIoTHubConfigResponse.Result, nil -} - -func (c *TuyaClient) Request(method string, url string, body any) ([]byte, error) { - var bodyReader io.Reader - if body != nil { - jsonBody, err := json.Marshal(body) - if err != nil { - return nil, err - } - bodyReader = bytes.NewReader(jsonBody) - } - - req, err := http.NewRequest(method, url, bodyReader) - if err != nil { - return nil, err - } - - ts := time.Now().UnixNano() / 1000000 - sign := c.calBusinessSign(ts) - - req.Header.Set("Accept", "*") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Access-Control-Allow-Origin", "*") - req.Header.Set("Access-Control-Allow-Methods", "*") - req.Header.Set("Access-Control-Allow-Headers", "*") - req.Header.Set("mode", "no-cors") - req.Header.Set("client_id", c.clientId) - req.Header.Set("access_token", c.accessToken) - req.Header.Set("sign", sign) - req.Header.Set("t", strconv.FormatInt(ts, 10)) - - response, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - defer response.Body.Close() - - res, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - - if response.StatusCode != http.StatusOK { - return nil, err - } - - return res, nil -} - -func (c *TuyaClient) getVideoCodecs() []*core.Codec { - if len(c.skill.Videos) > 0 { - codecs := make([]*core.Codec, 0) - - for _, video := range c.skill.Videos { - name := core.CodecH264 - if c.isHEVC(video.StreamType) { - name = core.CodecH265 - } - - codec := &core.Codec{ - Name: name, - ClockRate: uint32(video.SampleRate), - } - - codecs = append(codecs, codec) - } - - if len(codecs) > 0 { - return codecs - } - } - - return nil -} - -func (c *TuyaClient) getAudioCodecs() []*core.Codec { - if len(c.skill.Audios) > 0 { - codecs := make([]*core.Codec, 0) - - for _, audio := range c.skill.Audios { - name := getAudioCodecName(&audio) - - codec := &core.Codec{ - Name: name, - ClockRate: uint32(audio.SampleRate), - Channels: uint8(audio.Channels), - } - codecs = append(codecs, codec) - } - - if len(codecs) > 0 { - return codecs - } - } - - return nil -} - -// https://protect-us.ismartlife.me/ -func getAudioCodecName(audioSkill *AudioSkill) string { - switch audioSkill.CodecType { - // case 100: - // return "ADPCM" - case 101: - return core.CodecPCML - case 102, 103, 104: - return core.CodecAAC - case 105: - return core.CodecPCMU - case 106: - return core.CodecPCMA - // case 107: - // return "G726-32" - // case 108: - // return "SPEEX" - case 109: - return core.CodecMP3 - default: - return core.CodecPCML - } -} - -func (c *TuyaClient) getStreamType(streamResolution string) int { - // Default streamType if nothing is found - defaultStreamType := 1 - - if c.skill == nil || len(c.skill.Videos) == 0 { - return defaultStreamType - } - - // Find the highest and lowest resolution - var highestResType = defaultStreamType - var highestRes = 0 - var lowestResType = defaultStreamType - var lowestRes = 0 - - for _, video := range c.skill.Videos { - res := video.Width * video.Height - - // Highest Resolution - if res > highestRes { - highestRes = res - highestResType = video.StreamType - } - - // Lower Resolution (or first if not set yet) - if lowestRes == 0 || res < lowestRes { - lowestRes = res - lowestResType = video.StreamType - } - } - - // Return the streamType based on the selection - switch streamResolution { - case "hd": - return highestResType - case "sd": - return lowestResType - default: - return defaultStreamType - } -} - -func (c *TuyaClient) isHEVC(streamType int) bool { - for _, video := range c.skill.Videos { - if video.StreamType == streamType { - return video.CodecType == 4 - } - } - - return false -} - -func (c *TuyaClient) calBusinessSign(ts int64) string { - data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts) - val := md5.Sum([]byte(data)) - res := fmt.Sprintf("%X", val) - return res -} diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 5a865690..4a43c49a 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -15,16 +15,17 @@ import ( ) type Client struct { - api *TuyaClient - conn *webrtc.Conn - pc *pion.PeerConnection - dc *pion.DataChannel - videoSSRC uint32 - audioSSRC uint32 - isHEVC bool - connected core.Waiter - closed bool - handlers map[uint32]func(*rtp.Packet) + api TuyaAPI + conn *webrtc.Conn + pc *pion.PeerConnection + dc *pion.DataChannel + videoSSRC uint32 + audioSSRC uint32 + streamType int + isHEVC bool + connected core.Waiter + closed bool + handlers map[uint32]func(*rtp.Packet) } type DataChannelMessage struct { @@ -41,15 +42,6 @@ type RecvMessage struct { } `json:"audio"` } -const ( - DefaultCnURL = "openapi.tuyacn.com" - DefaultWestUsURL = "openapi.tuyaus.com" - DefaultEastUsURL = "openapi-ueaz.tuyaus.com" - DefaultCentralEuURL = "openapi.tuyaeu.com" - DefaultWestEuURL = "openapi-weaz.tuyaeu.com" - DefaultInURL = "openapi.tuyain.com" -) - func Dial(rawURL string) (core.Producer, error) { u, err := url.Parse(rawURL) if err != nil { @@ -57,274 +49,292 @@ func Dial(rawURL string) (core.Producer, error) { } query := u.Query() - deviceID := query.Get("device_id") - uid := query.Get("uid") + + // Open API + tokenInfo := query.Get("token") + terminalId := query.Get("terminal_id") + + // Cloud API clientId := query.Get("client_id") clientSecret := query.Get("client_secret") + + // Shared params + deviceId := query.Get("device_id") + uid := query.Get("uid") + + // Stream params streamResolution := query.Get("resolution") streamMode := query.Get("mode") + useOpenApi := deviceId != "" && uid != "" && tokenInfo != "" && terminalId != "" + useCloudApi := deviceId != "" && ((streamMode == "webrtc" || streamMode == "") && uid != "") && clientId != "" && clientSecret != "" + if streamResolution == "" || (streamResolution != "hd" && streamResolution != "sd") { streamResolution = "hd" } - useRTSP := streamMode == "rtsp" - useHLS := streamMode == "hls" - useWebRTC := streamMode == "webrtc" || streamMode == "" - - // check if host is correct - switch u.Hostname() { - case DefaultCnURL: - case DefaultWestUsURL: - case DefaultEastUsURL: - case DefaultCentralEuURL: - case DefaultWestEuURL: - case DefaultInURL: - default: - return nil, fmt.Errorf("tuya: wrong host %s", u.Hostname()) + if streamMode == "" || (streamMode != "rtsp" && streamMode != "hls" && streamMode != "flv" && streamMode != "rtmp" && streamMode != "webrtc") { + if useOpenApi { + streamMode = "rtsp" + } else { + streamMode = "webrtc" + } } - if deviceID == "" || clientId == "" || clientSecret == "" { - return nil, errors.New("tuya: no device_id, client_id or client_secret") - } - - if useWebRTC && uid == "" { - return nil, errors.New("tuya: no uid") - } - - if !useRTSP && !useHLS && !useWebRTC { - return nil, errors.New("tuya: wrong stream type") - } - - // Initialize Tuya API client - tuyaAPI, err := NewTuyaClient(u.Hostname(), deviceID, uid, clientId, clientSecret, streamMode) - if err != nil { - return nil, fmt.Errorf("tuya: %w", err) + if !useOpenApi && !useCloudApi { + return nil, errors.New("tuya: wrong query params") } client := &Client{ - api: tuyaAPI, handlers: make(map[uint32]func(*rtp.Packet)), } - if useRTSP { - if client.api.rtspURL == "" { - return nil, errors.New("tuya: no rtsp url") + if useOpenApi { + if client.api, err = NewTuyaOpenApiClient(u.Hostname(), uid, deviceId, terminalId, tokenInfo, streamMode); err != nil { + return nil, fmt.Errorf("tuya: %w", err) } - return streams.GetProducer(client.api.rtspURL) - } else if useHLS { - if client.api.hlsURL == "" { - return nil, errors.New("tuya: no hls url") - } - return streams.GetProducer(client.api.hlsURL) } else { - client.isHEVC = client.api.isHEVC(client.api.getStreamType(streamResolution)) - - // Create a new PeerConnection - conf := pion.Configuration{ - ICEServers: client.api.iceServers, - ICETransportPolicy: pion.ICETransportPolicyAll, - BundlePolicy: pion.BundlePolicyMaxBundle, + if client.api, err = NewTuyaCloudApiClient(u.Hostname(), uid, deviceId, clientId, clientSecret, streamMode); err != nil { + return nil, fmt.Errorf("tuya: %w", err) } + } - api, err := webrtc.NewAPI() + if streamMode != "webrtc" { + streamUrl, err := client.api.GetStreamUrl(streamMode) if err != nil { - client.Stop() - return nil, err + return nil, fmt.Errorf("tuya: %w", err) } - client.pc, err = api.NewPeerConnection(conf) - if err != nil { - client.Stop() - return nil, err + return streams.GetProducer(streamUrl) + } + + if err := client.api.Init(); err != nil { + return nil, fmt.Errorf("tuya: %w", err) + } + + client.streamType = client.api.GetStreamType(streamResolution) + client.isHEVC = client.api.IsHEVC(client.streamType) + + // Create a new PeerConnection + conf := pion.Configuration{ + ICEServers: client.api.GetICEServers(), + ICETransportPolicy: pion.ICETransportPolicyAll, + BundlePolicy: pion.BundlePolicyMaxBundle, + } + + api, err := webrtc.NewAPI() + if err != nil { + client.Close(err) + return nil, err + } + + client.pc, err = api.NewPeerConnection(conf) + if err != nil { + client.Close(err) + return nil, err + } + + // protect from sending ICE candidate before Offer + var sendOffer core.Waiter + + // protect from blocking on errors + defer sendOffer.Done(nil) + + // Create new WebRTC connection + client.conn = webrtc.NewConn(client.pc) + client.conn.FormatName = "tuya/webrtc" + client.conn.Mode = core.ModeActiveProducer + client.conn.Protocol = "mqtt" + + mqttClient := client.api.GetMqtt() + if mqttClient == nil { + err = errors.New("tuya: no mqtt client") + client.Close(err) + return nil, err + } + + // Set up MQTT handlers + mqttClient.handleAnswer = func(answer AnswerFrame) { + // fmt.Printf("tuya: answer: %s\n", answer.Sdp) + + desc := pion.SessionDescription{ + Type: pion.SDPTypePranswer, + SDP: answer.Sdp, } - // protect from sending ICE candidate before Offer - var sendOffer core.Waiter - - // protect from blocking on errors - defer sendOffer.Done(nil) - - // Create new WebRTC connection - client.conn = webrtc.NewConn(client.pc) - client.conn.FormatName = "tuya/webrtc" - client.conn.Mode = core.ModeActiveProducer - client.conn.Protocol = "mqtt" - - // Set up MQTT handlers - client.api.mqtt.handleAnswer = func(answer AnswerFrame) { - // fmt.Printf("tuya: answer: %s\n", answer.Sdp) - - desc := pion.SessionDescription{ - Type: pion.SDPTypePranswer, - SDP: answer.Sdp, - } - - if err = client.pc.SetRemoteDescription(desc); err != nil { - client.connected.Done(err) - return - } - - if err = client.conn.SetAnswer(answer.Sdp); err != nil { - client.connected.Done(err) - return - } - - if client.isHEVC { - // Tuya seems to answers always with H264 and PCMU/8000 and PCMA/8000 codecs, replace with real codecs - - for _, media := range client.conn.Medias { - if media.Kind == core.KindVideo { - codecs := client.api.getVideoCodecs() - if codecs != nil { - media.Codecs = codecs - } - } - } - - for _, media := range client.conn.Medias { - if media.Kind == core.KindAudio { - codecs := client.api.getAudioCodecs() - if codecs != nil { - media.Codecs = codecs - } - } - } - } + if err = client.pc.SetRemoteDescription(desc); err != nil { + client.Close(err) + return } - client.api.mqtt.handleCandidate = func(candidate CandidateFrame) { - // fmt.Printf("tuya: candidate: %s\n", candidate.Candidate) - - if candidate.Candidate != "" { - client.conn.AddCandidate(candidate.Candidate) - if err != nil { - client.Stop() - } - } + if err = client.conn.SetAnswer(answer.Sdp); err != nil { + client.Close(err) + return } - client.api.mqtt.handleDisconnect = func() { - // fmt.Println("tuya: disconnect") - client.Stop() - } - - client.api.mqtt.handleError = func(err error) { - // fmt.Printf("tuya: error: %s\n", err.Error()) - client.Stop() - } - - // On HEVC, use DataChannel to receive video/audio if client.isHEVC { - // Create a new DataChannel - maxRetransmits := uint16(5) - ordered := true - client.dc, err = client.pc.CreateDataChannel("fmp4Stream", &pion.DataChannelInit{ - MaxRetransmits: &maxRetransmits, - Ordered: &ordered, - }) + // Tuya seems to answers always with H264 and PCMU/8000 and PCMA/8000 codecs, replace with real codecs - // Set up data channel handler - client.dc.OnMessage(func(msg pion.DataChannelMessage) { - if msg.IsString { - client.probe(msg) - } else { - packet := &rtp.Packet{} - if err := packet.Unmarshal(msg.Data); err != nil { - // skip - return - } - - if handler, ok := client.handlers[packet.SSRC]; ok { - handler(packet) + for _, media := range client.conn.Medias { + if media.Kind == core.KindVideo { + codecs := client.api.GetVideoCodecs() + if codecs != nil { + media.Codecs = codecs } } - }) + } - client.dc.OnError(func(err error) { - // fmt.Printf("tuya: datachannel error: %s\n", err.Error()) - client.connected.Done(err) - }) - - client.dc.OnClose(func() { - // fmt.Println("tuya: datachannel closed") - client.connected.Done(errors.New("datachannel: closed")) - }) - - client.dc.OnOpen(func() { - // fmt.Println("tuya: datachannel opened") - - codecRequest, _ := json.Marshal(DataChannelMessage{ - Type: "codec", - Msg: "", - }) - - if err := client.sendMessageToDataChannel(codecRequest); err != nil { - client.connected.Done(fmt.Errorf("failed to send codec request: %w", err)) + for _, media := range client.conn.Medias { + if media.Kind == core.KindAudio { + codecs := client.api.GetAudioCodecs() + if codecs != nil { + media.Codecs = codecs + } } - }) + } } + } - // Set up pc handler - client.conn.Listen(func(msg any) { - switch msg := msg.(type) { - case *pion.ICECandidate: - _ = sendOffer.Wait() - if err := client.api.sendCandidate("a=" + msg.ToJSON().Candidate); err != nil { - client.connected.Done(err) + mqttClient.handleCandidate = func(candidate CandidateFrame) { + // fmt.Printf("tuya: candidate: %s\n", candidate.Candidate) + + if candidate.Candidate != "" { + client.conn.AddCandidate(candidate.Candidate) + if err != nil { + client.Close(err) + } + } + } + + mqttClient.handleDisconnect = func() { + // fmt.Println("tuya: disconnect") + client.Close(errors.New("mqtt: disconnect")) + } + + mqttClient.handleError = func(err error) { + // fmt.Printf("tuya: error: %s\n", err.Error()) + client.Close(err) + } + + // On HEVC, use DataChannel to receive video/audio + if client.isHEVC { + // Create a new DataChannel + maxRetransmits := uint16(5) + ordered := true + client.dc, err = client.pc.CreateDataChannel("fmp4Stream", &pion.DataChannelInit{ + MaxRetransmits: &maxRetransmits, + Ordered: &ordered, + }) + + // Set up data channel handler + client.dc.OnMessage(func(msg pion.DataChannelMessage) { + if msg.IsString { + if connected, err := client.probe(msg); err != nil { + client.Close(err) + } else if connected { + client.connected.Done(nil) + } + } else { + packet := &rtp.Packet{} + if err := packet.Unmarshal(msg.Data); err != nil { + // skip + return } - case pion.PeerConnectionState: - switch msg { - case pion.PeerConnectionStateNew: - break - case pion.PeerConnectionStateConnecting: - break - case pion.PeerConnectionStateConnected: - // On HEVC, wait for DataChannel to be opened and camera to send codec info - if !client.isHEVC { - client.connected.Done(nil) - } - default: - client.Stop() - client.connected.Done(errors.New("webrtc: " + msg.String())) + if handler, ok := client.handlers[packet.SSRC]; ok { + handler(packet) } } }) - // Audio first, otherwise tuya will send corrupt sdp - medias := []*core.Media{ - {Kind: core.KindAudio, Direction: core.DirectionSendRecv}, - {Kind: core.KindVideo, Direction: core.DirectionRecvonly}, - } + client.dc.OnError(func(err error) { + // fmt.Printf("tuya: datachannel error: %s\n", err.Error()) + client.Close(err) + }) - // Create offer - offer, err := client.conn.CreateOffer(medias) - if err != nil { - client.Stop() - return nil, err - } + client.dc.OnClose(func() { + // fmt.Println("tuya: datachannel closed") + client.Close(errors.New("datachannel: closed")) + }) - // horter sdp, remove a=extmap... line, device ONLY allow 8KB json payload - // https://github.com/tuya/webrtc-demo-go/blob/04575054f18ccccb6bc9d82939dd46d449544e20/static/js/main.js#L224 - re := regexp.MustCompile(`\r\na=extmap[^\r\n]*`) - offer = re.ReplaceAllString(offer, "") + client.dc.OnOpen(func() { + // fmt.Println("tuya: datachannel opened") - // Send offer - if err := client.api.sendOffer(offer, streamResolution); err != nil { - client.Stop() - return nil, fmt.Errorf("tuya: %w", err) - } + codecRequest, _ := json.Marshal(DataChannelMessage{ + Type: "codec", + Msg: "", + }) - sendOffer.Done(nil) - - // Wait for connection - if err = client.connected.Wait(); err != nil { - return nil, fmt.Errorf("tuya: %w", err) - } - - return client, nil + if err := client.sendMessageToDataChannel(codecRequest); err != nil { + client.Close(fmt.Errorf("failed to send codec request: %w", err)) + } + }) } + + // Set up pc handler + client.conn.Listen(func(msg any) { + switch msg := msg.(type) { + case *pion.ICECandidate: + _ = sendOffer.Wait() + if err := mqttClient.SendCandidate("a=" + msg.ToJSON().Candidate); err != nil { + client.Close(err) + } + + case pion.PeerConnectionState: + switch msg { + case pion.PeerConnectionStateNew: + break + case pion.PeerConnectionStateConnecting: + break + case pion.PeerConnectionStateConnected: + // On HEVC, wait for DataChannel to be opened and camera to send codec info + if !client.isHEVC { + if streamResolution == "hd" { + _ = mqttClient.SendResolution(0) + } + client.connected.Done(nil) + } + default: + client.Close(errors.New("webrtc: " + msg.String())) + } + } + }) + + // Audio first, otherwise tuya will send corrupt sdp + medias := []*core.Media{ + {Kind: core.KindAudio, Direction: core.DirectionSendRecv}, + {Kind: core.KindVideo, Direction: core.DirectionRecvonly}, + } + + // Create offer + offer, err := client.conn.CreateOffer(medias) + if err != nil { + client.Close(err) + return nil, err + } + + // horter sdp, remove a=extmap... line, device ONLY allow 8KB json payload + // https://github.com/tuya/webrtc-demo-go/blob/04575054f18ccccb6bc9d82939dd46d449544e20/static/js/main.js#L224 + re := regexp.MustCompile(`\r\na=extmap[^\r\n]*`) + offer = re.ReplaceAllString(offer, "") + + // Send offer + if err := mqttClient.SendOffer(offer, streamResolution, client.streamType, client.isHEVC); err != nil { + err = fmt.Errorf("tuya: %w", err) + client.Close(err) + return nil, err + } + + sendOffer.Done(nil) + + // Wait for connection + if err = client.connected.Wait(); err != nil { + err = fmt.Errorf("tuya: %w", err) + client.Close(err) + return nil, err + } + + return client, nil } func (c *Client) GetMedias() []*core.Media { @@ -343,7 +353,10 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece return errors.New("webrtc: can't get track") } - _ = c.api.sendSpeaker(1) + mqttClient := c.api.GetMqtt() + if mqttClient != nil { + _ = mqttClient.SendSpeaker(1) + } payloadType := codec.PayloadType @@ -411,22 +424,25 @@ func (c *Client) Stop() error { return nil } +func (c *Client) Close(err error) error { + c.connected.Done(err) + return c.Stop() +} + func (c *Client) MarshalJSON() ([]byte, error) { return c.conn.MarshalJSON() } -func (c *Client) probe(msg pion.DataChannelMessage) { +func (c *Client) probe(msg pion.DataChannelMessage) (bool, error) { // fmt.Printf("[tuya] Received string message: %s\n", string(msg.Data)) var message DataChannelMessage if err := json.Unmarshal([]byte(msg.Data), &message); err != nil { - c.connected.Done(fmt.Errorf("failed to parse datachannel message: %w", err)) + return false, err } switch message.Type { case "codec": - // fmt.Printf("[tuya] Codec info from camera: %s\n", message.Msg) - frameRequest, _ := json.Marshal(DataChannelMessage{ Type: "start", Msg: "frame", @@ -434,14 +450,13 @@ func (c *Client) probe(msg pion.DataChannelMessage) { err := c.sendMessageToDataChannel(frameRequest) if err != nil { - c.connected.Done(fmt.Errorf("failed to send frame request: %w", err)) + return false, err } case "recv": var recvMessage RecvMessage if err := json.Unmarshal([]byte(message.Msg), &recvMessage); err != nil { - c.connected.Done(fmt.Errorf("failed to parse recv message: %w", err)) - return + return false, err } c.videoSSRC = recvMessage.Video.SSRC @@ -454,11 +469,13 @@ func (c *Client) probe(msg pion.DataChannelMessage) { err := c.sendMessageToDataChannel(completeMsg) if err != nil { - c.connected.Done(fmt.Errorf("failed to send complete message: %w", err)) + return false, err } - c.connected.Done(nil) + return true, nil } + + return false, nil } func (c *Client) sendMessageToDataChannel(message []byte) error { diff --git a/pkg/tuya/cloud_api.go b/pkg/tuya/cloud_api.go new file mode 100644 index 00000000..bd74daf0 --- /dev/null +++ b/pkg/tuya/cloud_api.go @@ -0,0 +1,312 @@ +package tuya + +import ( + "bytes" + "crypto/md5" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/AlexxIT/go2rtc/pkg/webrtc" + "github.com/google/uuid" +) + +type Token struct { + UID string `json:"uid"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpireTime int64 `json:"expire_time"` +} + +type WebRTCConfigResponse struct { + Timestamp int64 `json:"t"` + Success bool `json:"success"` + Result WebRTCConfig `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` +} + +type TokenResponse struct { + Timestamp int64 `json:"t"` + Success bool `json:"success"` + Result Token `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` +} + +type OpenIoTHubConfigRequest struct { + UID string `json:"uid"` + UniqueID string `json:"unique_id"` + LinkType string `json:"link_type"` + Topics string `json:"topics"` +} + +type OpenIoTHubConfig struct { + Url string `json:"url"` + ClientID string `json:"client_id"` + Username string `json:"username"` + Password string `json:"password"` + SinkTopic struct { + IPC string `json:"ipc"` + } `json:"sink_topic"` + SourceSink struct { + IPC string `json:"ipc"` + } `json:"source_topic"` + ExpireTime int `json:"expire_time"` +} + +type OpenIoTHubConfigResponse struct { + Timestamp int `json:"t"` + Success bool `json:"success"` + Result OpenIoTHubConfig `json:"result"` + Msg string `json:"msg,omitempty"` + Code int `json:"code,omitempty"` +} + +type TuyaCloudApiClient struct { + TuyaClient + clientId string + clientSecret string + refreshingToken bool +} + +func NewTuyaCloudApiClient(baseUrl string, uid string, deviceId string, clientId string, clientSecret string, streamMode string) (*TuyaCloudApiClient, error) { + mqttClient := NewTuyaMqttClient(deviceId) + + client := &TuyaCloudApiClient{ + TuyaClient: TuyaClient{ + httpClient: &http.Client{Timeout: 15 * time.Second}, + mqtt: mqttClient, + uid: uid, + deviceId: deviceId, + streamMode: streamMode, + expireTime: 0, + baseUrl: baseUrl, + }, + clientId: clientId, + clientSecret: clientSecret, + refreshingToken: false, + } + + return client, nil +} + +// WebRTC Flow +func (c *TuyaCloudApiClient) Init() error { + if err := c.initToken(); err != nil { + return fmt.Errorf("failed to initialize token: %w", err) + } + + webrtcConfig, err := c.loadWebrtcConfig() + if err != nil { + return fmt.Errorf("failed to load webrtc config: %w", err) + } + + hubConfig, err := c.loadHubConfig() + if err != nil { + return fmt.Errorf("failed to load hub config: %w", err) + } + + if err := c.mqtt.Start(hubConfig, webrtcConfig, c.skill.WebRTC); err != nil { + return fmt.Errorf("failed to start MQTT: %w", err) + } + + return nil +} + +func (c *TuyaCloudApiClient) GetStreamUrl(streamType string) (streamUrl string, err error) { + if err := c.initToken(); err != nil { + return "", fmt.Errorf("failed to initialize token: %w", err) + } + + url := fmt.Sprintf("https://%s/v1.0/devices/%s/stream/actions/allocate", c.baseUrl, c.deviceId) + + request := &AllocateRequest{ + Type: streamType, + } + + body, err := c.request("POST", url, request) + if err != nil { + return "", err + } + + var allocResponse AllocateResponse + err = json.Unmarshal(body, &allocResponse) + if err != nil { + return "", err + } + + if !allocResponse.Success { + return "", fmt.Errorf(allocResponse.Msg) + } + + return allocResponse.Result.URL, nil +} + +func (c *TuyaCloudApiClient) initToken() (err error) { + if c.refreshingToken { + return nil + } + + now := time.Now().Unix() + if (c.expireTime - 60) > now { + return nil + } + + c.refreshingToken = true + + url := fmt.Sprintf("https://%s/v1.0/token?grant_type=1", c.baseUrl) + + c.accessToken = "" + c.refreshToken = "" + + body, err := c.request("GET", url, nil) + if err != nil { + return err + } + + var tokenResponse TokenResponse + err = json.Unmarshal(body, &tokenResponse) + if err != nil { + return err + } + + if !tokenResponse.Success { + return fmt.Errorf(tokenResponse.Msg) + } + + c.accessToken = tokenResponse.Result.AccessToken + c.refreshToken = tokenResponse.Result.RefreshToken + c.expireTime = tokenResponse.Timestamp + tokenResponse.Result.ExpireTime + c.refreshingToken = false + + return nil +} + +func (c *TuyaCloudApiClient) loadWebrtcConfig() (*WebRTCConfig, error) { + url := fmt.Sprintf("https://%s/v1.0/users/%s/devices/%s/webrtc-configs", c.baseUrl, c.uid, c.deviceId) + + body, err := c.request("GET", url, nil) + if err != nil { + return nil, err + } + + var webRTCConfigResponse WebRTCConfigResponse + err = json.Unmarshal(body, &webRTCConfigResponse) + if err != nil { + return nil, err + } + + if !webRTCConfigResponse.Success { + return nil, fmt.Errorf(webRTCConfigResponse.Msg) + } + + err = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), &c.skill) + if err != nil { + return nil, err + } + + iceServers, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) + if err != nil { + return nil, err + } + + c.iceServers, err = webrtc.UnmarshalICEServers(iceServers) + if err != nil { + return nil, err + } + + return &webRTCConfigResponse.Result, nil +} + +func (c *TuyaCloudApiClient) loadHubConfig() (config *MQTTConfig, err error) { + url := fmt.Sprintf("https://%s/v2.0/open-iot-hub/access/config", c.baseUrl) + + request := &OpenIoTHubConfigRequest{ + UID: c.uid, + UniqueID: uuid.New().String(), + LinkType: "mqtt", + Topics: "ipc", + } + + body, err := c.request("POST", url, request) + if err != nil { + return nil, err + } + + var openIoTHubConfigResponse OpenIoTHubConfigResponse + err = json.Unmarshal(body, &openIoTHubConfigResponse) + if err != nil { + return nil, err + } + + if !openIoTHubConfigResponse.Success { + return nil, fmt.Errorf(openIoTHubConfigResponse.Msg) + } + + return &MQTTConfig{ + Url: openIoTHubConfigResponse.Result.Url, + Username: openIoTHubConfigResponse.Result.Username, + Password: openIoTHubConfigResponse.Result.Password, + ClientID: openIoTHubConfigResponse.Result.ClientID, + PublishTopic: openIoTHubConfigResponse.Result.SinkTopic.IPC, + SubscribeTopic: openIoTHubConfigResponse.Result.SourceSink.IPC, + }, nil +} + +func (c *TuyaCloudApiClient) request(method string, url string, body any) ([]byte, error) { + var bodyReader io.Reader + if body != nil { + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(jsonBody) + } + + req, err := http.NewRequest(method, url, bodyReader) + if err != nil { + return nil, err + } + + ts := time.Now().UnixNano() / 1000000 + sign := c.calBusinessSign(ts) + + req.Header.Set("Accept", "*") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Access-Control-Allow-Origin", "*") + req.Header.Set("Access-Control-Allow-Methods", "*") + req.Header.Set("Access-Control-Allow-Headers", "*") + req.Header.Set("mode", "no-cors") + req.Header.Set("client_id", c.clientId) + req.Header.Set("access_token", c.accessToken) + req.Header.Set("sign", sign) + req.Header.Set("t", strconv.FormatInt(ts, 10)) + + response, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer response.Body.Close() + + res, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + if response.StatusCode != http.StatusOK { + return nil, err + } + + return res, nil +} + +func (c *TuyaCloudApiClient) calBusinessSign(ts int64) string { + data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts) + val := md5.Sum([]byte(data)) + res := fmt.Sprintf("%X", val) + return res +} diff --git a/pkg/tuya/crypto.go b/pkg/tuya/crypto.go new file mode 100644 index 00000000..b8f84615 --- /dev/null +++ b/pkg/tuya/crypto.go @@ -0,0 +1,134 @@ +package tuya + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "math/rand" +) + +// https://github.com/tuya/tuya-device-sharing-sdk/blob/main/tuya_sharing/customerapi.py +func AesGCMEncrypt(rawData string, secret string) (string, error) { + nonce := []byte(RandomNonce(12)) + + block, err := aes.NewCipher([]byte(secret)) + if err != nil { + return "", err + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return "", err + } + + ciphertext := aesgcm.Seal(nil, nonce, []byte(rawData), nil) + nonceB64 := base64.StdEncoding.EncodeToString(nonce) + ciphertextB64 := base64.StdEncoding.EncodeToString(ciphertext) + + return nonceB64 + ciphertextB64, nil +} + +func AesGCMDecrypt(cipherData string, secret string) (string, error) { + if len(cipherData) <= 16 { + return "", fmt.Errorf("invalid ciphertext length") + } + + nonceB64 := cipherData[:16] + ciphertextB64 := cipherData[16:] + + nonce, err := base64.StdEncoding.DecodeString(nonceB64) + if err != nil { + return "", err + } + + ciphertext, err := base64.StdEncoding.DecodeString(ciphertextB64) + if err != nil { + return "", err + } + + block, err := aes.NewCipher([]byte(secret)) + if err != nil { + return "", err + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return "", err + } + + plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return "", err + } + + return string(plaintext), nil +} + +func SecretGenerating(rid, sid, hashKey string) string { + message := hashKey + mod := 16 + + if sid != "" { + sidLength := len(sid) + length := sidLength + if length > mod { + length = mod + } + + ecode := "" + for i := 0; i < length; i++ { + idx := int(sid[i]) % mod + ecode += string(sid[idx]) + } + message += "_" + message += ecode + } + + h := hmac.New(sha256.New, []byte(rid)) + h.Write([]byte(message)) + byteTemp := h.Sum(nil) + secret := hex.EncodeToString(byteTemp) + + return secret[:16] +} + +func RestfulSign(hashKey, queryEncdata, bodyEncdata string, data map[string]string) string { + headers := []string{"X-appKey", "X-requestId", "X-sid", "X-time", "X-token"} + headerSignStr := "" + + for _, item := range headers { + val, exists := data[item] + if exists && val != "" { + headerSignStr += item + "=" + val + "||" + } + } + + signStr := "" + if len(headerSignStr) > 2 { + signStr = headerSignStr[:len(headerSignStr)-2] + } + + if queryEncdata != "" { + signStr += queryEncdata + } + if bodyEncdata != "" { + signStr += bodyEncdata + } + + h := hmac.New(sha256.New, []byte(hashKey)) + h.Write([]byte(signStr)) + return hex.EncodeToString(h.Sum(nil)) +} + +func RandomNonce(length int) string { + const charset = "ABCDEFGHJKMNPQRSTWXYZabcdefhijkmnprstwxyz2345678" + result := make([]byte, length) + for i := range result { + result[i] = charset[rand.Intn(len(charset))] + } + return string(result) +} diff --git a/pkg/tuya/helper.go b/pkg/tuya/helper.go new file mode 100644 index 00000000..0b97b256 --- /dev/null +++ b/pkg/tuya/helper.go @@ -0,0 +1,72 @@ +package tuya + +import ( + "encoding/base64" + "encoding/json" + "fmt" +) + +func FormToJSON(content any) string { + if content == nil { + return "{}" + } + + jsonBytes, err := json.Marshal(content) + if err != nil { + return "{}" + } + + return string(jsonBytes) +} + +func ToBase64(tokenInfo *TokenInfo) (string, error) { + jsonData, err := json.Marshal(tokenInfo) + if err != nil { + return "", fmt.Errorf("error marshalling token: %v", err) + } + + encoded := base64.URLEncoding.EncodeToString(jsonData) + + return encoded, nil +} + +func FromBase64(encodedTokenInfo string) (*TokenInfo, error) { + jsonData, err := base64.URLEncoding.DecodeString(encodedTokenInfo) + if err != nil { + return nil, fmt.Errorf("error decoding token: %v", err) + } + + var tokenInfo TokenInfo + err = json.Unmarshal(jsonData, &tokenInfo) + if err != nil { + return nil, fmt.Errorf("error unmarshalling token: %v", err) + } + + return &tokenInfo, nil +} + +func ParseTokenInfo(tokenInfoOrString any) (*TokenInfo, error) { + var tokenInfo *TokenInfo + var err error + + switch v := tokenInfoOrString.(type) { + case string: + tokenInfo, err = FromBase64(v) + if err != nil { + return nil, fmt.Errorf("failed to decode base64 token: %w", err) + } + case *TokenInfo: + tokenInfo = v + case TokenInfo: + copyOfV := v + tokenInfo = ©OfV + default: + return nil, fmt.Errorf("invalid type: %T", v) + } + + if tokenInfo == nil { + return nil, fmt.Errorf("token info is nil") + } + + return tokenInfo, nil +} diff --git a/pkg/tuya/interface.go b/pkg/tuya/interface.go new file mode 100644 index 00000000..dae19a82 --- /dev/null +++ b/pkg/tuya/interface.go @@ -0,0 +1,259 @@ +package tuya + +import ( + "net/http" + + "github.com/AlexxIT/go2rtc/pkg/core" + pionWebrtc "github.com/pion/webrtc/v4" +) + +type TuyaAPI interface { + GetMqtt() *TuyaMqttClient + + GetStreamType(streamResolution string) int + IsHEVC(streamType int) bool + + GetVideoCodecs() []*core.Codec + GetAudioCodecs() []*core.Codec + + GetStreamUrl(streamUrl string) (string, error) + GetICEServers() []pionWebrtc.ICEServer + + Init() error + Close() +} + +type TuyaClient struct { + TuyaAPI + + httpClient *http.Client + mqtt *TuyaMqttClient + streamMode string + baseUrl string + accessToken string + refreshToken string + expireTime int64 + deviceId string + uid string + skill *Skill + iceServers []pionWebrtc.ICEServer +} + +type AudioAttributes struct { + CallMode []int `json:"call_mode"` // 1 = one way, 2 = two way + HardwareCapability []int `json:"hardware_capability"` // 1 = mic, 2 = speaker +} + +type OpenApiICE struct { + Urls string `json:"urls"` + Username string `json:"username"` + Credential string `json:"credential"` + TTL int `json:"ttl"` +} + +type WebICE struct { + Urls string `json:"urls"` + Username string `json:"username,omitempty"` + Credential string `json:"credential,omitempty"` +} + +type P2PConfig struct { + Ices []OpenApiICE `json:"ices"` +} + +type AudioSkill struct { + Channels int `json:"channels"` + DataBit int `json:"dataBit"` + CodecType int `json:"codecType"` + SampleRate int `json:"sampleRate"` +} + +type VideoSkill struct { + StreamType int `json:"streamType"` // 2 = main stream (hd), 4 = sub stream (sd) + ProfileId string `json:"profileId,omitempty"` + CodecType int `json:"codecType"` // 2 = H264, 4 = H265 + Width int `json:"width"` + Height int `json:"height"` + SampleRate int `json:"sampleRate"` +} + +type Skill struct { + WebRTC int `json:"webrtc"` + Audios []AudioSkill `json:"audios"` + Videos []VideoSkill `json:"videos"` +} + +type WebRTCConfig struct { + AudioAttributes AudioAttributes `json:"audio_attributes"` + Auth string `json:"auth"` + ID string `json:"id"` + MotoID string `json:"moto_id"` + P2PConfig P2PConfig `json:"p2p_config"` + ProtocolVersion string `json:"protocol_version"` + Skill string `json:"skill"` + SupportsWebRTCRecord bool `json:"supports_webrtc_record"` + SupportsWebRTC bool `json:"supports_webrtc"` + VedioClaritiy int `json:"vedio_clarity"` + VideoClaritiy int `json:"video_clarity"` + VideoClarities []int `json:"video_clarities"` +} + +type MQTTConfig struct { + Url string `json:"url"` + PublishTopic string `json:"publish_topic"` + SubscribeTopic string `json:"subscribe_topic"` + ClientID string `json:"client_id"` + Username string `json:"username"` + Password string `json:"password"` +} + +type Allocate struct { + URL string `json:"url"` +} + +type AllocateRequest struct { + Type string `json:"type"` +} + +type AllocateResponse struct { + Success bool `json:"success"` + Result Allocate `json:"result"` + Msg string `json:"msg,omitempty"` +} + +func (c *TuyaClient) GetICEServers() []pionWebrtc.ICEServer { + return c.iceServers +} + +func (c *TuyaClient) GetMqtt() *TuyaMqttClient { + return c.mqtt +} + +func (c *TuyaClient) GetStreamType(streamResolution string) int { + // Default streamType if nothing is found + defaultStreamType := 1 + + if c.skill == nil || len(c.skill.Videos) == 0 { + return defaultStreamType + } + + // Find the highest and lowest resolution + var highestResType = defaultStreamType + var highestRes = 0 + var lowestResType = defaultStreamType + var lowestRes = 0 + + for _, video := range c.skill.Videos { + res := video.Width * video.Height + + // Highest Resolution + if res > highestRes { + highestRes = res + highestResType = video.StreamType + } + + // Lower Resolution (or first if not set yet) + if lowestRes == 0 || res < lowestRes { + lowestRes = res + lowestResType = video.StreamType + } + } + + // Return the streamType based on the selection + switch streamResolution { + case "hd": + return highestResType + case "sd": + return lowestResType + default: + return defaultStreamType + } +} + +func (c *TuyaClient) IsHEVC(streamType int) bool { + for _, video := range c.skill.Videos { + if video.StreamType == streamType { + return video.CodecType == 4 + } + } + + return false +} + +func (c *TuyaClient) GetVideoCodecs() []*core.Codec { + if len(c.skill.Videos) > 0 { + codecs := make([]*core.Codec, 0) + + for _, video := range c.skill.Videos { + name := core.CodecH264 + if c.IsHEVC(video.StreamType) { + name = core.CodecH265 + } + + codec := &core.Codec{ + Name: name, + ClockRate: uint32(video.SampleRate), + } + + codecs = append(codecs, codec) + } + + if len(codecs) > 0 { + return codecs + } + } + + return nil +} + +func (c *TuyaClient) GetAudioCodecs() []*core.Codec { + if len(c.skill.Audios) > 0 { + codecs := make([]*core.Codec, 0) + + for _, audio := range c.skill.Audios { + name := getAudioCodecName(&audio) + + codec := &core.Codec{ + Name: name, + ClockRate: uint32(audio.SampleRate), + Channels: uint8(audio.Channels), + } + codecs = append(codecs, codec) + } + + if len(codecs) > 0 { + return codecs + } + } + + return nil +} + +func (c *TuyaClient) Close() { + c.mqtt.Stop() + c.httpClient.CloseIdleConnections() +} + +// https://protect-us.ismartlife.me/ +func getAudioCodecName(audioSkill *AudioSkill) string { + switch audioSkill.CodecType { + // case 100: + // return "ADPCM" + case 101: + return core.CodecPCML + case 102, 103, 104: + return core.CodecAAC + case 105: + return core.CodecPCMU + case 106: + return core.CodecPCMA + // case 107: + // return "G726-32" + // case 108: + // return "SPEEX" + case 109: + return core.CodecMP3 + default: + return core.CodecPCML + } +} diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index 6fd3d4a0..deb90b9e 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -10,13 +10,18 @@ import ( mqtt "github.com/eclipse/paho.mqtt.golang" ) -type TuyaMQTT struct { +type TuyaMqttClient struct { client mqtt.Client waiter core.Waiter publishTopic string subscribeTopic string + auth string uid string + motoId string + deviceId string + sessionId string closed bool + webrtcVersion int handleAnswer func(answer AnswerFrame) handleCandidate func(candidate CandidateFrame) handleDisconnect func() @@ -56,14 +61,14 @@ type CandidateFrame struct { Candidate string `json:"candidate"` } -// type ResolutionFrame struct { -// Mode string `json:"mode"` -// Value int `json:"value"` // 0: HD, 1: SD -// } +type ResolutionFrame struct { + Mode string `json:"mode"` + Value int `json:"cmdValue"` // 0: HD, 1: SD +} type SpeakerFrame struct { Mode string `json:"mode"` - Value int `json:"value"` // 0: off, 1: on + Value int `json:"cmdValue"` // 0: off, 1: on } type DisconnectFrame struct { @@ -77,20 +82,27 @@ type MqttMessage struct { Data MqttFrame `json:"data"` } -func (c *TuyaClient) StartMQTT() error { - hubConfig, err := c.LoadHubConfig() - if err != nil { - return err +func NewTuyaMqttClient(deviceId string) *TuyaMqttClient { + return &TuyaMqttClient{ + deviceId: deviceId, + sessionId: core.RandString(6, 62), + waiter: core.Waiter{}, } +} - c.mqtt.publishTopic = hubConfig.SinkTopic.IPC - c.mqtt.subscribeTopic = hubConfig.SourceSink.IPC +func (c *TuyaMqttClient) Start(hubConfig *MQTTConfig, webrtcConfig *WebRTCConfig, webrtcVersion int) error { + c.webrtcVersion = webrtcVersion + c.motoId = webrtcConfig.MotoID + c.auth = webrtcConfig.Auth - c.mqtt.publishTopic = strings.Replace(c.mqtt.publishTopic, "moto_id", c.motoId, 1) - c.mqtt.publishTopic = strings.Replace(c.mqtt.publishTopic, "{device_id}", c.deviceId, 1) + c.publishTopic = hubConfig.PublishTopic + c.subscribeTopic = hubConfig.SubscribeTopic - parts := strings.Split(c.mqtt.subscribeTopic, "/") - c.mqtt.uid = parts[3] + c.publishTopic = strings.Replace(c.publishTopic, "moto_id", c.motoId, 1) + c.publishTopic = strings.Replace(c.publishTopic, "{device_id}", c.deviceId, 1) + + parts := strings.Split(c.subscribeTopic, "/") + c.uid = parts[3] opts := mqtt.NewClientOptions().AddBroker(hubConfig.Url). SetClientID(hubConfig.ClientID). @@ -99,113 +111,27 @@ func (c *TuyaClient) StartMQTT() error { SetOnConnectHandler(c.onConnect). SetConnectTimeout(10 * time.Second) - c.mqtt.client = mqtt.NewClient(opts) + c.client = mqtt.NewClient(opts) - if token := c.mqtt.client.Connect(); token.Wait() && token.Error() != nil { + if token := c.client.Connect(); token.Wait() && token.Error() != nil { return token.Error() } - if err := c.mqtt.waiter.Wait(); err != nil { + if err := c.waiter.Wait(); err != nil { return err } return nil } -func (c *TuyaClient) StopMQTT() { - if c.mqtt.client != nil { - _ = c.sendDisconnect() - c.mqtt.client.Disconnect(1000) +func (c *TuyaMqttClient) Stop() { + if c.client != nil { + _ = c.SendDisconnect() + c.client.Disconnect(1000) } } -func (c *TuyaClient) onConnect(client mqtt.Client) { - if token := client.Subscribe(c.mqtt.subscribeTopic, 1, c.consume); token.Wait() && token.Error() != nil { - c.mqtt.waiter.Done(token.Error()) - return - } - - c.mqtt.waiter.Done(nil) -} - -func (c *TuyaClient) consume(client mqtt.Client, msg mqtt.Message) { - var rmqtt MqttMessage - if err := json.Unmarshal(msg.Payload(), &rmqtt); err != nil { - c.mqtt.onError(err) - return - } - - if rmqtt.Data.Header.SessionID != c.sessionId { - return - } - - switch rmqtt.Data.Header.Type { - case "answer": - c.mqtt.onMqttAnswer(&rmqtt) - case "candidate": - c.mqtt.onMqttCandidate(&rmqtt) - case "disconnect": - c.mqtt.onMqttDisconnect() - } -} - -func (c *TuyaMQTT) onMqttAnswer(msg *MqttMessage) { - var answerFrame AnswerFrame - if err := json.Unmarshal(msg.Data.Message, &answerFrame); err != nil { - c.onError(err) - return - } - - c.onAnswer(answerFrame) -} - -func (c *TuyaMQTT) onMqttCandidate(msg *MqttMessage) { - var candidateFrame CandidateFrame - if err := json.Unmarshal(msg.Data.Message, &candidateFrame); err != nil { - c.onError(err) - return - } - - // candidate from device start with "a=", end with "\r\n", which are not needed by Chrome webRTC - candidateFrame.Candidate = strings.TrimPrefix(candidateFrame.Candidate, "a=") - candidateFrame.Candidate = strings.TrimSuffix(candidateFrame.Candidate, "\r\n") - - c.onCandidate(candidateFrame) -} - -func (c *TuyaMQTT) onMqttDisconnect() { - c.closed = true - c.onDisconnect() -} - -func (c *TuyaMQTT) onAnswer(answer AnswerFrame) { - if c.handleAnswer != nil { - c.handleAnswer(answer) - } -} - -func (c *TuyaMQTT) onCandidate(candidate CandidateFrame) { - if c.handleCandidate != nil { - c.handleCandidate(candidate) - } -} - -func (c *TuyaMQTT) onDisconnect() { - if c.handleDisconnect != nil { - c.handleDisconnect() - } -} - -func (c *TuyaMQTT) onError(err error) { - if c.handleError != nil { - c.handleError(err) - } -} - -func (c *TuyaClient) sendOffer(sdp string, streamResolution string) error { - streamType := c.getStreamType(streamResolution) - isHEVC := c.isHEVC(streamType) - +func (c *TuyaMqttClient) SendOffer(sdp string, streamResolution string, streamType int, isHEVC bool) error { if isHEVC { // On HEVC we use streamType 0 for main stream (hd) and 1 for sub stream (sd) if streamResolution == "hd" { @@ -224,40 +150,125 @@ func (c *TuyaClient) sendOffer(sdp string, streamResolution string) error { }) } -func (c *TuyaClient) sendCandidate(candidate string) error { +func (c *TuyaMqttClient) SendCandidate(candidate string) error { return c.sendMqttMessage("candidate", 302, "", CandidateFrame{ Mode: "webrtc", Candidate: candidate, }) } -// func (c *TuyaClient) sendResolution(resolution int) error { -// isClaritySupperted := (c.skill.WebRTC & (1 << 5)) != 0 -// if !isClaritySupperted { -// return nil -// } +func (c *TuyaMqttClient) SendResolution(resolution int) error { + // isClaritySupperted := (c.webrtcVersion & (1 << 5)) != 0 + // if !isClaritySupperted { + // return nil + // } -// return c.sendMqttMessage("resolution", 302, "", ResolutionFrame{ -// Mode: "webrtc", -// Value: resolution, -// }) -// } + // Protocol 312 is used for clarity + return c.sendMqttMessage("resolution", 312, "", ResolutionFrame{ + Mode: "webrtc", + Value: resolution, + }) +} -func (c *TuyaClient) sendSpeaker(speaker int) error { - return c.sendMqttMessage("speaker", 302, "", SpeakerFrame{ +func (c *TuyaMqttClient) SendSpeaker(speaker int) error { + // Protocol 312 is used for speaker + return c.sendMqttMessage("speaker", 312, "", SpeakerFrame{ Mode: "webrtc", Value: speaker, }) } -func (c *TuyaClient) sendDisconnect() error { +func (c *TuyaMqttClient) SendDisconnect() error { return c.sendMqttMessage("disconnect", 302, "", DisconnectFrame{ Mode: "webrtc", }) } -func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transactionID string, data interface{}) error { - if c.mqtt.closed { +func (c *TuyaMqttClient) onConnect(client mqtt.Client) { + if token := client.Subscribe(c.subscribeTopic, 1, c.consume); token.Wait() && token.Error() != nil { + c.waiter.Done(token.Error()) + return + } + + c.waiter.Done(nil) +} + +func (c *TuyaMqttClient) consume(client mqtt.Client, msg mqtt.Message) { + var rmqtt MqttMessage + if err := json.Unmarshal(msg.Payload(), &rmqtt); err != nil { + c.onError(err) + return + } + + if rmqtt.Data.Header.SessionID != c.sessionId { + return + } + + switch rmqtt.Data.Header.Type { + case "answer": + c.onMqttAnswer(&rmqtt) + case "candidate": + c.onMqttCandidate(&rmqtt) + case "disconnect": + c.onMqttDisconnect() + } +} + +func (c *TuyaMqttClient) onMqttAnswer(msg *MqttMessage) { + var answerFrame AnswerFrame + if err := json.Unmarshal(msg.Data.Message, &answerFrame); err != nil { + c.onError(err) + return + } + + c.onAnswer(answerFrame) +} + +func (c *TuyaMqttClient) onMqttCandidate(msg *MqttMessage) { + var candidateFrame CandidateFrame + if err := json.Unmarshal(msg.Data.Message, &candidateFrame); err != nil { + c.onError(err) + return + } + + // candidate from device start with "a=", end with "\r\n", which are not needed by Chrome webRTC + candidateFrame.Candidate = strings.TrimPrefix(candidateFrame.Candidate, "a=") + candidateFrame.Candidate = strings.TrimSuffix(candidateFrame.Candidate, "\r\n") + + c.onCandidate(candidateFrame) +} + +func (c *TuyaMqttClient) onMqttDisconnect() { + c.closed = true + c.onDisconnect() +} + +func (c *TuyaMqttClient) onAnswer(answer AnswerFrame) { + if c.handleAnswer != nil { + c.handleAnswer(answer) + } +} + +func (c *TuyaMqttClient) onCandidate(candidate CandidateFrame) { + if c.handleCandidate != nil { + c.handleCandidate(candidate) + } +} + +func (c *TuyaMqttClient) onDisconnect() { + if c.handleDisconnect != nil { + c.handleDisconnect() + } +} + +func (c *TuyaMqttClient) onError(err error) { + if c.handleError != nil { + c.handleError(err) + } +} + +func (c *TuyaMqttClient) sendMqttMessage(messageType string, protocol int, transactionID string, data interface{}) error { + if c.closed { return fmt.Errorf("mqtt client is closed, send mqtt message fail") } @@ -273,7 +284,7 @@ func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transacti Data: MqttFrame{ Header: MqttFrameHeader{ Type: messageType, - From: c.mqtt.uid, + From: c.uid, To: c.deviceId, SessionID: c.sessionId, MotoID: c.motoId, @@ -288,7 +299,7 @@ func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transacti return err } - token := c.mqtt.client.Publish(c.mqtt.publishTopic, 1, false, payload) + token := c.client.Publish(c.publishTopic, 1, false, payload) if token.Wait() && token.Error() != nil { return token.Error() } diff --git a/pkg/tuya/sharing_api.go b/pkg/tuya/sharing_api.go new file mode 100644 index 00000000..88ec223d --- /dev/null +++ b/pkg/tuya/sharing_api.go @@ -0,0 +1,473 @@ +package tuya + +import ( + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/google/uuid" +) + +const ( + TUYA_HOST = "apigw.iotbing.com" + TUYA_CLIENT_ID = "HA_3y9q4ak7g4ephrvke" + TUYA_SCHEMA = "haauthorize" +) + +type OpenApiMQTTConfig struct { + ClientID string `json:"clientId"` + ExpireTime int `json:"expireTime"` + Password string `json:"password"` + Topic struct { + DevID struct { + Pub string `json:"pub"` + Sub string `json:"sub"` + } `json:"devId"` + OwnerID struct { + Sub string `json:"sub"` + } `json:"ownerId"` + } `json:"topic"` + URL string `json:"url"` + Username string `json:"username"` +} + +type OpenApiMQTTConfigRequest struct { + LinkID string `json:"linkId"` +} + +type OpenApiMQTTConfigResponse struct { + Success bool `json:"success"` + Result OpenApiMQTTConfig `json:"result"` + Msg string `json:"msg,omitempty"` +} + +type TokenInfo struct { + AccessToken string `json:"access_token"` + ExpireTime int64 `json:"expire_time"` + RefreshToken string `json:"refresh_token"` +} + +type LoginResult struct { + AccessToken string `json:"access_token"` + Endpoint string `json:"endpoint"` + ExpireTime int64 `json:"expire_time"` // seconds + RefreshToken string `json:"refresh_token"` + TerminalID string `json:"terminal_id"` + UID string `json:"uid"` + Username string `json:"username"` +} + +type LoginResponse struct { + Timestamp int64 `json:"t"` + Success bool `json:"success"` + Result LoginResult `json:"result"` + Msg string `json:"msg,omitempty"` +} + +type QRResponse struct { + Success bool `json:"success"` + Result struct { + Code string `json:"qrcode"` + } `json:"result"` + Msg string `json:"msg,omitempty"` +} + +type Home struct { + ID int `json:"id"` + Name string `json:"name"` + OwnerID string `json:"ownerId"` + Background string `json:"background"` + GeoName string `json:"geoName"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + GmtCreate int64 `json:"gmtCreate"` + GmtModified int64 `json:"gmtModified"` + GroupID int64 `json:"groupId"` + Status bool `json:"status"` + UID string `json:"uid"` +} + +type HomesResponse struct { + Success bool `json:"success"` + Result []Home `json:"result"` + Msg string `json:"msg,omitempty"` +} + +type DeviceFunction struct { + Code string `json:"code"` + Desc string `json:"desc"` + Name string `json:"name"` + Type string `json:"type"` + Values map[string]any `json:"values"` +} + +type DeviceStatusRange struct { + Code string `json:"code"` + Type string `json:"type"` + Values map[string]any `json:"values"` +} + +type Device struct { + ID string `json:"id"` + Name string `json:"name"` + LocalKey string `json:"local_key"` + Category string `json:"category"` + ProductID string `json:"product_id"` + ProductName string `json:"product_name"` + Sub bool `json:"sub"` + UUID string `json:"uuid"` + AssetID string `json:"asset_id"` + Online bool `json:"online"` + Icon string `json:"icon"` + IP string `json:"ip"` + TimeZone string `json:"time_zone"` + ActiveTime int64 `json:"active_time"` + CreateTime int64 `json:"create_time"` + UpdateTime int64 `json:"update_time"` +} + +type DeviceRequest struct { + HomeID string `json:"homeId"` +} + +type DeviceResponse struct { + Success bool `json:"success"` + Result []Device `json:"result"` + Msg string `json:"msg,omitempty"` +} + +type TuyaOpenApiClient struct { + TuyaClient + terminalId string + refreshingToken bool +} + +func NewTuyaOpenApiClient( + baseUrl string, + uid string, + deviceId string, + terminalId string, + tokenInfoOrString any, + streamMode string, +) (*TuyaOpenApiClient, error) { + tokenInfo, err := ParseTokenInfo(tokenInfoOrString) + if err != nil { + return nil, fmt.Errorf("failed to parse token info: %w", err) + } + + mqttClient := NewTuyaMqttClient(deviceId) + + client := &TuyaOpenApiClient{ + TuyaClient: TuyaClient{ + httpClient: &http.Client{Timeout: 15 * time.Second}, + mqtt: mqttClient, + uid: uid, + deviceId: deviceId, + accessToken: tokenInfo.AccessToken, + refreshToken: tokenInfo.RefreshToken, + expireTime: tokenInfo.ExpireTime, + streamMode: streamMode, + baseUrl: baseUrl, + }, + terminalId: terminalId, + refreshingToken: false, + } + + return client, nil +} + +// WebRTC Flow (not supported yet) +func (c *TuyaOpenApiClient) Init() error { + if err := c.initToken(); err != nil { + return fmt.Errorf("failed to initialize token: %w", err) + } + + return fmt.Errorf("stream mode %s is not supported", c.streamMode) +} + +func (c *TuyaOpenApiClient) GetStreamUrl(streamType string) (streamUrl string, err error) { + if err := c.initToken(); err != nil { + return "", fmt.Errorf("failed to initialize token: %w", err) + } + + urlPath := fmt.Sprintf("/v1.0/m/ipc/%s/stream/actions/allocate", c.deviceId) + + request := &AllocateRequest{ + Type: streamType, + } + + body, err := c.request("POST", urlPath, nil, request) + if err != nil { + return "", err + } + + var allocResponse AllocateResponse + err = json.Unmarshal(body, &allocResponse) + if err != nil { + return "", err + } + + if !allocResponse.Success { + return "", fmt.Errorf(allocResponse.Msg) + } + + return allocResponse.Result.URL, nil +} + +func (c *TuyaOpenApiClient) GetAllDevices() ([]Device, error) { + homes, err := c.queryHomes() + if err != nil { + return nil, err + } + + time.Sleep(500 * time.Millisecond) + deviceMap := make(map[string]Device) + + for i, home := range homes { + if i > 0 { + time.Sleep(300 * time.Millisecond) + } + + devices, err := c.queryDevicesByHome(home.OwnerID) + if err != nil { + return nil, err + } + + for _, device := range devices { + // https://github.com/home-assistant/core/blob/088cfc3576e0018ad1df373c08549092918e6530/homeassistant/components/tuya/camera.py#L19 + if device.Category == "sp" || device.Category == "dghsxj" { + deviceMap[device.ID] = device + } + } + } + + var devices []Device + for _, device := range deviceMap { + devices = append(devices, device) + } + + return devices, nil +} + +func (c *TuyaOpenApiClient) loadHubConfig() (config *MQTTConfig, err error) { + request := OpenApiMQTTConfigRequest{ + LinkID: fmt.Sprintf("tuya-device-sharing-sdk-go.%s", uuid.New().String()), + } + + body, err := c.request("POST", "/v1.0/m/life/ha/access/config", nil, request) + if err != nil { + return nil, err + } + + var mqttConfigResponse OpenApiMQTTConfigResponse + if err := json.Unmarshal(body, &mqttConfigResponse); err != nil { + return nil, err + } + + if !mqttConfigResponse.Success { + return nil, fmt.Errorf("failed to get MQTT config: %s", mqttConfigResponse.Msg) + } + + return &MQTTConfig{ + Url: mqttConfigResponse.Result.URL, + Username: mqttConfigResponse.Result.Username, + Password: mqttConfigResponse.Result.Password, + ClientID: mqttConfigResponse.Result.ClientID, + PublishTopic: mqttConfigResponse.Result.Topic.DevID.Pub, + SubscribeTopic: mqttConfigResponse.Result.Topic.DevID.Sub, + }, nil +} + +func (c *TuyaOpenApiClient) queryHomes() ([]Home, error) { + body, err := c.request("GET", "/v1.0/m/life/users/homes", nil, nil) + if err != nil { + return nil, err + } + + var homesResponse HomesResponse + if err := json.Unmarshal(body, &homesResponse); err != nil { + return nil, err + } + + if !homesResponse.Success { + return nil, fmt.Errorf("failed to get homes: %s", homesResponse.Msg) + } + + return homesResponse.Result, nil +} + +func (c *TuyaOpenApiClient) queryDevicesByHome(homeID string) ([]Device, error) { + params := DeviceRequest{ + HomeID: homeID, + } + + body, err := c.request("GET", "/v1.0/m/life/ha/home/devices", params, nil) + if err != nil { + return nil, err + } + + var devicesResponse DeviceResponse + if err := json.Unmarshal(body, &devicesResponse); err != nil { + return nil, err + } + + if !devicesResponse.Success { + return nil, fmt.Errorf("failed to get devices: %s", devicesResponse.Msg) + } + + return devicesResponse.Result, nil +} + +// https://github.com/tuya/tuya-device-sharing-sdk/blob/main/tuya_sharing/customerapi.py +func (c *TuyaOpenApiClient) request( + method string, + path string, + params any, + body any, +) ([]byte, error) { + rid := uuid.New().String() + sid := "" + + md5Hash := md5.New() + ridRefreshToken := rid + c.refreshToken + md5Hash.Write([]byte(ridRefreshToken)) + hashKey := hex.EncodeToString(md5Hash.Sum(nil)) + secret := SecretGenerating(rid, sid, hashKey) + + queryEncdata := "" + var reqURL string + if params != nil { + jsonData := FormToJSON(params) + + encryptedData, err := AesGCMEncrypt(jsonData, secret) + if err != nil { + return nil, err + } + + queryEncdata = encryptedData + reqURL = fmt.Sprintf("https://%s%s?encdata=%s", c.baseUrl, path, queryEncdata) + } else { + reqURL = fmt.Sprintf("https://%s%s", c.baseUrl, path) + } + + bodyEncdata := "" + var reqBody io.Reader + if body != nil { + jsonData := FormToJSON(body) + + encryptedData, err := AesGCMEncrypt(jsonData, secret) + if err != nil { + return nil, err + } + + bodyEncdata = encryptedData + encBody := map[string]string{"encdata": bodyEncdata} + bodyBytes, _ := json.Marshal(encBody) + reqBody = strings.NewReader(string(bodyBytes)) + } + + req, err := http.NewRequest(method, reqURL, reqBody) + if err != nil { + return nil, err + } + + t := time.Now().Add(2*time.Second).UnixNano() / int64(time.Millisecond) + headers := map[string]string{ + "X-appKey": TUYA_CLIENT_ID, + "X-requestId": rid, + "X-sid": sid, + "X-time": fmt.Sprintf("%d", t), + "Content-Type": "application/json", + } + + if c.accessToken != "" { + headers["X-token"] = c.accessToken + } + + sign := RestfulSign(hashKey, queryEncdata, bodyEncdata, headers) + headers["X-sign"] = sign + + for key, value := range headers { + req.Header.Set(key, value) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var resultObj map[string]any + if err := json.Unmarshal(respBody, &resultObj); err != nil { + return nil, err + } + + if resultStr, ok := resultObj["result"].(string); ok { + decrypted, err := AesGCMDecrypt(resultStr, secret) + if err != nil { + return nil, err + } + + var decryptedObj any + if err := json.Unmarshal([]byte(decrypted), &decryptedObj); err == nil { + resultObj["result"] = decryptedObj + } else { + resultObj["result"] = decrypted + } + + updatedResponse, err := json.Marshal(resultObj) + if err != nil { + return nil, fmt.Errorf("failed to marshal updated response: %w", err) + } + + return updatedResponse, nil + } + + return respBody, nil +} + +func (c *TuyaOpenApiClient) initToken() error { + if c.refreshingToken { + return nil + } + + now := time.Now().Unix() + if (c.expireTime - 60) > now { + return nil + } + + c.refreshingToken = true + + urlPath := fmt.Sprintf("/v1.0/m/token/%s", c.refreshToken) + + body, err := c.request("GET", urlPath, nil, nil) + if err != nil { + return err + } + + var loginResponse LoginResponse + if err := json.Unmarshal(body, &loginResponse); err != nil { + return err + } + + if !loginResponse.Success { + return fmt.Errorf("failed to get token: %s", loginResponse.Msg) + } + + c.accessToken = loginResponse.Result.AccessToken + c.refreshToken = loginResponse.Result.RefreshToken + c.expireTime = loginResponse.Timestamp + loginResponse.Result.ExpireTime + c.refreshingToken = false + + return nil +} diff --git a/www/add.html b/www/add.html index c8808736..f16d4d45 100644 --- a/www/add.html +++ b/www/add.html @@ -28,6 +28,7 @@ } + @@ -280,6 +281,110 @@ document.getElementById('ring-token-form').addEventListener('submit', handleRingAuth); + +
+

Attention: Cameras added through QR Code does not support webrtc mode!

+
+ + +
+ + + +
+
+ +
From fbd8d995ed0132088f9499eeaa13c86c698e5848 Mon Sep 17 00:00:00 2001 From: seydx Date: Thu, 22 May 2025 00:16:31 +0200 Subject: [PATCH 27/60] refactor and increase timeout --- pkg/tuya/{sharing_api.go => open_api.go} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename pkg/tuya/{sharing_api.go => open_api.go} (99%) diff --git a/pkg/tuya/sharing_api.go b/pkg/tuya/open_api.go similarity index 99% rename from pkg/tuya/sharing_api.go rename to pkg/tuya/open_api.go index 88ec223d..4bd2a044 100644 --- a/pkg/tuya/sharing_api.go +++ b/pkg/tuya/open_api.go @@ -225,12 +225,12 @@ func (c *TuyaOpenApiClient) GetAllDevices() ([]Device, error) { return nil, err } - time.Sleep(500 * time.Millisecond) + time.Sleep(2 * time.Second) deviceMap := make(map[string]Device) for i, home := range homes { if i > 0 { - time.Sleep(300 * time.Millisecond) + time.Sleep(500 * time.Millisecond) } devices, err := c.queryDevicesByHome(home.OwnerID) From 7ee3f6e4f7629f54669c4fa108960eca5508b18e Mon Sep 17 00:00:00 2001 From: seydx <34152761+seydx@users.noreply.github.com> Date: Thu, 22 May 2025 00:41:22 +0200 Subject: [PATCH 28/60] Update README.md Co-authored-by: Felipe Santos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 00b9cc45..d2d47ef8 100644 --- a/README.md +++ b/README.md @@ -587,7 +587,7 @@ The `Cloud API` requires setting up a cloud project in the Tuya Developer Platfo - `flv` - FLV stream _(only available for `Open API`)_ - `rtmp` - RTMP stream _(only available for `Open API`)_ -- Use `resolution` parameter to select the stream (only available for `Cloud API` and not all cameras support `hd` stream): +- Use `resolution` parameter to select the stream (only available for `Cloud API` and not all cameras support `hd` stream through WebRTC even if the camera has it): - `hd` - HD stream (default) - `sd` - SD stream From 42b7eea8524c295076d03af02afa52e5bf19f828 Mon Sep 17 00:00:00 2001 From: seydx Date: Thu, 22 May 2025 21:27:28 +0200 Subject: [PATCH 29/60] fix: correct transceiver direction check in getSender method --- pkg/tuya/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 4a43c49a..7692639c 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -490,7 +490,7 @@ func (c *Client) sendMessageToDataChannel(message []byte) error { func (c *Client) getSender() *webrtc.Track { for _, tr := range c.pc.GetTransceivers() { if tr.Kind() == pion.RTPCodecTypeAudio { - if tr.Kind() == pion.RTPCodecType(pion.RTPTransceiverDirectionSendonly) || tr.Kind() == pion.RTPCodecType(pion.RTPTransceiverDirectionSendrecv) { + if tr.Direction() == pion.RTPTransceiverDirectionSendonly || tr.Direction() == pion.RTPTransceiverDirectionSendrecv { if s := tr.Sender(); s != nil { if t := s.Track().(*webrtc.Track); t != nil { return t From 5be5d9247caaf09be4f68102f79118366d26b7cd Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 26 May 2025 18:29:31 +0200 Subject: [PATCH 30/60] refactor, simplify api, add support for email/password auth --- README.md | 48 +--- internal/tuya/tuya.go | 282 +++++++++++--------- pkg/tuya/client.go | 39 +-- pkg/tuya/cloud_api.go | 13 +- pkg/tuya/crypto.go | 134 ---------- pkg/tuya/helper.go | 103 ++++---- pkg/tuya/interface.go | 28 +- pkg/tuya/open_api.go | 473 --------------------------------- pkg/tuya/tuya_api.go | 590 ++++++++++++++++++++++++++++++++++++++++++ www/add.html | 88 ++----- 10 files changed, 857 insertions(+), 941 deletions(-) delete mode 100644 pkg/tuya/crypto.go delete mode 100644 pkg/tuya/open_api.go create mode 100644 pkg/tuya/tuya_api.go diff --git a/README.md b/README.md index d2d47ef8..7a257ae0 100644 --- a/README.md +++ b/README.md @@ -568,58 +568,38 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. #### Source: Tuya -[Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. Go2rtc supports `Cloud API` and `Open API`. +[Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. Go2rtc supports `Cloud API` and `Tuya API`. -The `Cloud API` requires setting up a cloud project in the Tuya Developer Platform to retrieve the required credentials. The `Open API` does not require a cloud project and the cameras can be added through the interface via QR code (user code required), but it does not support webrtc mode and two way audio. +The `Cloud API` requires setting up a cloud project in the Tuya Developer Platform to retrieve the required credentials. The `Tuya API` does not require a cloud project and the cameras can be added through the interface via email/password. **Cloud API**: -- Obtain `device_id`, `client_id`, `client_secret`, and `uid` (if using `mode=webrtc`) from [Tuya IoT Platform](https://iot.tuya.com/). [Here's a guide](https://xzetsubou.github.io/hass-localtuya/cloud_api/). +- Obtain `device_id`, `client_id`, `client_secret`, and `uid` from [Tuya IoT Platform](https://iot.tuya.com/). [Here's a guide](https://xzetsubou.github.io/hass-localtuya/cloud_api/). **Open API**: -- To get your user code, open the Tuya Smart app or Smart Life app and go to `Profile` > `Settings` > `Account and Security` > `User Code` -- Open the Go2rtc interface and go to `Add` > `Tuya` and enter your `User Code` in the `User Code` field. Click on `Generate QR Code` and scan it with the Tuya Smart app or Smart Life app. After scanning, click on `Login`. All cameras in your home (not shared ones) will be listed in the Go2rtc interface. Copy/Paste stream URLs to your `go2rtc.yaml` file. +- Smart Life accounts are not supported, you need to create a Tuya account. If the cameras are already added to the Smart Life app, you need to remove them and add them again to the Tuya Smart app. **Configuring the stream:** -- Use `mode` parameter to select the stream type (not all cameras support all modes): - - `webrtc` - WebRTC stream _(default for `Cloud API`)_ - - `rtsp` - RTSP stream _(default for `Open API`)_ - - `hls` - HLS stream - - `flv` - FLV stream _(only available for `Open API`)_ - - `rtmp` - RTMP stream _(only available for `Open API`)_ - -- Use `resolution` parameter to select the stream (only available for `Cloud API` and not all cameras support `hd` stream through WebRTC even if the camera has it): +- Use `resolution` parameter to select the stream (not all cameras support `hd` stream through WebRTC even if the camera has it): - `hd` - HD stream (default) - `sd` - SD stream ```yaml streams: - # Cloud API: WebRTC stream + # Cloud API: WebRTC main stream tuya_webrtc: - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX - - # Cloud API: WebRTC stream (same as above) - tuya_webrtc_2: - - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&mode=webrtc - # Cloud API: WebRTC stream (HD) - tuya_webrtc_hd: - - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=hd - - # Cloud API: WebRTC stream (SD) + # Cloud API: WebRTC sub stream tuya_webrtc_sd: - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=sd - - # Cloud API: RTSP stream when available (no "uid" required) - tuya_rtsp: - - tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=rtsp - - # Cloud API: HLS stream when available (no "uid" required) - tuya_hls: - - tuya://openapi.tuyaus.com?device_id=XXX&client_id=XXX&client_secret=XXX&mode=hls - # Open API: RTSP stream - tuya_openapi: - - tuya://apigw.tuyaeu.com?device_id=XXX&terminal_id=XXX&token=XXX&uid=XXX + # Tuya API: WebRTC main stream + tuya: + - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX + + # Tuya API: WebRTC sub stream + tuya: + - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX&resolution=sd ``` #### Source: GoPro diff --git a/internal/tuya/tuya.go b/internal/tuya/tuya.go index b5457253..cb25daa5 100644 --- a/internal/tuya/tuya.go +++ b/internal/tuya/tuya.go @@ -1,13 +1,13 @@ package tuya import ( + "bytes" "encoding/json" + "errors" "fmt" - "io" "net/http" "net/url" - "strings" - "time" + "strconv" "github.com/AlexxIT/go2rtc/internal/api" "github.com/AlexxIT/go2rtc/internal/streams" @@ -15,8 +15,6 @@ import ( "github.com/AlexxIT/go2rtc/pkg/tuya" ) -var users = make(map[string]tuya.LoginResponse) - func Init() { streams.HandleFunc("tuya", func(source string) (core.Producer, error) { return tuya.Dial(source) @@ -27,74 +25,41 @@ func Init() { func apiTuya(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() - userCode := query.Get("user_code") - token := query.Get("token") + region := query.Get("region") + email := query.Get("email") + password := query.Get("password") - if userCode == "" { - http.Error(w, "user_code is required", http.StatusBadRequest) + if email == "" || password == "" || region == "" { + http.Error(w, "email, password and region are required", http.StatusBadRequest) return } - var auth *tuya.LoginResponse - if loginResponse, ok := users[userCode]; ok { - expireTime := loginResponse.Timestamp + loginResponse.Result.ExpireTime - - if expireTime > time.Now().Unix() { - auth = &loginResponse - } else { - delete(users, userCode) - token = "" + var tuyaRegion *tuya.Region + for _, r := range tuya.AvailableRegions { + if r.Host == region { + tuyaRegion = &r + break } } - if auth == nil && token == "" { - qrCode, err := getQRCode(userCode) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // response qrCode - json.NewEncoder(w).Encode(map[string]interface{}{ - "qrCode": qrCode, - }) - + if tuyaRegion == nil { + http.Error(w, fmt.Sprintf("invalid region: %s", region), http.StatusBadRequest) return } - if auth == nil && token != "" { - authResponse, err := login(userCode, token) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + httpClient := tuya.CreateHTTPClientWithSession() - auth = authResponse - } - - if auth == nil { - http.Error(w, "failed to get auth", http.StatusInternalServerError) - return - } - - tokenInfo := tuya.TokenInfo{ - AccessToken: auth.Result.AccessToken, - ExpireTime: auth.Timestamp + auth.Result.ExpireTime, - RefreshToken: auth.Result.RefreshToken, - } - - tokenInfoBase64, err := tuya.ToBase64(&tokenInfo) + _, err := login(httpClient, tuyaRegion.Host, email, password, tuyaRegion.Continent) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, fmt.Sprintf("login failed: %v", err), http.StatusInternalServerError) return } - tuyaAPI, err := tuya.NewTuyaOpenApiClient( - strings.Replace(auth.Result.Endpoint, "https://", "", 1), - auth.Result.UID, - "", - auth.Result.TerminalID, - tokenInfo, + tuyaAPI, err := tuya.NewTuyaApiClient( + httpClient, + tuyaRegion.Host, + email, + password, "", ) @@ -103,25 +68,52 @@ func apiTuya(w http.ResponseWriter, r *http.Request) { return } - devices, err := tuyaAPI.GetAllDevices() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + var devices []tuya.Device + + homes, _ := tuyaAPI.GetHomeList() + if homes != nil && len(homes.Result) > 0 { + for _, home := range homes.Result { + roomList, err := tuyaAPI.GetRoomList(strconv.Itoa(home.Gid)) + if err != nil { + continue + } + + for _, room := range roomList.Result { + for _, device := range room.DeviceList { + if (device.Category == "sp" || device.Category == "dghsxj") && !containsDevice(devices, device.DeviceId) { + devices = append(devices, device) + } + } + } + } + } + + sharedHomes, _ := tuyaAPI.GetSharedHomeList() + if sharedHomes != nil && len(sharedHomes.Result.SecurityWebCShareInfoList) > 0 { + for _, sharedHome := range sharedHomes.Result.SecurityWebCShareInfoList { + for _, device := range sharedHome.DeviceInfoList { + if (device.Category == "sp" || device.Category == "dghsxj") && !containsDevice(devices, device.DeviceId) { + devices = append(devices, device) + } + } + } + } + + if len(devices) == 0 { + http.Error(w, "no cameras found", http.StatusNotFound) return } var items []*api.Source for _, device := range devices { cleanQuery := url.Values{} - cleanQuery.Set("uid", auth.Result.UID) - cleanQuery.Set("token", tokenInfoBase64) - cleanQuery.Set("terminal_id", auth.Result.TerminalID) - cleanQuery.Set("device_id", device.ID) - - endpoint := strings.Replace(auth.Result.Endpoint, "https://", "tuya://", 1) - url := fmt.Sprintf("%s?%s", endpoint, cleanQuery.Encode()) + cleanQuery.Set("device_id", device.DeviceId) + cleanQuery.Set("email", email) + cleanQuery.Set("password", password) + url := fmt.Sprintf("tuya://%s?%s", tuyaRegion.Host, cleanQuery.Encode()) items = append(items, &api.Source{ - Name: device.Name, + Name: device.DeviceName, URL: url, }) } @@ -129,86 +121,128 @@ func apiTuya(w http.ResponseWriter, r *http.Request) { api.ResponseSources(w, items) } -func login(userCode string, qrCode string) (*tuya.LoginResponse, error) { - url := fmt.Sprintf("https://%s/v1.0/m/life/home-assistant/qrcode/tokens/%s?clientid=%s&usercode=%s", tuya.TUYA_HOST, qrCode, tuya.TUYA_CLIENT_ID, userCode) - - req, err := http.NewRequest("GET", url, nil) +func login(client *http.Client, serverHost, email, password, countryCode string) (*tuya.LoginResult, error) { + tokenResp, err := getLoginToken(client, serverHost, email, countryCode) if err != nil { return nil, err } - httpClient := &http.Client{ - Timeout: 10 * time.Second, - } - - response, err := httpClient.Do(req) + encryptedPassword, err := tuya.EncryptPassword(password, tokenResp.Result.PbKey) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to encrypt password: %v", err) } - defer response.Body.Close() - res, err := io.ReadAll(response.Body) + var loginResp *tuya.PasswordLoginResponse + var url string + + loginReq := tuya.PasswordLoginRequest{ + CountryCode: countryCode, + Passwd: encryptedPassword, + Token: tokenResp.Result.Token, + IfEncrypt: 1, + Options: `{"group":1}`, + } + + if tuya.IsEmailAddress(email) { + url = fmt.Sprintf("https://%s/api/private/email/login", serverHost) + loginReq.Email = email + } else { + url = fmt.Sprintf("https://%s/api/private/phone/login", serverHost) + loginReq.Mobile = email + } + + loginResp, err = performLogin(client, url, loginReq, serverHost) + if err != nil { return nil, err } - if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get QR code: %s", string(res)) + if !loginResp.Success { + return nil, errors.New(loginResp.ErrorMsg) } - var loginResponse tuya.LoginResponse - err = json.Unmarshal(res, &loginResponse) - if err != nil { - return nil, err - } - - if !loginResponse.Success { - return nil, fmt.Errorf("failed to login: %s", loginResponse.Msg) - } - - users[userCode] = loginResponse - - return &loginResponse, nil + return &loginResp.Result, nil } -func getQRCode(userCode string) (string, error) { - url := fmt.Sprintf("https://%s/v1.0/m/life/home-assistant/qrcode/tokens?clientid=%s&schema=%s&usercode=%s", tuya.TUYA_HOST, tuya.TUYA_CLIENT_ID, tuya.TUYA_SCHEMA, userCode) +func getLoginToken(client *http.Client, serverHost, username, countryCode string) (*tuya.LoginTokenResponse, error) { + url := fmt.Sprintf("https://%s/api/login/token", serverHost) - req, err := http.NewRequest("POST", url, nil) + tokenReq := tuya.LoginTokenRequest{ + CountryCode: countryCode, + Username: username, + IsUid: false, + } + + jsonData, err := json.Marshal(tokenReq) if err != nil { - return "", err + return nil, err } - req.Header.Set("Content-Type", "text/plain") - - httpClient := &http.Client{ - Timeout: 10 * time.Second, - } - - response, err := httpClient.Do(req) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) if err != nil { - return "", err + return nil, err } - defer response.Body.Close() - res, err := io.ReadAll(response.Body) + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("Accept", "*/*") + req.Header.Set("Origin", fmt.Sprintf("https://%s", serverHost)) + req.Header.Set("Referer", fmt.Sprintf("https://%s/login", serverHost)) + req.Header.Set("X-Requested-With", "XMLHttpRequest") + + resp, err := client.Do(req) if err != nil { - return "", err + return nil, err + } + defer resp.Body.Close() + + var tokenResp tuya.LoginTokenResponse + if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { + return nil, err } - if response.StatusCode != http.StatusOK { - return "", err + if !tokenResp.Success { + return nil, err } - var qrResponse tuya.QRResponse - err = json.Unmarshal(res, &qrResponse) - if err != nil { - return "", err - } - - if !qrResponse.Success { - return "", fmt.Errorf("failed to get QR code: %s", qrResponse.Msg) - } - - return qrResponse.Result.Code, nil + return &tokenResp, nil +} + +func performLogin(client *http.Client, url string, loginReq tuya.PasswordLoginRequest, serverHost string) (*tuya.PasswordLoginResponse, error) { + jsonData, err := json.Marshal(loginReq) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("Accept", "*/*") + req.Header.Set("Origin", fmt.Sprintf("https://%s", serverHost)) + req.Header.Set("Referer", fmt.Sprintf("https://%s/login", serverHost)) + req.Header.Set("X-Requested-With", "XMLHttpRequest") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var loginResp tuya.PasswordLoginResponse + if err := json.NewDecoder(resp.Body).Decode(&loginResp); err != nil { + return nil, err + } + + return &loginResp, nil +} + +func containsDevice(devices []tuya.Device, deviceID string) bool { + for _, device := range devices { + if device.DeviceId == deviceID { + return true + } + } + return false } diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 7692639c..f5b964e9 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -7,7 +7,6 @@ import ( "net/url" "regexp" - "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/webrtc" "github.com/pion/rtp" @@ -50,38 +49,29 @@ func Dial(rawURL string) (core.Producer, error) { query := u.Query() - // Open API - tokenInfo := query.Get("token") - terminalId := query.Get("terminal_id") + // Tuya API + email := query.Get("email") + password := query.Get("password") // Cloud API + uid := query.Get("uid") clientId := query.Get("client_id") clientSecret := query.Get("client_secret") // Shared params deviceId := query.Get("device_id") - uid := query.Get("uid") // Stream params streamResolution := query.Get("resolution") - streamMode := query.Get("mode") - useOpenApi := deviceId != "" && uid != "" && tokenInfo != "" && terminalId != "" - useCloudApi := deviceId != "" && ((streamMode == "webrtc" || streamMode == "") && uid != "") && clientId != "" && clientSecret != "" + useTuyaApi := deviceId != "" && email != "" && password != "" + useCloudApi := deviceId != "" && uid != "" && clientId != "" && clientSecret != "" if streamResolution == "" || (streamResolution != "hd" && streamResolution != "sd") { streamResolution = "hd" } - if streamMode == "" || (streamMode != "rtsp" && streamMode != "hls" && streamMode != "flv" && streamMode != "rtmp" && streamMode != "webrtc") { - if useOpenApi { - streamMode = "rtsp" - } else { - streamMode = "webrtc" - } - } - - if !useOpenApi && !useCloudApi { + if !useTuyaApi && !useCloudApi { return nil, errors.New("tuya: wrong query params") } @@ -89,25 +79,16 @@ func Dial(rawURL string) (core.Producer, error) { handlers: make(map[uint32]func(*rtp.Packet)), } - if useOpenApi { - if client.api, err = NewTuyaOpenApiClient(u.Hostname(), uid, deviceId, terminalId, tokenInfo, streamMode); err != nil { + if useTuyaApi { + if client.api, err = NewTuyaApiClient(nil, u.Hostname(), email, password, deviceId); err != nil { return nil, fmt.Errorf("tuya: %w", err) } } else { - if client.api, err = NewTuyaCloudApiClient(u.Hostname(), uid, deviceId, clientId, clientSecret, streamMode); err != nil { + if client.api, err = NewTuyaCloudApiClient(u.Hostname(), uid, deviceId, clientId, clientSecret); err != nil { return nil, fmt.Errorf("tuya: %w", err) } } - if streamMode != "webrtc" { - streamUrl, err := client.api.GetStreamUrl(streamMode) - if err != nil { - return nil, fmt.Errorf("tuya: %w", err) - } - - return streams.GetProducer(streamUrl) - } - if err := client.api.Init(); err != nil { return nil, fmt.Errorf("tuya: %w", err) } diff --git a/pkg/tuya/cloud_api.go b/pkg/tuya/cloud_api.go index bd74daf0..4d25c2be 100644 --- a/pkg/tuya/cloud_api.go +++ b/pkg/tuya/cloud_api.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/md5" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -68,24 +69,26 @@ type OpenIoTHubConfigResponse struct { type TuyaCloudApiClient struct { TuyaClient + uid string clientId string clientSecret string + accessToken string + refreshToken string refreshingToken bool } -func NewTuyaCloudApiClient(baseUrl string, uid string, deviceId string, clientId string, clientSecret string, streamMode string) (*TuyaCloudApiClient, error) { +func NewTuyaCloudApiClient(baseUrl, uid, deviceId, clientId, clientSecret string) (*TuyaCloudApiClient, error) { mqttClient := NewTuyaMqttClient(deviceId) client := &TuyaCloudApiClient{ TuyaClient: TuyaClient{ httpClient: &http.Client{Timeout: 15 * time.Second}, mqtt: mqttClient, - uid: uid, deviceId: deviceId, - streamMode: streamMode, expireTime: 0, baseUrl: baseUrl, }, + uid: uid, clientId: clientId, clientSecret: clientSecret, refreshingToken: false, @@ -140,7 +143,7 @@ func (c *TuyaCloudApiClient) GetStreamUrl(streamType string) (streamUrl string, } if !allocResponse.Success { - return "", fmt.Errorf(allocResponse.Msg) + return "", errors.New(allocResponse.Msg) } return allocResponse.Result.URL, nil @@ -175,7 +178,7 @@ func (c *TuyaCloudApiClient) initToken() (err error) { } if !tokenResponse.Success { - return fmt.Errorf(tokenResponse.Msg) + return errors.New(tokenResponse.Msg) } c.accessToken = tokenResponse.Result.AccessToken diff --git a/pkg/tuya/crypto.go b/pkg/tuya/crypto.go deleted file mode 100644 index b8f84615..00000000 --- a/pkg/tuya/crypto.go +++ /dev/null @@ -1,134 +0,0 @@ -package tuya - -import ( - "crypto/aes" - "crypto/cipher" - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "fmt" - "math/rand" -) - -// https://github.com/tuya/tuya-device-sharing-sdk/blob/main/tuya_sharing/customerapi.py -func AesGCMEncrypt(rawData string, secret string) (string, error) { - nonce := []byte(RandomNonce(12)) - - block, err := aes.NewCipher([]byte(secret)) - if err != nil { - return "", err - } - - aesgcm, err := cipher.NewGCM(block) - if err != nil { - return "", err - } - - ciphertext := aesgcm.Seal(nil, nonce, []byte(rawData), nil) - nonceB64 := base64.StdEncoding.EncodeToString(nonce) - ciphertextB64 := base64.StdEncoding.EncodeToString(ciphertext) - - return nonceB64 + ciphertextB64, nil -} - -func AesGCMDecrypt(cipherData string, secret string) (string, error) { - if len(cipherData) <= 16 { - return "", fmt.Errorf("invalid ciphertext length") - } - - nonceB64 := cipherData[:16] - ciphertextB64 := cipherData[16:] - - nonce, err := base64.StdEncoding.DecodeString(nonceB64) - if err != nil { - return "", err - } - - ciphertext, err := base64.StdEncoding.DecodeString(ciphertextB64) - if err != nil { - return "", err - } - - block, err := aes.NewCipher([]byte(secret)) - if err != nil { - return "", err - } - - aesgcm, err := cipher.NewGCM(block) - if err != nil { - return "", err - } - - plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil) - if err != nil { - return "", err - } - - return string(plaintext), nil -} - -func SecretGenerating(rid, sid, hashKey string) string { - message := hashKey - mod := 16 - - if sid != "" { - sidLength := len(sid) - length := sidLength - if length > mod { - length = mod - } - - ecode := "" - for i := 0; i < length; i++ { - idx := int(sid[i]) % mod - ecode += string(sid[idx]) - } - message += "_" - message += ecode - } - - h := hmac.New(sha256.New, []byte(rid)) - h.Write([]byte(message)) - byteTemp := h.Sum(nil) - secret := hex.EncodeToString(byteTemp) - - return secret[:16] -} - -func RestfulSign(hashKey, queryEncdata, bodyEncdata string, data map[string]string) string { - headers := []string{"X-appKey", "X-requestId", "X-sid", "X-time", "X-token"} - headerSignStr := "" - - for _, item := range headers { - val, exists := data[item] - if exists && val != "" { - headerSignStr += item + "=" + val + "||" - } - } - - signStr := "" - if len(headerSignStr) > 2 { - signStr = headerSignStr[:len(headerSignStr)-2] - } - - if queryEncdata != "" { - signStr += queryEncdata - } - if bodyEncdata != "" { - signStr += bodyEncdata - } - - h := hmac.New(sha256.New, []byte(hashKey)) - h.Write([]byte(signStr)) - return hex.EncodeToString(h.Sum(nil)) -} - -func RandomNonce(length int) string { - const charset = "ABCDEFGHJKMNPQRSTWXYZabcdefhijkmnprstwxyz2345678" - result := make([]byte, length) - for i := range result { - result[i] = charset[rand.Intn(len(charset))] - } - return string(result) -} diff --git a/pkg/tuya/helper.go b/pkg/tuya/helper.go index 0b97b256..7c9eb410 100644 --- a/pkg/tuya/helper.go +++ b/pkg/tuya/helper.go @@ -1,72 +1,69 @@ package tuya import ( - "encoding/base64" - "encoding/json" - "fmt" + "crypto/md5" + cryptoRand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "errors" + "net/http" + "net/http/cookiejar" + "regexp" + "time" + + "golang.org/x/net/publicsuffix" ) -func FormToJSON(content any) string { - if content == nil { - return "{}" +func EncryptPassword(password, pbKey string) (string, error) { + // Hash password with MD5 + hasher := md5.New() + hasher.Write([]byte(password)) + hashedPassword := hex.EncodeToString(hasher.Sum(nil)) + + // Decode PEM public key + block, _ := pem.Decode([]byte("-----BEGIN PUBLIC KEY-----\n" + pbKey + "\n-----END PUBLIC KEY-----")) + if block == nil { + return "", errors.New("failed to decode PEM block") } - jsonBytes, err := json.Marshal(content) + pubKey, err := x509.ParsePKIXPublicKey(block.Bytes) if err != nil { - return "{}" + return "", err } - return string(jsonBytes) + rsaPubKey, ok := pubKey.(*rsa.PublicKey) + if !ok { + return "", errors.New("not an RSA public key") + } + + // Encrypt with RSA + encrypted, err := rsa.EncryptPKCS1v15(cryptoRand.Reader, rsaPubKey, []byte(hashedPassword)) + if err != nil { + return "", err + } + + // Convert to hex string + return hex.EncodeToString(encrypted), nil } -func ToBase64(tokenInfo *TokenInfo) (string, error) { - jsonData, err := json.Marshal(tokenInfo) - if err != nil { - return "", fmt.Errorf("error marshalling token: %v", err) - } - - encoded := base64.URLEncoding.EncodeToString(jsonData) - - return encoded, nil +func IsEmailAddress(input string) bool { + emailRegex := regexp.MustCompile(`^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$`) + return emailRegex.MatchString(input) } -func FromBase64(encodedTokenInfo string) (*TokenInfo, error) { - jsonData, err := base64.URLEncoding.DecodeString(encodedTokenInfo) +func CreateHTTPClientWithSession() *http.Client { + jar, err := cookiejar.New(&cookiejar.Options{ + PublicSuffixList: publicsuffix.List, + }) + if err != nil { - return nil, fmt.Errorf("error decoding token: %v", err) + return nil } - var tokenInfo TokenInfo - err = json.Unmarshal(jsonData, &tokenInfo) - if err != nil { - return nil, fmt.Errorf("error unmarshalling token: %v", err) + return &http.Client{ + Timeout: 30 * time.Second, + Jar: jar, } - - return &tokenInfo, nil -} - -func ParseTokenInfo(tokenInfoOrString any) (*TokenInfo, error) { - var tokenInfo *TokenInfo - var err error - - switch v := tokenInfoOrString.(type) { - case string: - tokenInfo, err = FromBase64(v) - if err != nil { - return nil, fmt.Errorf("failed to decode base64 token: %w", err) - } - case *TokenInfo: - tokenInfo = v - case TokenInfo: - copyOfV := v - tokenInfo = ©OfV - default: - return nil, fmt.Errorf("invalid type: %T", v) - } - - if tokenInfo == nil { - return nil, fmt.Errorf("token info is nil") - } - - return tokenInfo, nil } diff --git a/pkg/tuya/interface.go b/pkg/tuya/interface.go index dae19a82..f4f530aa 100644 --- a/pkg/tuya/interface.go +++ b/pkg/tuya/interface.go @@ -26,17 +26,13 @@ type TuyaAPI interface { type TuyaClient struct { TuyaAPI - httpClient *http.Client - mqtt *TuyaMqttClient - streamMode string - baseUrl string - accessToken string - refreshToken string - expireTime int64 - deviceId string - uid string - skill *Skill - iceServers []pionWebrtc.ICEServer + httpClient *http.Client + mqtt *TuyaMqttClient + baseUrl string + expireTime int64 + deviceId string + skill *Skill + iceServers []pionWebrtc.ICEServer } type AudioAttributes struct { @@ -44,11 +40,11 @@ type AudioAttributes struct { HardwareCapability []int `json:"hardware_capability"` // 1 = mic, 2 = speaker } -type OpenApiICE struct { +type ICEServer struct { Urls string `json:"urls"` - Username string `json:"username"` - Credential string `json:"credential"` - TTL int `json:"ttl"` + Username string `json:"username,omitempty"` + Credential string `json:"credential,omitempty"` + TTL int `json:"ttl,omitempty"` } type WebICE struct { @@ -58,7 +54,7 @@ type WebICE struct { } type P2PConfig struct { - Ices []OpenApiICE `json:"ices"` + Ices []ICEServer `json:"ices"` } type AudioSkill struct { diff --git a/pkg/tuya/open_api.go b/pkg/tuya/open_api.go deleted file mode 100644 index 4bd2a044..00000000 --- a/pkg/tuya/open_api.go +++ /dev/null @@ -1,473 +0,0 @@ -package tuya - -import ( - "crypto/md5" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "time" - - "github.com/google/uuid" -) - -const ( - TUYA_HOST = "apigw.iotbing.com" - TUYA_CLIENT_ID = "HA_3y9q4ak7g4ephrvke" - TUYA_SCHEMA = "haauthorize" -) - -type OpenApiMQTTConfig struct { - ClientID string `json:"clientId"` - ExpireTime int `json:"expireTime"` - Password string `json:"password"` - Topic struct { - DevID struct { - Pub string `json:"pub"` - Sub string `json:"sub"` - } `json:"devId"` - OwnerID struct { - Sub string `json:"sub"` - } `json:"ownerId"` - } `json:"topic"` - URL string `json:"url"` - Username string `json:"username"` -} - -type OpenApiMQTTConfigRequest struct { - LinkID string `json:"linkId"` -} - -type OpenApiMQTTConfigResponse struct { - Success bool `json:"success"` - Result OpenApiMQTTConfig `json:"result"` - Msg string `json:"msg,omitempty"` -} - -type TokenInfo struct { - AccessToken string `json:"access_token"` - ExpireTime int64 `json:"expire_time"` - RefreshToken string `json:"refresh_token"` -} - -type LoginResult struct { - AccessToken string `json:"access_token"` - Endpoint string `json:"endpoint"` - ExpireTime int64 `json:"expire_time"` // seconds - RefreshToken string `json:"refresh_token"` - TerminalID string `json:"terminal_id"` - UID string `json:"uid"` - Username string `json:"username"` -} - -type LoginResponse struct { - Timestamp int64 `json:"t"` - Success bool `json:"success"` - Result LoginResult `json:"result"` - Msg string `json:"msg,omitempty"` -} - -type QRResponse struct { - Success bool `json:"success"` - Result struct { - Code string `json:"qrcode"` - } `json:"result"` - Msg string `json:"msg,omitempty"` -} - -type Home struct { - ID int `json:"id"` - Name string `json:"name"` - OwnerID string `json:"ownerId"` - Background string `json:"background"` - GeoName string `json:"geoName"` - Lat float64 `json:"lat"` - Lon float64 `json:"lon"` - GmtCreate int64 `json:"gmtCreate"` - GmtModified int64 `json:"gmtModified"` - GroupID int64 `json:"groupId"` - Status bool `json:"status"` - UID string `json:"uid"` -} - -type HomesResponse struct { - Success bool `json:"success"` - Result []Home `json:"result"` - Msg string `json:"msg,omitempty"` -} - -type DeviceFunction struct { - Code string `json:"code"` - Desc string `json:"desc"` - Name string `json:"name"` - Type string `json:"type"` - Values map[string]any `json:"values"` -} - -type DeviceStatusRange struct { - Code string `json:"code"` - Type string `json:"type"` - Values map[string]any `json:"values"` -} - -type Device struct { - ID string `json:"id"` - Name string `json:"name"` - LocalKey string `json:"local_key"` - Category string `json:"category"` - ProductID string `json:"product_id"` - ProductName string `json:"product_name"` - Sub bool `json:"sub"` - UUID string `json:"uuid"` - AssetID string `json:"asset_id"` - Online bool `json:"online"` - Icon string `json:"icon"` - IP string `json:"ip"` - TimeZone string `json:"time_zone"` - ActiveTime int64 `json:"active_time"` - CreateTime int64 `json:"create_time"` - UpdateTime int64 `json:"update_time"` -} - -type DeviceRequest struct { - HomeID string `json:"homeId"` -} - -type DeviceResponse struct { - Success bool `json:"success"` - Result []Device `json:"result"` - Msg string `json:"msg,omitempty"` -} - -type TuyaOpenApiClient struct { - TuyaClient - terminalId string - refreshingToken bool -} - -func NewTuyaOpenApiClient( - baseUrl string, - uid string, - deviceId string, - terminalId string, - tokenInfoOrString any, - streamMode string, -) (*TuyaOpenApiClient, error) { - tokenInfo, err := ParseTokenInfo(tokenInfoOrString) - if err != nil { - return nil, fmt.Errorf("failed to parse token info: %w", err) - } - - mqttClient := NewTuyaMqttClient(deviceId) - - client := &TuyaOpenApiClient{ - TuyaClient: TuyaClient{ - httpClient: &http.Client{Timeout: 15 * time.Second}, - mqtt: mqttClient, - uid: uid, - deviceId: deviceId, - accessToken: tokenInfo.AccessToken, - refreshToken: tokenInfo.RefreshToken, - expireTime: tokenInfo.ExpireTime, - streamMode: streamMode, - baseUrl: baseUrl, - }, - terminalId: terminalId, - refreshingToken: false, - } - - return client, nil -} - -// WebRTC Flow (not supported yet) -func (c *TuyaOpenApiClient) Init() error { - if err := c.initToken(); err != nil { - return fmt.Errorf("failed to initialize token: %w", err) - } - - return fmt.Errorf("stream mode %s is not supported", c.streamMode) -} - -func (c *TuyaOpenApiClient) GetStreamUrl(streamType string) (streamUrl string, err error) { - if err := c.initToken(); err != nil { - return "", fmt.Errorf("failed to initialize token: %w", err) - } - - urlPath := fmt.Sprintf("/v1.0/m/ipc/%s/stream/actions/allocate", c.deviceId) - - request := &AllocateRequest{ - Type: streamType, - } - - body, err := c.request("POST", urlPath, nil, request) - if err != nil { - return "", err - } - - var allocResponse AllocateResponse - err = json.Unmarshal(body, &allocResponse) - if err != nil { - return "", err - } - - if !allocResponse.Success { - return "", fmt.Errorf(allocResponse.Msg) - } - - return allocResponse.Result.URL, nil -} - -func (c *TuyaOpenApiClient) GetAllDevices() ([]Device, error) { - homes, err := c.queryHomes() - if err != nil { - return nil, err - } - - time.Sleep(2 * time.Second) - deviceMap := make(map[string]Device) - - for i, home := range homes { - if i > 0 { - time.Sleep(500 * time.Millisecond) - } - - devices, err := c.queryDevicesByHome(home.OwnerID) - if err != nil { - return nil, err - } - - for _, device := range devices { - // https://github.com/home-assistant/core/blob/088cfc3576e0018ad1df373c08549092918e6530/homeassistant/components/tuya/camera.py#L19 - if device.Category == "sp" || device.Category == "dghsxj" { - deviceMap[device.ID] = device - } - } - } - - var devices []Device - for _, device := range deviceMap { - devices = append(devices, device) - } - - return devices, nil -} - -func (c *TuyaOpenApiClient) loadHubConfig() (config *MQTTConfig, err error) { - request := OpenApiMQTTConfigRequest{ - LinkID: fmt.Sprintf("tuya-device-sharing-sdk-go.%s", uuid.New().String()), - } - - body, err := c.request("POST", "/v1.0/m/life/ha/access/config", nil, request) - if err != nil { - return nil, err - } - - var mqttConfigResponse OpenApiMQTTConfigResponse - if err := json.Unmarshal(body, &mqttConfigResponse); err != nil { - return nil, err - } - - if !mqttConfigResponse.Success { - return nil, fmt.Errorf("failed to get MQTT config: %s", mqttConfigResponse.Msg) - } - - return &MQTTConfig{ - Url: mqttConfigResponse.Result.URL, - Username: mqttConfigResponse.Result.Username, - Password: mqttConfigResponse.Result.Password, - ClientID: mqttConfigResponse.Result.ClientID, - PublishTopic: mqttConfigResponse.Result.Topic.DevID.Pub, - SubscribeTopic: mqttConfigResponse.Result.Topic.DevID.Sub, - }, nil -} - -func (c *TuyaOpenApiClient) queryHomes() ([]Home, error) { - body, err := c.request("GET", "/v1.0/m/life/users/homes", nil, nil) - if err != nil { - return nil, err - } - - var homesResponse HomesResponse - if err := json.Unmarshal(body, &homesResponse); err != nil { - return nil, err - } - - if !homesResponse.Success { - return nil, fmt.Errorf("failed to get homes: %s", homesResponse.Msg) - } - - return homesResponse.Result, nil -} - -func (c *TuyaOpenApiClient) queryDevicesByHome(homeID string) ([]Device, error) { - params := DeviceRequest{ - HomeID: homeID, - } - - body, err := c.request("GET", "/v1.0/m/life/ha/home/devices", params, nil) - if err != nil { - return nil, err - } - - var devicesResponse DeviceResponse - if err := json.Unmarshal(body, &devicesResponse); err != nil { - return nil, err - } - - if !devicesResponse.Success { - return nil, fmt.Errorf("failed to get devices: %s", devicesResponse.Msg) - } - - return devicesResponse.Result, nil -} - -// https://github.com/tuya/tuya-device-sharing-sdk/blob/main/tuya_sharing/customerapi.py -func (c *TuyaOpenApiClient) request( - method string, - path string, - params any, - body any, -) ([]byte, error) { - rid := uuid.New().String() - sid := "" - - md5Hash := md5.New() - ridRefreshToken := rid + c.refreshToken - md5Hash.Write([]byte(ridRefreshToken)) - hashKey := hex.EncodeToString(md5Hash.Sum(nil)) - secret := SecretGenerating(rid, sid, hashKey) - - queryEncdata := "" - var reqURL string - if params != nil { - jsonData := FormToJSON(params) - - encryptedData, err := AesGCMEncrypt(jsonData, secret) - if err != nil { - return nil, err - } - - queryEncdata = encryptedData - reqURL = fmt.Sprintf("https://%s%s?encdata=%s", c.baseUrl, path, queryEncdata) - } else { - reqURL = fmt.Sprintf("https://%s%s", c.baseUrl, path) - } - - bodyEncdata := "" - var reqBody io.Reader - if body != nil { - jsonData := FormToJSON(body) - - encryptedData, err := AesGCMEncrypt(jsonData, secret) - if err != nil { - return nil, err - } - - bodyEncdata = encryptedData - encBody := map[string]string{"encdata": bodyEncdata} - bodyBytes, _ := json.Marshal(encBody) - reqBody = strings.NewReader(string(bodyBytes)) - } - - req, err := http.NewRequest(method, reqURL, reqBody) - if err != nil { - return nil, err - } - - t := time.Now().Add(2*time.Second).UnixNano() / int64(time.Millisecond) - headers := map[string]string{ - "X-appKey": TUYA_CLIENT_ID, - "X-requestId": rid, - "X-sid": sid, - "X-time": fmt.Sprintf("%d", t), - "Content-Type": "application/json", - } - - if c.accessToken != "" { - headers["X-token"] = c.accessToken - } - - sign := RestfulSign(hashKey, queryEncdata, bodyEncdata, headers) - headers["X-sign"] = sign - - for key, value := range headers { - req.Header.Set(key, value) - } - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - var resultObj map[string]any - if err := json.Unmarshal(respBody, &resultObj); err != nil { - return nil, err - } - - if resultStr, ok := resultObj["result"].(string); ok { - decrypted, err := AesGCMDecrypt(resultStr, secret) - if err != nil { - return nil, err - } - - var decryptedObj any - if err := json.Unmarshal([]byte(decrypted), &decryptedObj); err == nil { - resultObj["result"] = decryptedObj - } else { - resultObj["result"] = decrypted - } - - updatedResponse, err := json.Marshal(resultObj) - if err != nil { - return nil, fmt.Errorf("failed to marshal updated response: %w", err) - } - - return updatedResponse, nil - } - - return respBody, nil -} - -func (c *TuyaOpenApiClient) initToken() error { - if c.refreshingToken { - return nil - } - - now := time.Now().Unix() - if (c.expireTime - 60) > now { - return nil - } - - c.refreshingToken = true - - urlPath := fmt.Sprintf("/v1.0/m/token/%s", c.refreshToken) - - body, err := c.request("GET", urlPath, nil, nil) - if err != nil { - return err - } - - var loginResponse LoginResponse - if err := json.Unmarshal(body, &loginResponse); err != nil { - return err - } - - if !loginResponse.Success { - return fmt.Errorf("failed to get token: %s", loginResponse.Msg) - } - - c.accessToken = loginResponse.Result.AccessToken - c.refreshToken = loginResponse.Result.RefreshToken - c.expireTime = loginResponse.Timestamp + loginResponse.Result.ExpireTime - c.refreshingToken = false - - return nil -} diff --git a/pkg/tuya/tuya_api.go b/pkg/tuya/tuya_api.go new file mode 100644 index 00000000..454efb46 --- /dev/null +++ b/pkg/tuya/tuya_api.go @@ -0,0 +1,590 @@ +package tuya + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "time" + + "github.com/AlexxIT/go2rtc/pkg/webrtc" +) + +type LoginTokenRequest struct { + CountryCode string `json:"countryCode"` + Username string `json:"username"` + IsUid bool `json:"isUid"` +} + +type LoginTokenResponse struct { + Result LoginToken `json:"result"` + Success bool `json:"success"` + Msg string `json:"errorMsg,omitempty"` +} + +type LoginToken struct { + Token string `json:"token"` + Exponent string `json:"exponent"` + PublicKey string `json:"publicKey"` + PbKey string `json:"pbKey"` +} + +type PasswordLoginRequest struct { + CountryCode string `json:"countryCode"` + Email string `json:"email,omitempty"` + Mobile string `json:"mobile,omitempty"` + Passwd string `json:"passwd"` + Token string `json:"token"` + IfEncrypt int `json:"ifencrypt"` + Options string `json:"options"` +} + +type PasswordLoginResponse struct { + Result LoginResult `json:"result"` + Success bool `json:"success"` + Status string `json:"status"` + ErrorMsg string `json:"errorMsg,omitempty"` +} + +type LoginResult struct { + Attribute int `json:"attribute"` + ClientId string `json:"clientId"` + DataVersion int `json:"dataVersion"` + Domain Domain `json:"domain"` + Ecode string `json:"ecode"` + Email string `json:"email"` + Extras Extras `json:"extras"` + HeadPic string `json:"headPic"` + ImproveCompanyInfo bool `json:"improveCompanyInfo"` + Nickname string `json:"nickname"` + PartnerIdentity string `json:"partnerIdentity"` + PhoneCode string `json:"phoneCode"` + Receiver string `json:"receiver"` + RegFrom int `json:"regFrom"` + Sid string `json:"sid"` + SnsNickname string `json:"snsNickname"` + TempUnit int `json:"tempUnit"` + Timezone string `json:"timezone"` + TimezoneId string `json:"timezoneId"` + Uid string `json:"uid"` + UserType int `json:"userType"` + Username string `json:"username"` +} + +type Domain struct { + AispeechHttpsUrl string `json:"aispeechHttpsUrl"` + AispeechQuicUrl string `json:"aispeechQuicUrl"` + DeviceHttpUrl string `json:"deviceHttpUrl"` + DeviceHttpsPskUrl string `json:"deviceHttpsPskUrl"` + DeviceHttpsUrl string `json:"deviceHttpsUrl"` + DeviceMediaMqttUrl string `json:"deviceMediaMqttUrl"` + DeviceMediaMqttsUrl string `json:"deviceMediaMqttsUrl"` + DeviceMqttsPskUrl string `json:"deviceMqttsPskUrl"` + DeviceMqttsUrl string `json:"deviceMqttsUrl"` + GwApiUrl string `json:"gwApiUrl"` + GwMqttUrl string `json:"gwMqttUrl"` + HttpPort int `json:"httpPort"` + HttpsPort int `json:"httpsPort"` + HttpsPskPort int `json:"httpsPskPort"` + MobileApiUrl string `json:"mobileApiUrl"` + MobileMediaMqttUrl string `json:"mobileMediaMqttUrl"` + MobileMqttUrl string `json:"mobileMqttUrl"` + MobileMqttsUrl string `json:"mobileMqttsUrl"` + MobileQuicUrl string `json:"mobileQuicUrl"` + MqttPort int `json:"mqttPort"` + MqttQuicUrl string `json:"mqttQuicUrl"` + MqttsPort int `json:"mqttsPort"` + MqttsPskPort int `json:"mqttsPskPort"` + RegionCode string `json:"regionCode"` +} + +type Extras struct { + HomeId string `json:"homeId"` + SceneType string `json:"sceneType"` +} + +type AppInfoResponse struct { + Result AppInfo `json:"result"` + T int64 `json:"t"` + Success bool `json:"success"` + Msg string `json:"errorMsg,omitempty"` +} + +type AppInfo struct { + AppId int `json:"appId"` + AppName string `json:"appName"` + ClientId string `json:"clientId"` + Icon string `json:"icon"` +} + +type MQTTConfigResponse struct { + Result TuyaApiMQTTConfig `json:"result"` + Success bool `json:"success"` + Msg string `json:"errorMsg,omitempty"` +} + +type TuyaApiMQTTConfig struct { + Msid string `json:"msid"` + Password string `json:"password"` +} + +type HomeListResponse struct { + Result []Home `json:"result"` + T int64 `json:"t"` + Success bool `json:"success"` + Msg string `json:"errorMsg,omitempty"` +} + +type SharedHomeListResponse struct { + Result SharedHome `json:"result"` + T int64 `json:"t"` + Success bool `json:"success"` + Msg string `json:"errorMsg,omitempty"` +} + +type SharedHome struct { + SecurityWebCShareInfoList []struct { + DeviceInfoList []Device `json:"deviceInfoList"` + Nickname string `json:"nickname"` + Username string `json:"username"` + } `json:"securityWebCShareInfoList"` +} + +type Home struct { + Admin bool `json:"admin"` + Background string `json:"background"` + DealStatus int `json:"dealStatus"` + DisplayOrder int `json:"displayOrder"` + GeoName string `json:"geoName"` + Gid int `json:"gid"` + GmtCreate int64 `json:"gmtCreate"` + GmtModified int64 `json:"gmtModified"` + GroupId int `json:"groupId"` + GroupUserId int `json:"groupUserId"` + Id int `json:"id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + ManagementStatus bool `json:"managementStatus"` + Name string `json:"name"` + OwnerId string `json:"ownerId"` + Role int `json:"role"` + Status bool `json:"status"` + Uid string `json:"uid"` +} + +type RoomListRequest struct { + HomeId string `json:"homeId"` +} + +type RoomListResponse struct { + Result []Room `json:"result"` + T int64 `json:"t"` + Success bool `json:"success"` + Msg string `json:"errorMsg,omitempty"` +} + +type Room struct { + DeviceCount int `json:"deviceCount"` + DeviceList []Device `json:"deviceList"` + RoomId string `json:"roomId"` + RoomName string `json:"roomName"` +} + +type Device struct { + Category string `json:"category"` + DeviceId string `json:"deviceId"` + DeviceName string `json:"deviceName"` + P2pType int `json:"p2pType"` + ProductId string `json:"productId"` + SupportCloudStorage bool `json:"supportCloudStorage"` + Uuid string `json:"uuid"` +} + +type TuyaApiWebRTCConfigRequest struct { + DevId string `json:"devId"` + ClientTraceId string `json:"clientTraceId"` +} + +type TuyaApiWebRTCConfigResponse struct { + Result TuyaWebRTCConfig `json:"result"` + Success bool `json:"success"` + Msg string `json:"errorMsg,omitempty"` +} + +type TuyaWebRTCConfig struct { + AudioAttributes AudioAttributes `json:"audioAttributes"` + Auth string `json:"auth"` + GatewayId string `json:"gatewayId"` + Id string `json:"id"` + LocalKey string `json:"localKey"` + MotoId string `json:"motoId"` + NodeId string `json:"nodeId"` + P2PConfig P2PConfig `json:"p2pConfig"` + ProtocolVersion string `json:"protocolVersion"` + Skill string `json:"skill"` + Sub bool `json:"sub"` + SupportWebrtcRecord bool `json:"supportWebrtcRecord"` + SupportsPtz bool `json:"supportsPtz"` + SupportsWebrtc bool `json:"supportsWebrtc"` + VedioClarity int `json:"vedioClarity"` + VedioClaritys []int `json:"vedioClaritys"` + VideoClarity int `json:"videoClarity"` +} + +type TuyaApiClient struct { + TuyaClient + + email string + password string + countryCode string + mqttsUrl string +} + +type Region struct { + Name string `json:"name"` + Host string `json:"host"` + Description string `json:"description"` + Continent string `json:"continent"` +} + +var AvailableRegions = []Region{ + {"eu-central", "protect-eu.ismartlife.me", "Central Europe", "EU"}, + {"eu-east", "protect-we.ismartlife.me", "East Europe", "EU"}, + {"us-west", "protect-us.ismartlife.me", "West America", "AZ"}, + {"us-east", "protect-ue.ismartlife.me", "East America", "AZ"}, + {"china", "protect.ismartlife.me", "China", "AY"}, + {"india", "protect-in.ismartlife.me", "India", "IN"}, +} + +func NewTuyaApiClient(httpClient *http.Client, baseUrl, email, password, deviceId string) (*TuyaApiClient, error) { + var region *Region + for _, r := range AvailableRegions { + if r.Host == baseUrl { + region = &r + break + } + } + + if region == nil { + return nil, fmt.Errorf("invalid region: %s", baseUrl) + } + + if httpClient == nil { + httpClient = CreateHTTPClientWithSession() + } + + mqttClient := NewTuyaMqttClient(deviceId) + + client := &TuyaApiClient{ + TuyaClient: TuyaClient{ + httpClient: httpClient, + mqtt: mqttClient, + deviceId: deviceId, + expireTime: 0, + baseUrl: baseUrl, + }, + email: email, + password: password, + countryCode: region.Continent, + } + + return client, nil +} + +// WebRTC Flow +func (c *TuyaApiClient) Init() error { + if err := c.initToken(); err != nil { + return fmt.Errorf("failed to initialize token: %w", err) + } + + webrtcConfig, err := c.loadWebrtcConfig() + if err != nil { + return fmt.Errorf("failed to load webrtc config: %w", err) + } + + hubConfig, err := c.loadHubConfig() + if err != nil { + return fmt.Errorf("failed to load hub config: %w", err) + } + + if err := c.mqtt.Start(hubConfig, webrtcConfig, c.skill.WebRTC); err != nil { + return fmt.Errorf("failed to start MQTT: %w", err) + } + + return nil +} + +func (c *TuyaApiClient) GetStreamUrl(streamType string) (streamUrl string, err error) { + return "", errors.New("not supported") +} + +func (c *TuyaApiClient) GetAppInfo() (*AppInfoResponse, error) { + url := fmt.Sprintf("https://%s/api/customized/web/app/info", c.baseUrl) + + body, err := c.request("POST", url, nil) + if err != nil { + return nil, err + } + + var appInfoResponse AppInfoResponse + if err := json.Unmarshal(body, &appInfoResponse); err != nil { + return nil, err + } + + if !appInfoResponse.Success { + return nil, errors.New(appInfoResponse.Msg) + } + + return &appInfoResponse, nil +} + +func (c *TuyaApiClient) GetHomeList() (*HomeListResponse, error) { + url := fmt.Sprintf("https://%s/api/new/common/homeList", c.baseUrl) + + body, err := c.request("POST", url, nil) + if err != nil { + return nil, err + } + + var homeListResponse HomeListResponse + if err := json.Unmarshal(body, &homeListResponse); err != nil { + return nil, err + } + + if !homeListResponse.Success { + return nil, errors.New(homeListResponse.Msg) + } + + return &homeListResponse, nil +} + +func (c *TuyaApiClient) GetSharedHomeList() (*SharedHomeListResponse, error) { + url := fmt.Sprintf("https://%s/api/new/playback/shareList", c.baseUrl) + + body, err := c.request("POST", url, nil) + if err != nil { + return nil, err + } + + var sharedHomeListResponse SharedHomeListResponse + if err := json.Unmarshal(body, &sharedHomeListResponse); err != nil { + return nil, err + } + + if !sharedHomeListResponse.Success { + return nil, errors.New(sharedHomeListResponse.Msg) + } + + return &sharedHomeListResponse, nil +} + +func (c *TuyaApiClient) GetRoomList(homeId string) (*RoomListResponse, error) { + url := fmt.Sprintf("https://%s/api/new/common/roomList", c.baseUrl) + + data := RoomListRequest{ + HomeId: homeId, + } + + body, err := c.request("POST", url, data) + if err != nil { + return nil, err + } + + var roomListResponse RoomListResponse + if err := json.Unmarshal(body, &roomListResponse); err != nil { + return nil, err + } + + if !roomListResponse.Success { + return nil, errors.New(roomListResponse.Msg) + } + + return &roomListResponse, nil +} + +func (c *TuyaApiClient) initToken() error { + tokenUrl := fmt.Sprintf("https://%s/api/login/token", c.baseUrl) + + tokenReq := LoginTokenRequest{ + CountryCode: c.countryCode, + Username: c.email, + IsUid: false, + } + + body, err := c.request("POST", tokenUrl, tokenReq) + if err != nil { + return err + } + + var tokenResp LoginTokenResponse + if err := json.Unmarshal(body, &tokenResp); err != nil { + return err + } + + if !tokenResp.Success { + return errors.New(tokenResp.Msg) + } + + encryptedPassword, err := EncryptPassword(c.password, tokenResp.Result.PbKey) + if err != nil { + return fmt.Errorf("failed to encrypt password: %v", err) + } + var loginUrl string + + loginReq := PasswordLoginRequest{ + CountryCode: c.countryCode, + Passwd: encryptedPassword, + Token: tokenResp.Result.Token, + IfEncrypt: 1, + Options: `{"group":1}`, + } + + if IsEmailAddress(c.email) { + loginUrl = fmt.Sprintf("https://%s/api/private/email/login", c.baseUrl) + loginReq.Email = c.email + } else { + loginUrl = fmt.Sprintf("https://%s/api/private/phone/login", c.baseUrl) + loginReq.Mobile = c.email + } + + body, err = c.request("POST", loginUrl, loginReq) + if err != nil { + return err + } + + var loginResp *PasswordLoginResponse + if err := json.Unmarshal(body, &loginResp); err != nil { + return err + } + + if !loginResp.Success { + return errors.New(loginResp.ErrorMsg) + } + + c.mqttsUrl = fmt.Sprintf("wss://%s/mqtt", loginResp.Result.Domain.MobileMqttsUrl) + c.expireTime = time.Now().Unix() + 2*24*60*60 // 2 days in seconds + + return nil +} + +func (c *TuyaApiClient) loadWebrtcConfig() (*WebRTCConfig, error) { + url := fmt.Sprintf("https://%s/api/jarvis/config", c.baseUrl) + + data := TuyaApiWebRTCConfigRequest{ + DevId: c.deviceId, + ClientTraceId: fmt.Sprintf("%x", rand.Int63()), + } + + body, err := c.request("POST", url, data) + if err != nil { + return nil, err + } + + var webRTCConfigResponse TuyaApiWebRTCConfigResponse + err = json.Unmarshal(body, &webRTCConfigResponse) + if err != nil { + return nil, err + } + + if !webRTCConfigResponse.Success { + return nil, errors.New(webRTCConfigResponse.Msg) + } + + err = json.Unmarshal([]byte(webRTCConfigResponse.Result.Skill), &c.skill) + if err != nil { + return nil, err + } + + iceServers, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) + if err != nil { + return nil, err + } + + c.iceServers, err = webrtc.UnmarshalICEServers(iceServers) + if err != nil { + return nil, err + } + + return &WebRTCConfig{ + AudioAttributes: webRTCConfigResponse.Result.AudioAttributes, + Auth: webRTCConfigResponse.Result.Auth, + ID: webRTCConfigResponse.Result.Id, + MotoID: webRTCConfigResponse.Result.MotoId, + P2PConfig: webRTCConfigResponse.Result.P2PConfig, + ProtocolVersion: webRTCConfigResponse.Result.ProtocolVersion, + Skill: webRTCConfigResponse.Result.Skill, + SupportsWebRTCRecord: webRTCConfigResponse.Result.SupportWebrtcRecord, + SupportsWebRTC: webRTCConfigResponse.Result.SupportsWebrtc, + VedioClaritiy: webRTCConfigResponse.Result.VedioClarity, + VideoClaritiy: webRTCConfigResponse.Result.VideoClarity, + VideoClarities: webRTCConfigResponse.Result.VedioClaritys, + }, nil +} + +func (c *TuyaApiClient) loadHubConfig() (config *MQTTConfig, err error) { + mqttUrl := fmt.Sprintf("https://%s/api/jarvis/mqtt", c.baseUrl) + + mqttBody, err := c.request("POST", mqttUrl, nil) + if err != nil { + return nil, err + } + + var mqttConfigResponse MQTTConfigResponse + err = json.Unmarshal(mqttBody, &mqttConfigResponse) + if err != nil { + return nil, err + } + + if !mqttConfigResponse.Success { + return nil, errors.New(mqttConfigResponse.Msg) + } + + return &MQTTConfig{ + Url: c.mqttsUrl, + ClientID: fmt.Sprintf("web_%s", mqttConfigResponse.Result.Msid), + Username: fmt.Sprintf("web_%s", mqttConfigResponse.Result.Msid), + Password: mqttConfigResponse.Result.Password, + PublishTopic: "/av/moto/moto_id/u/{device_id}", + SubscribeTopic: fmt.Sprintf("/av/u/%s", mqttConfigResponse.Result.Msid), + }, nil +} + +func (c *TuyaApiClient) request(method string, url string, body any) ([]byte, error) { + var bodyReader io.Reader + if body != nil { + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(jsonBody) + } + + req, err := http.NewRequest(method, url, bodyReader) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("Accept", "*/*") + req.Header.Set("Origin", fmt.Sprintf("https://%s", c.baseUrl)) + + response, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer response.Body.Close() + + res, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + if response.StatusCode != http.StatusOK { + return nil, err + } + + return res, nil +} \ No newline at end of file diff --git a/www/add.html b/www/add.html index f16d4d45..5a04cd96 100644 --- a/www/add.html +++ b/www/add.html @@ -28,7 +28,6 @@ } - @@ -283,14 +282,17 @@
-

Attention: Cameras added through QR Code does not support webrtc mode!

-
- - -
- - @@ -298,91 +300,31 @@
From 3036dd7cfea384b0b2241f1a9918db4a5a0233df Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 26 May 2025 18:36:21 +0200 Subject: [PATCH 31/60] refactor --- README.md | 16 +++++----- internal/tuya/tuya.go | 2 +- pkg/tuya/client.go | 6 ++-- pkg/tuya/{tuya_api.go => smart_api.go} | 42 +++++++++++++------------- 4 files changed, 33 insertions(+), 33 deletions(-) rename pkg/tuya/{tuya_api.go => smart_api.go} (92%) diff --git a/README.md b/README.md index 7a257ae0..a4992acd 100644 --- a/README.md +++ b/README.md @@ -568,14 +568,14 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. #### Source: Tuya -[Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. Go2rtc supports `Cloud API` and `Tuya API`. +[Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. Go2rtc supports `Tuya Cloud API` and `Tuya Smart API`. -The `Cloud API` requires setting up a cloud project in the Tuya Developer Platform to retrieve the required credentials. The `Tuya API` does not require a cloud project and the cameras can be added through the interface via email/password. +The `Tuya Cloud API` requires setting up a cloud project in the Tuya Developer Platform to retrieve the required credentials. The `Tuya Smart API` does not require a cloud project and the cameras can be added through the interface via email/password. -**Cloud API**: +**Tuya Cloud API**: - Obtain `device_id`, `client_id`, `client_secret`, and `uid` from [Tuya IoT Platform](https://iot.tuya.com/). [Here's a guide](https://xzetsubou.github.io/hass-localtuya/cloud_api/). -**Open API**: +**Tuya Smart API**: - Smart Life accounts are not supported, you need to create a Tuya account. If the cameras are already added to the Smart Life app, you need to remove them and add them again to the Tuya Smart app. **Configuring the stream:** @@ -585,19 +585,19 @@ The `Cloud API` requires setting up a cloud project in the Tuya Developer Platfo ```yaml streams: - # Cloud API: WebRTC main stream + # Tuya Cloud API: WebRTC main stream tuya_webrtc: - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX - # Cloud API: WebRTC sub stream + # Tuya Cloud API: WebRTC sub stream tuya_webrtc_sd: - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=sd - # Tuya API: WebRTC main stream + # Tuya Smart API: WebRTC main stream tuya: - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX - # Tuya API: WebRTC sub stream + # Tuya Smart API: WebRTC sub stream tuya: - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX&resolution=sd ``` diff --git a/internal/tuya/tuya.go b/internal/tuya/tuya.go index cb25daa5..c44bdbbc 100644 --- a/internal/tuya/tuya.go +++ b/internal/tuya/tuya.go @@ -55,7 +55,7 @@ func apiTuya(w http.ResponseWriter, r *http.Request) { return } - tuyaAPI, err := tuya.NewTuyaApiClient( + tuyaAPI, err := tuya.NewTuyaSmartApiClient( httpClient, tuyaRegion.Host, email, diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index f5b964e9..50fc80b4 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -49,11 +49,11 @@ func Dial(rawURL string) (core.Producer, error) { query := u.Query() - // Tuya API + // Tuya Smart API email := query.Get("email") password := query.Get("password") - // Cloud API + // Tuya Cloud API uid := query.Get("uid") clientId := query.Get("client_id") clientSecret := query.Get("client_secret") @@ -80,7 +80,7 @@ func Dial(rawURL string) (core.Producer, error) { } if useTuyaApi { - if client.api, err = NewTuyaApiClient(nil, u.Hostname(), email, password, deviceId); err != nil { + if client.api, err = NewTuyaSmartApiClient(nil, u.Hostname(), email, password, deviceId); err != nil { return nil, fmt.Errorf("tuya: %w", err) } } else { diff --git a/pkg/tuya/tuya_api.go b/pkg/tuya/smart_api.go similarity index 92% rename from pkg/tuya/tuya_api.go rename to pkg/tuya/smart_api.go index 454efb46..bcd6bc9d 100644 --- a/pkg/tuya/tuya_api.go +++ b/pkg/tuya/smart_api.go @@ -121,12 +121,12 @@ type AppInfo struct { } type MQTTConfigResponse struct { - Result TuyaApiMQTTConfig `json:"result"` + Result SmartApiMQTTConfig `json:"result"` Success bool `json:"success"` Msg string `json:"errorMsg,omitempty"` } -type TuyaApiMQTTConfig struct { +type SmartApiMQTTConfig struct { Msid string `json:"msid"` Password string `json:"password"` } @@ -203,18 +203,18 @@ type Device struct { Uuid string `json:"uuid"` } -type TuyaApiWebRTCConfigRequest struct { +type SmartApiWebRTCConfigRequest struct { DevId string `json:"devId"` ClientTraceId string `json:"clientTraceId"` } -type TuyaApiWebRTCConfigResponse struct { - Result TuyaWebRTCConfig `json:"result"` +type SmartApiWebRTCConfigResponse struct { + Result SmartApiWebRTCConfig `json:"result"` Success bool `json:"success"` Msg string `json:"errorMsg,omitempty"` } -type TuyaWebRTCConfig struct { +type SmartApiWebRTCConfig struct { AudioAttributes AudioAttributes `json:"audioAttributes"` Auth string `json:"auth"` GatewayId string `json:"gatewayId"` @@ -234,7 +234,7 @@ type TuyaWebRTCConfig struct { VideoClarity int `json:"videoClarity"` } -type TuyaApiClient struct { +type TuyaSmartApiClient struct { TuyaClient email string @@ -259,7 +259,7 @@ var AvailableRegions = []Region{ {"india", "protect-in.ismartlife.me", "India", "IN"}, } -func NewTuyaApiClient(httpClient *http.Client, baseUrl, email, password, deviceId string) (*TuyaApiClient, error) { +func NewTuyaSmartApiClient(httpClient *http.Client, baseUrl, email, password, deviceId string) (*TuyaSmartApiClient, error) { var region *Region for _, r := range AvailableRegions { if r.Host == baseUrl { @@ -278,7 +278,7 @@ func NewTuyaApiClient(httpClient *http.Client, baseUrl, email, password, deviceI mqttClient := NewTuyaMqttClient(deviceId) - client := &TuyaApiClient{ + client := &TuyaSmartApiClient{ TuyaClient: TuyaClient{ httpClient: httpClient, mqtt: mqttClient, @@ -295,7 +295,7 @@ func NewTuyaApiClient(httpClient *http.Client, baseUrl, email, password, deviceI } // WebRTC Flow -func (c *TuyaApiClient) Init() error { +func (c *TuyaSmartApiClient) Init() error { if err := c.initToken(); err != nil { return fmt.Errorf("failed to initialize token: %w", err) } @@ -317,11 +317,11 @@ func (c *TuyaApiClient) Init() error { return nil } -func (c *TuyaApiClient) GetStreamUrl(streamType string) (streamUrl string, err error) { +func (c *TuyaSmartApiClient) GetStreamUrl(streamType string) (streamUrl string, err error) { return "", errors.New("not supported") } -func (c *TuyaApiClient) GetAppInfo() (*AppInfoResponse, error) { +func (c *TuyaSmartApiClient) GetAppInfo() (*AppInfoResponse, error) { url := fmt.Sprintf("https://%s/api/customized/web/app/info", c.baseUrl) body, err := c.request("POST", url, nil) @@ -341,7 +341,7 @@ func (c *TuyaApiClient) GetAppInfo() (*AppInfoResponse, error) { return &appInfoResponse, nil } -func (c *TuyaApiClient) GetHomeList() (*HomeListResponse, error) { +func (c *TuyaSmartApiClient) GetHomeList() (*HomeListResponse, error) { url := fmt.Sprintf("https://%s/api/new/common/homeList", c.baseUrl) body, err := c.request("POST", url, nil) @@ -361,7 +361,7 @@ func (c *TuyaApiClient) GetHomeList() (*HomeListResponse, error) { return &homeListResponse, nil } -func (c *TuyaApiClient) GetSharedHomeList() (*SharedHomeListResponse, error) { +func (c *TuyaSmartApiClient) GetSharedHomeList() (*SharedHomeListResponse, error) { url := fmt.Sprintf("https://%s/api/new/playback/shareList", c.baseUrl) body, err := c.request("POST", url, nil) @@ -381,7 +381,7 @@ func (c *TuyaApiClient) GetSharedHomeList() (*SharedHomeListResponse, error) { return &sharedHomeListResponse, nil } -func (c *TuyaApiClient) GetRoomList(homeId string) (*RoomListResponse, error) { +func (c *TuyaSmartApiClient) GetRoomList(homeId string) (*RoomListResponse, error) { url := fmt.Sprintf("https://%s/api/new/common/roomList", c.baseUrl) data := RoomListRequest{ @@ -405,7 +405,7 @@ func (c *TuyaApiClient) GetRoomList(homeId string) (*RoomListResponse, error) { return &roomListResponse, nil } -func (c *TuyaApiClient) initToken() error { +func (c *TuyaSmartApiClient) initToken() error { tokenUrl := fmt.Sprintf("https://%s/api/login/token", c.baseUrl) tokenReq := LoginTokenRequest{ @@ -470,10 +470,10 @@ func (c *TuyaApiClient) initToken() error { return nil } -func (c *TuyaApiClient) loadWebrtcConfig() (*WebRTCConfig, error) { +func (c *TuyaSmartApiClient) loadWebrtcConfig() (*WebRTCConfig, error) { url := fmt.Sprintf("https://%s/api/jarvis/config", c.baseUrl) - data := TuyaApiWebRTCConfigRequest{ + data := SmartApiWebRTCConfigRequest{ DevId: c.deviceId, ClientTraceId: fmt.Sprintf("%x", rand.Int63()), } @@ -483,7 +483,7 @@ func (c *TuyaApiClient) loadWebrtcConfig() (*WebRTCConfig, error) { return nil, err } - var webRTCConfigResponse TuyaApiWebRTCConfigResponse + var webRTCConfigResponse SmartApiWebRTCConfigResponse err = json.Unmarshal(body, &webRTCConfigResponse) if err != nil { return nil, err @@ -524,7 +524,7 @@ func (c *TuyaApiClient) loadWebrtcConfig() (*WebRTCConfig, error) { }, nil } -func (c *TuyaApiClient) loadHubConfig() (config *MQTTConfig, err error) { +func (c *TuyaSmartApiClient) loadHubConfig() (config *MQTTConfig, err error) { mqttUrl := fmt.Sprintf("https://%s/api/jarvis/mqtt", c.baseUrl) mqttBody, err := c.request("POST", mqttUrl, nil) @@ -552,7 +552,7 @@ func (c *TuyaApiClient) loadHubConfig() (config *MQTTConfig, err error) { }, nil } -func (c *TuyaApiClient) request(method string, url string, body any) ([]byte, error) { +func (c *TuyaSmartApiClient) request(method string, url string, body any) ([]byte, error) { var bodyReader io.Reader if body != nil { jsonBody, err := json.Marshal(body) From a2d422f5cb22d20b10793715bba447332637d535 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 26 May 2025 18:42:24 +0200 Subject: [PATCH 32/60] format --- pkg/tuya/smart_api.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/tuya/smart_api.go b/pkg/tuya/smart_api.go index bcd6bc9d..5f4e74f9 100644 --- a/pkg/tuya/smart_api.go +++ b/pkg/tuya/smart_api.go @@ -122,8 +122,8 @@ type AppInfo struct { type MQTTConfigResponse struct { Result SmartApiMQTTConfig `json:"result"` - Success bool `json:"success"` - Msg string `json:"errorMsg,omitempty"` + Success bool `json:"success"` + Msg string `json:"errorMsg,omitempty"` } type SmartApiMQTTConfig struct { @@ -210,8 +210,8 @@ type SmartApiWebRTCConfigRequest struct { type SmartApiWebRTCConfigResponse struct { Result SmartApiWebRTCConfig `json:"result"` - Success bool `json:"success"` - Msg string `json:"errorMsg,omitempty"` + Success bool `json:"success"` + Msg string `json:"errorMsg,omitempty"` } type SmartApiWebRTCConfig struct { @@ -587,4 +587,4 @@ func (c *TuyaSmartApiClient) request(method string, url string, body any) ([]byt } return res, nil -} \ No newline at end of file +} From 9efc717633f7b2dc7d69fd4d29484c680e0f9ff6 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 26 May 2025 20:18:13 +0200 Subject: [PATCH 33/60] ... --- pkg/tuya/client.go | 6 +++--- pkg/tuya/mqtt.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 50fc80b4..fbf6c31b 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -64,14 +64,14 @@ func Dial(rawURL string) (core.Producer, error) { // Stream params streamResolution := query.Get("resolution") - useTuyaApi := deviceId != "" && email != "" && password != "" + useSmartApi := deviceId != "" && email != "" && password != "" useCloudApi := deviceId != "" && uid != "" && clientId != "" && clientSecret != "" if streamResolution == "" || (streamResolution != "hd" && streamResolution != "sd") { streamResolution = "hd" } - if !useTuyaApi && !useCloudApi { + if !useSmartApi && !useCloudApi { return nil, errors.New("tuya: wrong query params") } @@ -79,7 +79,7 @@ func Dial(rawURL string) (core.Producer, error) { handlers: make(map[uint32]func(*rtp.Packet)), } - if useTuyaApi { + if useSmartApi { if client.api, err = NewTuyaSmartApiClient(nil, u.Hostname(), email, password, deviceId); err != nil { return nil, fmt.Errorf("tuya: %w", err) } diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index deb90b9e..52f928a5 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -231,7 +231,7 @@ func (c *TuyaMqttClient) onMqttCandidate(msg *MqttMessage) { return } - // candidate from device start with "a=", end with "\r\n", which are not needed by Chrome webRTC + // fix candidates candidateFrame.Candidate = strings.TrimPrefix(candidateFrame.Candidate, "a=") candidateFrame.Candidate = strings.TrimSuffix(candidateFrame.Candidate, "\r\n") From c38c8a7fcedbe641c2bb6a2116f31a194648b55e Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 2 Jun 2025 22:23:23 +0300 Subject: [PATCH 34/60] readme --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a4992acd..4eb6c404 100644 --- a/README.md +++ b/README.md @@ -568,15 +568,16 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. #### Source: Tuya -[Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. Go2rtc supports `Tuya Cloud API` and `Tuya Smart API`. +[Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. Go2rtc supports `Tuya Smart API` and `Tuya Cloud API`. The `Tuya Cloud API` requires setting up a cloud project in the Tuya Developer Platform to retrieve the required credentials. The `Tuya Smart API` does not require a cloud project and the cameras can be added through the interface via email/password. +**Tuya Smart API (recommended)**: +- Smart Life accounts are not supported, you need to create a Tuya account. If the cameras are already added to the Smart Life app, you need to remove them and add them again to the Tuya Smart app. + **Tuya Cloud API**: - Obtain `device_id`, `client_id`, `client_secret`, and `uid` from [Tuya IoT Platform](https://iot.tuya.com/). [Here's a guide](https://xzetsubou.github.io/hass-localtuya/cloud_api/). - -**Tuya Smart API**: -- Smart Life accounts are not supported, you need to create a Tuya account. If the cameras are already added to the Smart Life app, you need to remove them and add them again to the Tuya Smart app. +- Please ensure that you have subscribed to the `IoT Video Live Stream` service (Free Trial) in the Tuya Developer Platform, otherwise the stream will not work (Tuya Developer Platform > Service API > Authorize > IoT Video Live Stream). **Configuring the stream:** - Use `resolution` parameter to select the stream (not all cameras support `hd` stream through WebRTC even if the camera has it): @@ -594,11 +595,11 @@ streams: - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=sd # Tuya Smart API: WebRTC main stream - tuya: + tuya_main: - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX # Tuya Smart API: WebRTC sub stream - tuya: + tuya_sub: - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX&resolution=sd ``` From e44f1ad53e1e59ab59ddcfbdf3b74d4879751d7c Mon Sep 17 00:00:00 2001 From: seydx <34152761+seydx@users.noreply.github.com> Date: Tue, 3 Jun 2025 06:43:43 +0300 Subject: [PATCH 35/60] Update README.md Co-authored-by: Felipe Santos --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4eb6c404..29af3e17 100644 --- a/README.md +++ b/README.md @@ -570,12 +570,12 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. [Tuya](https://www.tuya.com/) proprietary camera protocol with **two way audio** support. Go2rtc supports `Tuya Smart API` and `Tuya Cloud API`. -The `Tuya Cloud API` requires setting up a cloud project in the Tuya Developer Platform to retrieve the required credentials. The `Tuya Smart API` does not require a cloud project and the cameras can be added through the interface via email/password. - **Tuya Smart API (recommended)**: -- Smart Life accounts are not supported, you need to create a Tuya account. If the cameras are already added to the Smart Life app, you need to remove them and add them again to the Tuya Smart app. +- Cameras can be discovered through the go2rtc web interface via Tuya Smart account (Add > Tuya > Select region and fill in email and password > Login). +- Smart Life accounts are not supported, you need to create a Tuya Smart account. If the cameras are already added to the Smart Life app, you need to remove them and add them again to the Tuya Smart app. **Tuya Cloud API**: +- Requires setting up a cloud project in the Tuya Developer Platform. - Obtain `device_id`, `client_id`, `client_secret`, and `uid` from [Tuya IoT Platform](https://iot.tuya.com/). [Here's a guide](https://xzetsubou.github.io/hass-localtuya/cloud_api/). - Please ensure that you have subscribed to the `IoT Video Live Stream` service (Free Trial) in the Tuya Developer Platform, otherwise the stream will not work (Tuya Developer Platform > Service API > Authorize > IoT Video Live Stream). From 3149b6f750bf803c6997702d9690223317ced259 Mon Sep 17 00:00:00 2001 From: seydx <34152761+seydx@users.noreply.github.com> Date: Tue, 3 Jun 2025 06:44:03 +0300 Subject: [PATCH 36/60] Update README.md Co-authored-by: Felipe Santos --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 29af3e17..47482652 100644 --- a/README.md +++ b/README.md @@ -586,6 +586,14 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. ```yaml streams: + # Tuya Smart API: WebRTC main stream (use Add > Tuya to discover the URL) + tuya_main: + - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX + + # Tuya Smart API: WebRTC sub stream (use Add > Tuya to discover the URL) + tuya_sub: + - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX&resolution=sd + # Tuya Cloud API: WebRTC main stream tuya_webrtc: - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX @@ -593,14 +601,6 @@ streams: # Tuya Cloud API: WebRTC sub stream tuya_webrtc_sd: - tuya://openapi.tuyaus.com?device_id=XXX&uid=XXX&client_id=XXX&client_secret=XXX&resolution=sd - - # Tuya Smart API: WebRTC main stream - tuya_main: - - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX - - # Tuya Smart API: WebRTC sub stream - tuya_sub: - - tuya://protect-us.ismartlife.me?device_id=XXX&email=XXX&password=XXX&resolution=sd ``` #### Source: GoPro From 60b6b93ff8e6a3d4d8036056029244c8d77032cb Mon Sep 17 00:00:00 2001 From: seydx Date: Sun, 8 Jun 2025 20:34:25 +0200 Subject: [PATCH 37/60] fix concurrent writes and improve mqtt --- pkg/tuya/client.go | 49 ++++++++++++++++++++++++++++++++++------------ pkg/tuya/mqtt.go | 6 +++++- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index fbf6c31b..277848e7 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -6,6 +6,7 @@ import ( "fmt" "net/url" "regexp" + "sync" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/webrtc" @@ -24,6 +25,7 @@ type Client struct { isHEVC bool connected core.Waiter closed bool + handlersMu sync.RWMutex handlers map[uint32]func(*rtp.Packet) } @@ -222,7 +224,7 @@ func Dial(rawURL string) (core.Producer, error) { return } - if handler, ok := client.handlers[packet.SSRC]; ok { + if handler, ok := client.getHandler(packet.SSRC); ok { handler(packet) } } @@ -368,16 +370,20 @@ func (c *Client) Start() error { } } - c.handlers[c.videoSSRC] = func(packet *rtp.Packet) { - if video != nil { - video.WriteRTP(packet) - } + if c.videoSSRC != 0 { + c.setHandler(c.videoSSRC, func(packet *rtp.Packet) { + if video != nil { + video.WriteRTP(packet) + } + }) } - c.handlers[c.audioSSRC] = func(packet *rtp.Packet) { - if audio != nil { - audio.WriteRTP(packet) - } + if c.audioSSRC != 0 { + c.setHandler(c.audioSSRC, func(packet *rtp.Packet) { + if audio != nil { + audio.WriteRTP(packet) + } + }) } return c.conn.Start() @@ -390,9 +396,7 @@ func (c *Client) Stop() error { c.closed = true - for ssrc := range c.handlers { - delete(c.handlers, ssrc) - } + c.clearHandlers() if c.conn != nil { _ = c.conn.Stop() @@ -414,6 +418,27 @@ func (c *Client) MarshalJSON() ([]byte, error) { return c.conn.MarshalJSON() } +func (c *Client) setHandler(ssrc uint32, handler func(*rtp.Packet)) { + c.handlersMu.Lock() + defer c.handlersMu.Unlock() + c.handlers[ssrc] = handler +} + +func (c *Client) getHandler(ssrc uint32) (func(*rtp.Packet), bool) { + c.handlersMu.RLock() + defer c.handlersMu.RUnlock() + handler, ok := c.handlers[ssrc] + return handler, ok +} + +func (c *Client) clearHandlers() { + c.handlersMu.Lock() + defer c.handlersMu.Unlock() + for ssrc := range c.handlers { + delete(c.handlers, ssrc) + } +} + func (c *Client) probe(msg pion.DataChannelMessage) (bool, error) { // fmt.Printf("[tuya] Received string message: %s\n", string(msg.Data)) diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index 52f928a5..e5565487 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -109,7 +109,11 @@ func (c *TuyaMqttClient) Start(hubConfig *MQTTConfig, webrtcConfig *WebRTCConfig SetUsername(hubConfig.Username). SetPassword(hubConfig.Password). SetOnConnectHandler(c.onConnect). - SetConnectTimeout(10 * time.Second) + SetAutoReconnect(true). + SetMaxReconnectInterval(30 * time.Second). + SetConnectTimeout(15 * time.Second). + SetKeepAlive(30 * time.Second). + SetPingTimeout(15 * time.Second) c.client = mqtt.NewClient(opts) From 47e87281a1e494df8a55ef7cbfc46f97abbc45f8 Mon Sep 17 00:00:00 2001 From: seydx Date: Sun, 8 Jun 2025 20:38:25 +0200 Subject: [PATCH 38/60] increase timeout --- pkg/tuya/mqtt.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index e5565487..cdf1e8dd 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -111,9 +111,9 @@ func (c *TuyaMqttClient) Start(hubConfig *MQTTConfig, webrtcConfig *WebRTCConfig SetOnConnectHandler(c.onConnect). SetAutoReconnect(true). SetMaxReconnectInterval(30 * time.Second). - SetConnectTimeout(15 * time.Second). + SetConnectTimeout(30 * time.Second). SetKeepAlive(30 * time.Second). - SetPingTimeout(15 * time.Second) + SetPingTimeout(20 * time.Second) c.client = mqtt.NewClient(opts) From b58c1a7ed6c498367edbdb17847e9542c7eb3a24 Mon Sep 17 00:00:00 2001 From: seydx <34152761+seydx@users.noreply.github.com> Date: Thu, 12 Jun 2025 06:37:46 +0200 Subject: [PATCH 39/60] Update README.md Co-authored-by: Felipe Santos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 47482652..efafd260 100644 --- a/README.md +++ b/README.md @@ -572,7 +572,7 @@ Tested: KD110, KC200, KC401, KC420WS, EC71. **Tuya Smart API (recommended)**: - Cameras can be discovered through the go2rtc web interface via Tuya Smart account (Add > Tuya > Select region and fill in email and password > Login). -- Smart Life accounts are not supported, you need to create a Tuya Smart account. If the cameras are already added to the Smart Life app, you need to remove them and add them again to the Tuya Smart app. +- **Smart Life accounts are not supported**, you need to create a Tuya Smart account. If the cameras are already added to the Smart Life app, you need to remove them and add them again to the Tuya Smart app. **Tuya Cloud API**: - Requires setting up a cloud project in the Tuya Developer Platform. From 30c418542ceedb375bf61a9186cb5e3a750f7e23 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 16 Jun 2025 02:06:27 +0200 Subject: [PATCH 40/60] readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index efafd260..5446a091 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,7 @@ Supported sources: - [Hikvision ISAPI](#source-isapi) cameras - [Roborock vacuums](#source-roborock) models with cameras - [Exec](#source-exec) audio on server +- [Tuya](#source-tuya) cameras - [Any Browser](#incoming-browser) as IP-camera Two-way audio can be used in browser with [WebRTC](#module-webrtc) technology. The browser will give access to the microphone only for HTTPS sites ([read more](https://stackoverflow.com/questions/52759992/how-to-access-camera-and-microphone-in-chrome-without-https)). From da68101c097d3b696fd8461030f3b4f1d8ab6dae Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 16 Jun 2025 14:58:31 +0200 Subject: [PATCH 41/60] Optimize URL decoding and update MQTT keep-alive --- pkg/tuya/client.go | 8 ++++++-- pkg/tuya/mqtt.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 277848e7..c6a34d6e 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -6,6 +6,7 @@ import ( "fmt" "net/url" "regexp" + "strings" "sync" "github.com/AlexxIT/go2rtc/pkg/core" @@ -44,7 +45,8 @@ type RecvMessage struct { } func Dial(rawURL string) (core.Producer, error) { - u, err := url.Parse(rawURL) + escapedURL := strings.ReplaceAll(rawURL, "#", "%23") + u, err := url.Parse(escapedURL) if err != nil { return nil, err } @@ -277,8 +279,10 @@ func Dial(rawURL string) (core.Producer, error) { } client.connected.Done(nil) } - default: + case pion.PeerConnectionStateClosed: client.Close(errors.New("webrtc: " + msg.String())) + default: + // client.Close(errors.New("webrtc: " + msg.String())) } } }) diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index cdf1e8dd..0559e8bc 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -112,7 +112,7 @@ func (c *TuyaMqttClient) Start(hubConfig *MQTTConfig, webrtcConfig *WebRTCConfig SetAutoReconnect(true). SetMaxReconnectInterval(30 * time.Second). SetConnectTimeout(30 * time.Second). - SetKeepAlive(30 * time.Second). + SetKeepAlive(60 * time.Second). SetPingTimeout(20 * time.Second) c.client = mqtt.NewClient(opts) From c5dfa84ff2a593f786e8df9bb2a293990abdc9f4 Mon Sep 17 00:00:00 2001 From: seydx Date: Thu, 19 Jun 2025 10:29:27 +0200 Subject: [PATCH 42/60] readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5446a091..d40ff472 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,7 @@ Available source types: - [bubble](#source-bubble) - streaming from ESeeCloud/dvr163 NVR - [dvrip](#source-dvrip) - streaming from DVR-IP NVR - [tapo](#source-tapo) - TP-Link Tapo cameras with [two way audio](#two-way-audio) support +- [tuya](#source-tuya) - Tuya cameras with [two way audio](#two-way-audio) support - [kasa](#source-tapo) - TP-Link Kasa cameras - [gopro](#source-gopro) - GoPro cameras - [ivideon](#source-ivideon) - public cameras from [Ivideon](https://tv.ivideon.com/) service From e9611769bea1e821c0fc9eded3a0434114769e71 Mon Sep 17 00:00:00 2001 From: seydx Date: Thu, 10 Jul 2025 16:12:07 +0200 Subject: [PATCH 43/60] deps --- go.mod | 1 + go.sum | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/go.mod b/go.mod index 7abf1edd..af372221 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( require ( github.com/asticode/go-astikit v0.56.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/eclipse/paho.mqtt.golang v1.5.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/pion/datachannel v1.5.10 // indirect diff --git a/go.sum b/go.sum index 4dfebcf6..e14b77d9 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,7 @@ github.com/asticode/go-astits v1.13.0 h1:XOgkaadfZODnyZRR5Y0/DWkA9vrkLLPLeeOvDwf github.com/asticode/go-astits v1.13.0/go.mod h1:QSHmknZ51pf6KJdHKZHJTLlMegIrhega3LPWz3ND/iI= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= @@ -87,6 +88,7 @@ github.com/pion/webrtc/v4 v4.1.3 h1:YZ67Boj9X/hk190jJZ8+HFGQ6DqSZ/fYP3sLAZv7c3c= github.com/pion/webrtc/v4 v4.1.3/go.mod h1:rsq+zQ82ryfR9vbb0L1umPJ6Ogq7zm8mcn9fcGnxomM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= @@ -98,6 +100,8 @@ github.com/sigurn/crc16 v0.0.0-20240131213347-83fcde1e29d1 h1:NVK+OqnavpyFmUiKfU github.com/sigurn/crc16 v0.0.0-20240131213347-83fcde1e29d1/go.mod h1:9/etS5gpQq9BJsJMWg1wpLbfuSnkm8dPF6FdW2JXVhA= github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f h1:1R9KdKjCNSd7F8iGTxIpoID9prlYH8nuNYKt0XvweHA= github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f/go.mod h1:vQhwQ4meQEDfahT5kd61wLAF5AAeh5ZPLVI4JJ/tYo8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 h1:aeN+ghOV0b2VCmKKO3gqnDQ8mLbpABZgRR2FVYx4ouI= @@ -133,5 +137,6 @@ golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 863174839cd1135922c2c0a2c36fcb6d914cf524 Mon Sep 17 00:00:00 2001 From: seydx Date: Sun, 26 Oct 2025 16:39:59 +0100 Subject: [PATCH 44/60] Fix video/audio ssrc and low power cameras --- pkg/tuya/client.go | 18 ++++---- pkg/tuya/cloud_api.go | 7 +++ pkg/tuya/interface.go | 9 ++-- pkg/tuya/mqtt.go | 105 ++++++++++++++++++++++++++++++++++++------ pkg/tuya/smart_api.go | 7 +++ 5 files changed, 122 insertions(+), 24 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index c6a34d6e..d1a549a8 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -20,8 +20,8 @@ type Client struct { conn *webrtc.Conn pc *pion.PeerConnection dc *pion.DataChannel - videoSSRC uint32 - audioSSRC uint32 + videoSSRC *uint32 + audioSSRC *uint32 streamType int isHEVC bool connected core.Waiter @@ -374,16 +374,16 @@ func (c *Client) Start() error { } } - if c.videoSSRC != 0 { - c.setHandler(c.videoSSRC, func(packet *rtp.Packet) { + if c.videoSSRC != nil { + c.setHandler(*c.videoSSRC, func(packet *rtp.Packet) { if video != nil { video.WriteRTP(packet) } }) } - if c.audioSSRC != 0 { - c.setHandler(c.audioSSRC, func(packet *rtp.Packet) { + if c.audioSSRC != nil { + c.setHandler(*c.audioSSRC, func(packet *rtp.Packet) { if audio != nil { audio.WriteRTP(packet) } @@ -469,8 +469,10 @@ func (c *Client) probe(msg pion.DataChannelMessage) (bool, error) { return false, err } - c.videoSSRC = recvMessage.Video.SSRC - c.audioSSRC = recvMessage.Audio.SSRC + videoSSRC := recvMessage.Video.SSRC + audioSSRC := recvMessage.Audio.SSRC + c.videoSSRC = &videoSSRC + c.audioSSRC = &audioSSRC completeMsg, _ := json.Marshal(DataChannelMessage{ Type: "complete", diff --git a/pkg/tuya/cloud_api.go b/pkg/tuya/cloud_api.go index 4d25c2be..c34d0fe4 100644 --- a/pkg/tuya/cloud_api.go +++ b/pkg/tuya/cloud_api.go @@ -117,6 +117,10 @@ func (c *TuyaCloudApiClient) Init() error { return fmt.Errorf("failed to start MQTT: %w", err) } + if c.skill.LowPower > 0 { + _ = c.mqtt.WakeUp(c.localKey) + } + return nil } @@ -212,6 +216,9 @@ func (c *TuyaCloudApiClient) loadWebrtcConfig() (*WebRTCConfig, error) { return nil, err } + // Store LocalKey (not sure if cloud api provides this, but we need it for low power cameras) + c.localKey = webRTCConfigResponse.Result.LocalKey + iceServers, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) if err != nil { return nil, err diff --git a/pkg/tuya/interface.go b/pkg/tuya/interface.go index f4f530aa..74fcd585 100644 --- a/pkg/tuya/interface.go +++ b/pkg/tuya/interface.go @@ -31,6 +31,7 @@ type TuyaClient struct { baseUrl string expireTime int64 deviceId string + localKey string skill *Skill iceServers []pionWebrtc.ICEServer } @@ -74,15 +75,17 @@ type VideoSkill struct { } type Skill struct { - WebRTC int `json:"webrtc"` - Audios []AudioSkill `json:"audios"` - Videos []VideoSkill `json:"videos"` + WebRTC int `json:"webrtc"` + LowPower int `json:"lowPower,omitempty"` + Audios []AudioSkill `json:"audios"` + Videos []VideoSkill `json:"videos"` } type WebRTCConfig struct { AudioAttributes AudioAttributes `json:"audio_attributes"` Auth string `json:"auth"` ID string `json:"id"` + LocalKey string `json:"local_key,omitempty"` MotoID string `json:"moto_id"` P2PConfig P2PConfig `json:"p2p_config"` ProtocolVersion string `json:"protocol_version"` diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index 0559e8bc..79a30102 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -1,8 +1,11 @@ package tuya import ( + "encoding/hex" "encoding/json" + "errors" "fmt" + "hash/crc32" "strings" "time" @@ -13,6 +16,7 @@ import ( type TuyaMqttClient struct { client mqtt.Client waiter core.Waiter + wakeupWaiter core.Waiter publishTopic string subscribeTopic string auth string @@ -75,6 +79,15 @@ type DisconnectFrame struct { Mode string `json:"mode"` } +// {"protocol":4,"t":1761487814,"data":{"dps":{"152":"0","160":1,"170":false}}} +type DPSMessage struct { + Protocol int `json:"protocol"` + T int `json:"t"` + Data struct { + Dps map[string]interface{} `json:"dps"` + } `json:"data"` +} + type MqttMessage struct { Protocol int `json:"protocol"` Pv string `json:"pv"` @@ -84,9 +97,10 @@ type MqttMessage struct { func NewTuyaMqttClient(deviceId string) *TuyaMqttClient { return &TuyaMqttClient{ - deviceId: deviceId, - sessionId: core.RandString(6, 62), - waiter: core.Waiter{}, + deviceId: deviceId, + sessionId: core.RandString(6, 62), + waiter: core.Waiter{}, + wakeupWaiter: core.Waiter{}, } } @@ -129,26 +143,70 @@ func (c *TuyaMqttClient) Start(hubConfig *MQTTConfig, webrtcConfig *WebRTCConfig } func (c *TuyaMqttClient) Stop() { + c.closed = true + c.waiter.Done(errors.New("mqtt: stopped")) + c.wakeupWaiter.Done(errors.New("mqtt: stopped")) + if c.client != nil { _ = c.SendDisconnect() c.client.Disconnect(1000) } } -func (c *TuyaMqttClient) SendOffer(sdp string, streamResolution string, streamType int, isHEVC bool) error { - if isHEVC { - // On HEVC we use streamType 0 for main stream (hd) and 1 for sub stream (sd) - if streamResolution == "hd" { - streamType = 0 - } else { - streamType = 1 +func (c *TuyaMqttClient) WakeUp(localKey string) error { + // Calculate CRC32 of localKey + crc := crc32.ChecksumIEEE([]byte(localKey)) + + // Convert to hex string + hexStr := fmt.Sprintf("%08x", crc) + + // Convert hex string to byte array (2 chars at a time) + payload := make([]byte, len(hexStr)/2) + for i := 0; i < len(hexStr); i += 2 { + b, err := hex.DecodeString(hexStr[i : i+2]) + if err != nil { + return fmt.Errorf("failed to decode hex: %w", err) } + payload[i/2] = b[0] + } + + // Publish to wake-up topic: m/w/{deviceId} + wakeUpTopic := fmt.Sprintf("m/w/%s", c.deviceId) + token := c.client.Publish(wakeUpTopic, 1, false, payload) + if token.Wait() && token.Error() != nil { + return fmt.Errorf("failed to publish wake-up message: %w", token.Error()) + } + + // Subscribe to lowPower topic: smart/decrypt/in/{deviceId} + lowPowerTopic := fmt.Sprintf("smart/decrypt/in/%s", c.deviceId) + if token := c.client.Subscribe(lowPowerTopic, 1, c.onLowPowerMessage); token.Wait() && token.Error() != nil { + return fmt.Errorf("failed to subscribe to lowPower topic: %w", token.Error()) + } + + return nil +} + +func (c *TuyaMqttClient) SendOffer(sdp string, streamResolution string, streamType int, isHEVC bool) error { + // streamType comes from GetStreamType() and uses Skill StreamType values: + // - mainStream = 2 (HD) + // - substream = 4 (SD) + // + // But MQTT expects mapped stream_type values: + // - mainStream (2) → stream_type: 0 + // - substream (4) → stream_type: 1 + + mqttStreamType := streamType + switch streamType { + case 2: + mqttStreamType = 0 // mainStream (HD) + case 4: + mqttStreamType = 1 // substream (SD) } return c.sendMqttMessage("offer", 302, "", OfferFrame{ Mode: "webrtc", Sdp: sdp, - StreamType: streamType, + StreamType: mqttStreamType, Auth: c.auth, DatachannelEnable: isHEVC, }) @@ -189,7 +247,7 @@ func (c *TuyaMqttClient) SendDisconnect() error { } func (c *TuyaMqttClient) onConnect(client mqtt.Client) { - if token := client.Subscribe(c.subscribeTopic, 1, c.consume); token.Wait() && token.Error() != nil { + if token := client.Subscribe(c.subscribeTopic, 1, c.onMessage); token.Wait() && token.Error() != nil { c.waiter.Done(token.Error()) return } @@ -197,7 +255,7 @@ func (c *TuyaMqttClient) onConnect(client mqtt.Client) { c.waiter.Done(nil) } -func (c *TuyaMqttClient) consume(client mqtt.Client, msg mqtt.Message) { +func (c *TuyaMqttClient) onMessage(client mqtt.Client, msg mqtt.Message) { var rmqtt MqttMessage if err := json.Unmarshal(msg.Payload(), &rmqtt); err != nil { c.onError(err) @@ -218,6 +276,27 @@ func (c *TuyaMqttClient) consume(client mqtt.Client, msg mqtt.Message) { } } +func (c *TuyaMqttClient) onLowPowerMessage(client mqtt.Client, msg mqtt.Message) { + var message DPSMessage + if err := json.Unmarshal(msg.Payload(), &message); err != nil { + return + } + + // Check if protocol is 4 and dps[149] is true (motion detection = camera ready) + if message.Protocol == 4 { + if val, ok := message.Data.Dps["149"]; ok { + if ready, ok := val.(bool); ok && ready { + // Camera is now ready after wake-up (dps[149]:true received). + // However, we don't wait for this signal (like ismartlife.me doesn't either). + // The camera starts responding immediately after WakeUp() is called, + // so we proceed with the connection without blocking. + // This waiter is kept for potential future use or debugging. + c.wakeupWaiter.Done(nil) + } + } + } +} + func (c *TuyaMqttClient) onMqttAnswer(msg *MqttMessage) { var answerFrame AnswerFrame if err := json.Unmarshal(msg.Data.Message, &answerFrame); err != nil { diff --git a/pkg/tuya/smart_api.go b/pkg/tuya/smart_api.go index 5f4e74f9..3c96fe98 100644 --- a/pkg/tuya/smart_api.go +++ b/pkg/tuya/smart_api.go @@ -314,6 +314,10 @@ func (c *TuyaSmartApiClient) Init() error { return fmt.Errorf("failed to start MQTT: %w", err) } + if c.skill.LowPower > 0 { + _ = c.mqtt.WakeUp(c.localKey) + } + return nil } @@ -498,6 +502,9 @@ func (c *TuyaSmartApiClient) loadWebrtcConfig() (*WebRTCConfig, error) { return nil, err } + // Store LocalKey + c.localKey = webRTCConfigResponse.Result.LocalKey + iceServers, err := json.Marshal(&webRTCConfigResponse.Result.P2PConfig.Ices) if err != nil { return nil, err From fb8c6e1b1ba1958a941649132b36953514800d33 Mon Sep 17 00:00:00 2001 From: seydx Date: Sun, 26 Oct 2025 22:21:56 +0100 Subject: [PATCH 45/60] Update comments --- pkg/tuya/mqtt.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index 79a30102..cf36e09c 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -79,12 +79,18 @@ type DisconnectFrame struct { Mode string `json:"mode"` } -// {"protocol":4,"t":1761487814,"data":{"dps":{"152":"0","160":1,"170":false}}} -type DPSMessage struct { - Protocol int `json:"protocol"` - T int `json:"t"` +type MqttLowPowerMessage struct { + Protocol int `json:"protocol"` + T int `json:"t"` + S int `json:"s,omitempty"` + Type string `json:"type,omitempty"` Data struct { - Dps map[string]interface{} `json:"dps"` + DevID string `json:"devId,omitempty"` + Online bool `json:"online,omitempty"` + LastOnlineChangeTime int64 `json:"lastOnlineChangeTime,omitempty"` + GwID string `json:"gwId,omitempty"` + Cmd string `json:"cmd,omitempty"` + Dps map[string]interface{} `json:"dps,omitempty"` } `json:"data"` } @@ -277,12 +283,13 @@ func (c *TuyaMqttClient) onMessage(client mqtt.Client, msg mqtt.Message) { } func (c *TuyaMqttClient) onLowPowerMessage(client mqtt.Client, msg mqtt.Message) { - var message DPSMessage + var message MqttLowPowerMessage if err := json.Unmarshal(msg.Payload(), &message); err != nil { return } - // Check if protocol is 4 and dps[149] is true (motion detection = camera ready) + // Check if protocol is 4 and dps[149] is true + // https://developer.tuya.com/en/docs/iot-device-dev/doorbell_solution?id=Kayamyivh15ox#title-2-Battery if message.Protocol == 4 { if val, ok := message.Data.Dps["149"]; ok { if ready, ok := val.(bool); ok && ready { @@ -290,7 +297,7 @@ func (c *TuyaMqttClient) onLowPowerMessage(client mqtt.Client, msg mqtt.Message) // However, we don't wait for this signal (like ismartlife.me doesn't either). // The camera starts responding immediately after WakeUp() is called, // so we proceed with the connection without blocking. - // This waiter is kept for potential future use or debugging. + // This waiter is kept for potential future use. c.wakeupWaiter.Done(nil) } } From 721ed98afb07d043b40d71e0c4463214217884f4 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 27 Oct 2025 21:47:52 +0100 Subject: [PATCH 46/60] webrtc: export GetSenderTrack --- pkg/webrtc/conn.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/webrtc/conn.go b/pkg/webrtc/conn.go index 092b05c8..924fd550 100644 --- a/pkg/webrtc/conn.go +++ b/pkg/webrtc/conn.go @@ -161,16 +161,7 @@ func (c *Conn) AddCandidate(candidate string) error { return c.pc.AddICECandidate(webrtc.ICECandidateInit{Candidate: candidate}) } -func (c *Conn) getTranseiver(mid string) *webrtc.RTPTransceiver { - for _, tr := range c.pc.GetTransceivers() { - if tr.Mid() == mid { - return tr - } - } - return nil -} - -func (c *Conn) getSenderTrack(mid string) *Track { +func (c *Conn) GetSenderTrack(mid string) *Track { if tr := c.getTranseiver(mid); tr != nil { if s := tr.Sender(); s != nil { if t := s.Track().(*Track); t != nil { @@ -181,6 +172,15 @@ func (c *Conn) getSenderTrack(mid string) *Track { return nil } +func (c *Conn) getTranseiver(mid string) *webrtc.RTPTransceiver { + for _, tr := range c.pc.GetTransceivers() { + if tr.Mid() == mid { + return tr + } + } + return nil +} + func (c *Conn) getMediaCodec(remote *webrtc.TrackRemote) (*core.Media, *core.Codec) { for _, tr := range c.pc.GetTransceivers() { // search Transeiver for this TrackRemote From 8142d2fc431031bc851e6e6afc7fe2f08a8fc63f Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 27 Oct 2025 21:48:15 +0100 Subject: [PATCH 47/60] Refactor RepackG711 to use configurable packet size --- pkg/pcm/handlers.go | 15 +++++++++------ pkg/rtsp/consumer.go | 2 +- pkg/webrtc/consumer.go | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/pcm/handlers.go b/pkg/pcm/handlers.go index 18a96468..7eab1e15 100644 --- a/pkg/pcm/handlers.go +++ b/pkg/pcm/handlers.go @@ -12,8 +12,11 @@ import ( // 1. Fixes WebRTC audio quality issue (monotonic timestamp) // 2. Fixes Reolink Doorbell backchannel issue (zero timestamp) // https://github.com/AlexxIT/go2rtc/issues/331 -func RepackG711(zeroTS bool, handler core.HandlerFunc) core.HandlerFunc { - const PacketSize = 1024 +func RepackG711(zeroTS bool, size int, handler core.HandlerFunc) core.HandlerFunc { + packetSize := 1024 + if size > 0 { + packetSize = size + } var buf []byte var seq uint16 @@ -26,7 +29,7 @@ func RepackG711(zeroTS bool, handler core.HandlerFunc) core.HandlerFunc { mu.Lock() buf = append(buf, packet.Payload...) - if len(buf) < PacketSize { + if len(buf) < packetSize { mu.Unlock() return } @@ -39,7 +42,7 @@ func RepackG711(zeroTS bool, handler core.HandlerFunc) core.HandlerFunc { SequenceNumber: seq, SSRC: packet.SSRC, }, - Payload: buf[:PacketSize], + Payload: buf[:packetSize], } seq++ @@ -48,10 +51,10 @@ func RepackG711(zeroTS bool, handler core.HandlerFunc) core.HandlerFunc { // don't have this strange devices for tests if !zeroTS { pkt.Timestamp = ts - ts += PacketSize + ts += uint32(packetSize) } - buf = buf[PacketSize:] + buf = buf[packetSize:] mu.Unlock() diff --git a/pkg/rtsp/consumer.go b/pkg/rtsp/consumer.go index e6525d96..8fab94f3 100644 --- a/pkg/rtsp/consumer.go +++ b/pkg/rtsp/consumer.go @@ -56,7 +56,7 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv if c.mode == core.ModeActiveProducer && track.Codec.Name == core.CodecPCMA { // Fix Reolink Doorbell https://github.com/AlexxIT/go2rtc/issues/331 - sender.Handler = pcm.RepackG711(true, sender.Handler) + sender.Handler = pcm.RepackG711(true, 0, sender.Handler) } sender.HandleRTP(track) diff --git a/pkg/webrtc/consumer.go b/pkg/webrtc/consumer.go index ebc3a008..261ba10b 100644 --- a/pkg/webrtc/consumer.go +++ b/pkg/webrtc/consumer.go @@ -32,7 +32,7 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv panic(core.Caller()) } - localTrack := c.getSenderTrack(media.ID) + localTrack := c.GetSenderTrack(media.ID) if localTrack == nil { return errors.New("webrtc: can't get track") } @@ -66,7 +66,7 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv case core.CodecPCMA, core.CodecPCMU, core.CodecPCM, core.CodecPCML: // Fix audio quality https://github.com/AlexxIT/WebRTC/issues/500 // should be before ResampleToG711, because it will be called last - sender.Handler = pcm.RepackG711(false, sender.Handler) + sender.Handler = pcm.RepackG711(false, 0, sender.Handler) if codec.ClockRate == 0 { if codec.Name == core.CodecPCM || codec.Name == core.CodecPCML { From c6940eb0f3561d79afe0efc1b99e0f622f4680e2 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 27 Oct 2025 21:48:35 +0100 Subject: [PATCH 48/60] Refactor AddTrack to use GetSenderTrack and improve audio handling --- pkg/tuya/client.go | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index d1a549a8..6fb3222b 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -10,6 +10,7 @@ import ( "sync" "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/pcm" "github.com/AlexxIT/go2rtc/pkg/webrtc" "github.com/pion/rtp" pion "github.com/pion/webrtc/v4" @@ -333,9 +334,7 @@ func (c *Client) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, } func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiver) error { - // Manually handle backchannel, because repacking audio through go2rtc does not work - - localTrack := c.getSender() + localTrack := c.conn.GetSenderTrack(media.ID) if localTrack == nil { return errors.New("webrtc: can't get track") } @@ -350,10 +349,21 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece sender := core.NewSender(media, codec) sender.Handler = func(packet *rtp.Packet) { c.conn.Send += packet.MarshalSize() - //important to send with remote PayloadType _ = localTrack.WriteRTP(payloadType, packet) } + switch track.Codec.Name { + case core.CodecPCMA, core.CodecPCMU, core.CodecPCM, core.CodecPCML: + // https://developer.tuya.com/en/docs/iot-device-dev/tuyaos-package-ipc-device?id=Kcn1px33iptn2#title-29-Why%20can%E2%80%99t%20WebRTC%20play%20audio%3F + frameSize := 240 + if track.Codec.Name == core.CodecPCM { + frameSize = 560 + } + + sender.Handler = pcm.RepackG711(false, frameSize, sender.Handler) + sender.Handler = pcm.TranscodeHandler(codec, track.Codec, sender.Handler) + } + sender.HandleRTP(track) c.conn.Senders = append(c.conn.Senders, sender) @@ -498,18 +508,3 @@ func (c *Client) sendMessageToDataChannel(message []byte) error { return nil } - -func (c *Client) getSender() *webrtc.Track { - for _, tr := range c.pc.GetTransceivers() { - if tr.Kind() == pion.RTPCodecTypeAudio { - if tr.Direction() == pion.RTPTransceiverDirectionSendonly || tr.Direction() == pion.RTPTransceiverDirectionSendrecv { - if s := tr.Sender(); s != nil { - if t := s.Track().(*webrtc.Track); t != nil { - return t - } - } - } - } - } - return nil -} From 6d3d45e3375a9f46f4edaf2f015b3e76edbde4f4 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 27 Oct 2025 21:48:44 +0100 Subject: [PATCH 49/60] Handle speaker messages --- pkg/tuya/mqtt.go | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index cf36e09c..81eeaf11 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -17,6 +17,7 @@ type TuyaMqttClient struct { client mqtt.Client waiter core.Waiter wakeupWaiter core.Waiter + speakerWaiter core.Waiter publishTopic string subscribeTopic string auth string @@ -152,6 +153,7 @@ func (c *TuyaMqttClient) Stop() { c.closed = true c.waiter.Done(errors.New("mqtt: stopped")) c.wakeupWaiter.Done(errors.New("mqtt: stopped")) + c.speakerWaiter.Done(errors.New("mqtt: stopped")) if c.client != nil { _ = c.SendDisconnect() @@ -240,10 +242,18 @@ func (c *TuyaMqttClient) SendResolution(resolution int) error { func (c *TuyaMqttClient) SendSpeaker(speaker int) error { // Protocol 312 is used for speaker - return c.sendMqttMessage("speaker", 312, "", SpeakerFrame{ + if err := c.sendMqttMessage("speaker", 312, "", SpeakerFrame{ Mode: "webrtc", Value: speaker, - }) + }); err != nil { + return err + } + + // if err := c.speakerWaiter.Wait(); err != nil { + // return fmt.Errorf("speaker wait failed: %w", err) + // } + + return nil } func (c *TuyaMqttClient) SendDisconnect() error { @@ -279,6 +289,8 @@ func (c *TuyaMqttClient) onMessage(client mqtt.Client, msg mqtt.Message) { c.onMqttCandidate(&rmqtt) case "disconnect": c.onMqttDisconnect() + case "speaker": + c.onMqttSpeaker(&rmqtt) } } @@ -333,6 +345,21 @@ func (c *TuyaMqttClient) onMqttDisconnect() { c.onDisconnect() } +func (c *TuyaMqttClient) onMqttSpeaker(msg *MqttMessage) { + var speakerResponse struct { + ResCode int `json:"resCode"` + } + + if err := json.Unmarshal(msg.Data.Message, &speakerResponse); err == nil { + if speakerResponse.ResCode != 0 { + c.speakerWaiter.Done(fmt.Errorf("speaker failed with resCode: %d", speakerResponse.ResCode)) + return + } + } + + c.speakerWaiter.Done(nil) +} + func (c *TuyaMqttClient) onAnswer(answer AnswerFrame) { if c.handleAnswer != nil { c.handleAnswer(answer) From 0ff3bf67e1c1db0cf853c46942e28d92727895e5 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 27 Oct 2025 23:57:14 +0100 Subject: [PATCH 50/60] Comment out MQTT speaker message sending in AddTrack function --- pkg/tuya/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 6fb3222b..a7e43061 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -339,10 +339,10 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece return errors.New("webrtc: can't get track") } - mqttClient := c.api.GetMqtt() - if mqttClient != nil { - _ = mqttClient.SendSpeaker(1) - } + // mqttClient := c.api.GetMqtt() + // if mqttClient != nil { + // _ = mqttClient.SendSpeaker(1) + // } payloadType := codec.PayloadType From 0f27bb112404ae26edbc52bf740bcb0f7fb0abb5 Mon Sep 17 00:00:00 2001 From: seydx Date: Mon, 27 Oct 2025 23:57:54 +0100 Subject: [PATCH 51/60] Update SendOffer to include token and fix mqtt close --- pkg/tuya/mqtt.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index 81eeaf11..157c441a 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -21,6 +21,7 @@ type TuyaMqttClient struct { publishTopic string subscribeTopic string auth string + iceServers []ICEServer uid string motoId string deviceId string @@ -49,11 +50,12 @@ type MqttFrame struct { } type OfferFrame struct { - Mode string `json:"mode"` - Sdp string `json:"sdp"` - StreamType int `json:"stream_type"` - Auth string `json:"auth"` - DatachannelEnable bool `json:"datachannel_enable"` + Mode string `json:"mode"` + Sdp string `json:"sdp"` + StreamType int `json:"stream_type"` + Auth string `json:"auth"` + DatachannelEnable bool `json:"datachannel_enable"` + Token []ICEServer `json:"token"` } type AnswerFrame struct { @@ -115,6 +117,7 @@ func (c *TuyaMqttClient) Start(hubConfig *MQTTConfig, webrtcConfig *WebRTCConfig c.webrtcVersion = webrtcVersion c.motoId = webrtcConfig.MotoID c.auth = webrtcConfig.Auth + c.iceServers = webrtcConfig.P2PConfig.Ices c.publishTopic = hubConfig.PublishTopic c.subscribeTopic = hubConfig.SubscribeTopic @@ -150,15 +153,16 @@ func (c *TuyaMqttClient) Start(hubConfig *MQTTConfig, webrtcConfig *WebRTCConfig } func (c *TuyaMqttClient) Stop() { - c.closed = true c.waiter.Done(errors.New("mqtt: stopped")) c.wakeupWaiter.Done(errors.New("mqtt: stopped")) c.speakerWaiter.Done(errors.New("mqtt: stopped")) if c.client != nil { _ = c.SendDisconnect() - c.client.Disconnect(1000) + c.client.Disconnect(100) } + + c.closed = true } func (c *TuyaMqttClient) WakeUp(localKey string) error { @@ -217,6 +221,7 @@ func (c *TuyaMqttClient) SendOffer(sdp string, streamResolution string, streamTy StreamType: mqttStreamType, Auth: c.auth, DatachannelEnable: isHEVC, + Token: c.iceServers, }) } @@ -242,18 +247,16 @@ func (c *TuyaMqttClient) SendResolution(resolution int) error { func (c *TuyaMqttClient) SendSpeaker(speaker int) error { // Protocol 312 is used for speaker - if err := c.sendMqttMessage("speaker", 312, "", SpeakerFrame{ + return c.sendMqttMessage("speaker", 312, "", SpeakerFrame{ Mode: "webrtc", Value: speaker, - }); err != nil { - return err - } + }) // if err := c.speakerWaiter.Wait(); err != nil { // return fmt.Errorf("speaker wait failed: %w", err) // } - return nil + // return nil } func (c *TuyaMqttClient) SendDisconnect() error { From 25e7ac531eba942c0b5c6f88ceb3fdbe5d31813f Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 28 Oct 2025 11:12:44 +0100 Subject: [PATCH 52/60] Cleanup and update comments --- pkg/tuya/client.go | 39 +++++++++++++++++++++++++----------- pkg/tuya/interface.go | 26 +++++++++++++++++------- pkg/tuya/mqtt.go | 46 +++++++++++++++++++++++++------------------ 3 files changed, 73 insertions(+), 38 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index a7e43061..19b926d8 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -17,25 +17,28 @@ import ( ) type Client struct { - api TuyaAPI - conn *webrtc.Conn - pc *pion.PeerConnection + api TuyaAPI + conn *webrtc.Conn + pc *pion.PeerConnection + connected core.Waiter + closed bool + + // HEVC only: dc *pion.DataChannel videoSSRC *uint32 audioSSRC *uint32 streamType int isHEVC bool - connected core.Waiter - closed bool handlersMu sync.RWMutex handlers map[uint32]func(*rtp.Packet) } type DataChannelMessage struct { - Type string `json:"type"` + Type string `json:"type"` // "codec", "start", "recv", "complete" Msg string `json:"msg"` } +// RecvMessage contains SSRC values for video/audio streams type RecvMessage struct { Video struct { SSRC uint32 `json:"ssrc"` @@ -159,7 +162,8 @@ func Dial(rawURL string) (core.Producer, error) { } if client.isHEVC { - // Tuya seems to answers always with H264 and PCMU/8000 and PCMA/8000 codecs, replace with real codecs + // We need to replace the SDP codecs with the real ones from Skill. + // The actual media comes via DataChannel, not RTP tracks. for _, media := range client.conn.Medias { if media.Kind == core.KindVideo { @@ -202,9 +206,7 @@ func Dial(rawURL string) (core.Producer, error) { client.Close(err) } - // On HEVC, use DataChannel to receive video/audio if client.isHEVC { - // Create a new DataChannel maxRetransmits := uint16(5) ordered := true client.dc, err = client.pc.CreateDataChannel("fmp4Stream", &pion.DataChannelInit{ @@ -212,18 +214,22 @@ func Dial(rawURL string) (core.Producer, error) { Ordered: &ordered, }) - // Set up data channel handler + // DataChannel receives two types of messages: + // 1. String messages: Control messages (codec, recv) + // 2. Binary messages: RTP packets with video/audio client.dc.OnMessage(func(msg pion.DataChannelMessage) { if msg.IsString { + // Handle control messages (codec, recv, etc.) if connected, err := client.probe(msg); err != nil { client.Close(err) } else if connected { client.connected.Done(nil) } } else { + // Handle RTP packets - Route by SSRC retrieved from "recv" message packet := &rtp.Packet{} if err := packet.Unmarshal(msg.Data); err != nil { - // skip + // Skip invalid packets return } @@ -339,6 +345,9 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece return errors.New("webrtc: can't get track") } + // DISABLED: Speaker Protocol 312 command + // JavaScript client doesn't send this on first call either + // Only subsequent calls (when speakerChloron is set) send Protocol 312 // mqttClient := c.api.GetMqtt() // if mqttClient != nil { // _ = mqttClient.SendSpeaker(1) @@ -352,14 +361,16 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece _ = localTrack.WriteRTP(payloadType, packet) } + // Tuya cameras require specific frame sizes + // See: https://developer.tuya.com/en/docs/iot-device-dev/tuyaos-package-ipc-device?id=Kcn1px33iptn2#title-29 switch track.Codec.Name { case core.CodecPCMA, core.CodecPCMU, core.CodecPCM, core.CodecPCML: - // https://developer.tuya.com/en/docs/iot-device-dev/tuyaos-package-ipc-device?id=Kcn1px33iptn2#title-29-Why%20can%E2%80%99t%20WebRTC%20play%20audio%3F frameSize := 240 if track.Codec.Name == core.CodecPCM { frameSize = 560 } + // Repack to required frame size sender.Handler = pcm.RepackG711(false, frameSize, sender.Handler) sender.Handler = pcm.TranscodeHandler(codec, track.Codec, sender.Handler) } @@ -463,6 +474,7 @@ func (c *Client) probe(msg pion.DataChannelMessage) (bool, error) { switch message.Type { case "codec": + // Camera responded to our codec request - now request frame start frameRequest, _ := json.Marshal(DataChannelMessage{ Type: "start", Msg: "frame", @@ -474,6 +486,8 @@ func (c *Client) probe(msg pion.DataChannelMessage) (bool, error) { } case "recv": + // Camera sends SSRC values for video/audio streams + // We need these to route incoming RTP packets correctly var recvMessage RecvMessage if err := json.Unmarshal([]byte(message.Msg), &recvMessage); err != nil { return false, err @@ -484,6 +498,7 @@ func (c *Client) probe(msg pion.DataChannelMessage) (bool, error) { c.videoSSRC = &videoSSRC c.audioSSRC = &audioSSRC + // Send "complete" to tell camera we're ready to receive RTP packets completeMsg, _ := json.Marshal(DataChannelMessage{ Type: "complete", Msg: "", diff --git a/pkg/tuya/interface.go b/pkg/tuya/interface.go index 74fcd585..25ba0ddd 100644 --- a/pkg/tuya/interface.go +++ b/pkg/tuya/interface.go @@ -66,17 +66,17 @@ type AudioSkill struct { } type VideoSkill struct { - StreamType int `json:"streamType"` // 2 = main stream (hd), 4 = sub stream (sd) - ProfileId string `json:"profileId,omitempty"` - CodecType int `json:"codecType"` // 2 = H264, 4 = H265 + StreamType int `json:"streamType"` // 2 = main stream (HD), 4 = sub stream (SD) + CodecType int `json:"codecType"` // 2 = H264, 4 = H265 (HEVC) Width int `json:"width"` Height int `json:"height"` SampleRate int `json:"sampleRate"` + ProfileId string `json:"profileId,omitempty"` } type Skill struct { - WebRTC int `json:"webrtc"` - LowPower int `json:"lowPower,omitempty"` + WebRTC int `json:"webrtc"` // Bit flags: bit 4=speaker, bit 5=clarity, bit 6=record + LowPower int `json:"lowPower,omitempty"` // 1 = battery-powered camera Audios []AudioSkill `json:"audios"` Videos []VideoSkill `json:"videos"` } @@ -128,6 +128,14 @@ func (c *TuyaClient) GetMqtt() *TuyaMqttClient { return c.mqtt } +// GetStreamType returns the Skill StreamType for the requested resolution +// Returns Skill values (2 or 4), not MQTT values (0 or 1) +// - "hd" → highest resolution streamType (usually 2 = mainStream) +// - "sd" → lowest resolution streamType (usually 4 = substream) +// +// These values must be mapped before sending to MQTT: +// - streamType 2 → MQTT stream_type 0 +// - streamType 4 → MQTT stream_type 1 func (c *TuyaClient) GetStreamType(streamResolution string) int { // Default streamType if nothing is found defaultStreamType := 1 @@ -136,7 +144,7 @@ func (c *TuyaClient) GetStreamType(streamResolution string) int { return defaultStreamType } - // Find the highest and lowest resolution + // Find the highest and lowest resolution based on pixel count var highestResType = defaultStreamType var highestRes = 0 var lowestResType = defaultStreamType @@ -169,10 +177,14 @@ func (c *TuyaClient) GetStreamType(streamResolution string) int { } } +// IsHEVC checks if the given streamType uses H265 (HEVC) codec +// HEVC cameras use DataChannel, H264 cameras use RTP tracks +// - codecType 4 = H265 (HEVC) → DataChannel mode +// - codecType 2 = H264 → Normal RTP mode func (c *TuyaClient) IsHEVC(streamType int) bool { for _, video := range c.skill.Videos { if video.StreamType == streamType { - return video.CodecType == 4 + return video.CodecType == 4 // 4 = H265/HEVC } } diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index 157c441a..5f64ef48 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -52,9 +52,9 @@ type MqttFrame struct { type OfferFrame struct { Mode string `json:"mode"` Sdp string `json:"sdp"` - StreamType int `json:"stream_type"` + StreamType int `json:"stream_type"` // 0: mainStream(HD), 1: substream(SD) Auth string `json:"auth"` - DatachannelEnable bool `json:"datachannel_enable"` + DatachannelEnable bool `json:"datachannel_enable"` // true for HEVC, false for H264 Token []ICEServer `json:"token"` } @@ -165,8 +165,11 @@ func (c *TuyaMqttClient) Stop() { c.closed = true } +// WakeUp sends a wake-up signal to battery-powered cameras (LowPower mode). +// The camera wakes up and starts responding immediately - we don't wait for dps[149]. +// Note: LowPower cameras sleep after ~3 minutes of inactivity. func (c *TuyaMqttClient) WakeUp(localKey string) error { - // Calculate CRC32 of localKey + // Calculate CRC32 of localKey as wake-up payload crc := crc32.ChecksumIEEE([]byte(localKey)) // Convert to hex string @@ -189,7 +192,8 @@ func (c *TuyaMqttClient) WakeUp(localKey string) error { return fmt.Errorf("failed to publish wake-up message: %w", token.Error()) } - // Subscribe to lowPower topic: smart/decrypt/in/{deviceId} + // Subscribe to lowPower topic to receive dps[149] status updates + // (we don't wait for this signal - camera responds immediately) lowPowerTopic := fmt.Sprintf("smart/decrypt/in/%s", c.deviceId) if token := c.client.Subscribe(lowPowerTopic, 1, c.onLowPowerMessage); token.Wait() && token.Error() != nil { return fmt.Errorf("failed to subscribe to lowPower topic: %w", token.Error()) @@ -199,6 +203,7 @@ func (c *TuyaMqttClient) WakeUp(localKey string) error { } func (c *TuyaMqttClient) SendOffer(sdp string, streamResolution string, streamType int, isHEVC bool) error { + // Map Skill StreamType to MQTT stream_type values // streamType comes from GetStreamType() and uses Skill StreamType values: // - mainStream = 2 (HD) // - substream = 4 (SD) @@ -220,7 +225,7 @@ func (c *TuyaMqttClient) SendOffer(sdp string, streamResolution string, streamTy Sdp: sdp, StreamType: mqttStreamType, Auth: c.auth, - DatachannelEnable: isHEVC, + DatachannelEnable: isHEVC, // must be true for HEVC Token: c.iceServers, }) } @@ -233,30 +238,32 @@ func (c *TuyaMqttClient) SendCandidate(candidate string) error { } func (c *TuyaMqttClient) SendResolution(resolution int) error { - // isClaritySupperted := (c.webrtcVersion & (1 << 5)) != 0 - // if !isClaritySupperted { - // return nil - // } + // Check if camera supports clarity switching + isClaritySupported := (c.webrtcVersion & (1 << 5)) != 0 + if !isClaritySupported { + return nil + } - // Protocol 312 is used for clarity return c.sendMqttMessage("resolution", 312, "", ResolutionFrame{ Mode: "webrtc", - Value: resolution, + Value: resolution, // 0: HD, 1: SD }) } func (c *TuyaMqttClient) SendSpeaker(speaker int) error { - // Protocol 312 is used for speaker - return c.sendMqttMessage("speaker", 312, "", SpeakerFrame{ + if err := c.sendMqttMessage("speaker", 312, "", SpeakerFrame{ Mode: "webrtc", - Value: speaker, - }) + Value: speaker, // 0: off, 1: on + }); err != nil { + return err + } - // if err := c.speakerWaiter.Wait(); err != nil { - // return fmt.Errorf("speaker wait failed: %w", err) - // } + // Wait for camera response + if err := c.speakerWaiter.Wait(); err != nil { + return fmt.Errorf("speaker wait failed: %w", err) + } - // return nil + return nil } func (c *TuyaMqttClient) SendDisconnect() error { @@ -281,6 +288,7 @@ func (c *TuyaMqttClient) onMessage(client mqtt.Client, msg mqtt.Message) { return } + // Filter by session ID to prevent processing messages from other sessions if rmqtt.Data.Header.SessionID != c.sessionId { return } From 62a9046f019aeea3b69da7e60c0ca9c07f7bbff7 Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 28 Oct 2025 12:50:36 +0100 Subject: [PATCH 53/60] Revert configurable packet size for RepackG711 --- pkg/pcm/handlers.go | 15 ++++++--------- pkg/rtsp/consumer.go | 2 +- pkg/webrtc/consumer.go | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/pkg/pcm/handlers.go b/pkg/pcm/handlers.go index 7eab1e15..18a96468 100644 --- a/pkg/pcm/handlers.go +++ b/pkg/pcm/handlers.go @@ -12,11 +12,8 @@ import ( // 1. Fixes WebRTC audio quality issue (monotonic timestamp) // 2. Fixes Reolink Doorbell backchannel issue (zero timestamp) // https://github.com/AlexxIT/go2rtc/issues/331 -func RepackG711(zeroTS bool, size int, handler core.HandlerFunc) core.HandlerFunc { - packetSize := 1024 - if size > 0 { - packetSize = size - } +func RepackG711(zeroTS bool, handler core.HandlerFunc) core.HandlerFunc { + const PacketSize = 1024 var buf []byte var seq uint16 @@ -29,7 +26,7 @@ func RepackG711(zeroTS bool, size int, handler core.HandlerFunc) core.HandlerFun mu.Lock() buf = append(buf, packet.Payload...) - if len(buf) < packetSize { + if len(buf) < PacketSize { mu.Unlock() return } @@ -42,7 +39,7 @@ func RepackG711(zeroTS bool, size int, handler core.HandlerFunc) core.HandlerFun SequenceNumber: seq, SSRC: packet.SSRC, }, - Payload: buf[:packetSize], + Payload: buf[:PacketSize], } seq++ @@ -51,10 +48,10 @@ func RepackG711(zeroTS bool, size int, handler core.HandlerFunc) core.HandlerFun // don't have this strange devices for tests if !zeroTS { pkt.Timestamp = ts - ts += uint32(packetSize) + ts += PacketSize } - buf = buf[packetSize:] + buf = buf[PacketSize:] mu.Unlock() diff --git a/pkg/rtsp/consumer.go b/pkg/rtsp/consumer.go index 8fab94f3..e6525d96 100644 --- a/pkg/rtsp/consumer.go +++ b/pkg/rtsp/consumer.go @@ -56,7 +56,7 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv if c.mode == core.ModeActiveProducer && track.Codec.Name == core.CodecPCMA { // Fix Reolink Doorbell https://github.com/AlexxIT/go2rtc/issues/331 - sender.Handler = pcm.RepackG711(true, 0, sender.Handler) + sender.Handler = pcm.RepackG711(true, sender.Handler) } sender.HandleRTP(track) diff --git a/pkg/webrtc/consumer.go b/pkg/webrtc/consumer.go index 261ba10b..767394df 100644 --- a/pkg/webrtc/consumer.go +++ b/pkg/webrtc/consumer.go @@ -66,7 +66,7 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv case core.CodecPCMA, core.CodecPCMU, core.CodecPCM, core.CodecPCML: // Fix audio quality https://github.com/AlexxIT/WebRTC/issues/500 // should be before ResampleToG711, because it will be called last - sender.Handler = pcm.RepackG711(false, 0, sender.Handler) + sender.Handler = pcm.RepackG711(false, sender.Handler) if codec.ClockRate == 0 { if codec.Name == core.CodecPCM || codec.Name == core.CodecPCML { From 33e4527042c428aa59e5e0d05fbd71390684bf6e Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 28 Oct 2025 12:58:00 +0100 Subject: [PATCH 54/60] Revert repackaging for backchannel --- pkg/tuya/client.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 19b926d8..2ad74b84 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -10,7 +10,6 @@ import ( "sync" "github.com/AlexxIT/go2rtc/pkg/core" - "github.com/AlexxIT/go2rtc/pkg/pcm" "github.com/AlexxIT/go2rtc/pkg/webrtc" "github.com/pion/rtp" pion "github.com/pion/webrtc/v4" @@ -361,20 +360,6 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece _ = localTrack.WriteRTP(payloadType, packet) } - // Tuya cameras require specific frame sizes - // See: https://developer.tuya.com/en/docs/iot-device-dev/tuyaos-package-ipc-device?id=Kcn1px33iptn2#title-29 - switch track.Codec.Name { - case core.CodecPCMA, core.CodecPCMU, core.CodecPCM, core.CodecPCML: - frameSize := 240 - if track.Codec.Name == core.CodecPCM { - frameSize = 560 - } - - // Repack to required frame size - sender.Handler = pcm.RepackG711(false, frameSize, sender.Handler) - sender.Handler = pcm.TranscodeHandler(codec, track.Codec, sender.Handler) - } - sender.HandleRTP(track) c.conn.Senders = append(c.conn.Senders, sender) From 292b32af9963894dbe7014ba1a11c822794e98ad Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 28 Oct 2025 13:53:42 +0100 Subject: [PATCH 55/60] Optimize audio frame size handling in AddTrack to reduce latency for Tuya cameras --- pkg/tuya/client.go | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 2ad74b84..a010243b 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -355,9 +355,43 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece payloadType := codec.PayloadType sender := core.NewSender(media, codec) + + // Frame size affects audio delay with Tuya cameras: + // Browser sends standard 20ms frames (160 bytes for G.711), but this causes + // up to 4s delay on some Tuya cameras. Increasing to 240 bytes (30ms) reduces + // delay to ~2s. Higher values (320+ bytes) don't work and cause issues. + // Using 240 bytes (30ms) as optimal balance between latency and stability. + frameSize := 240 + + var buf []byte + var seq uint16 + var ts uint32 + sender.Handler = func(packet *rtp.Packet) { - c.conn.Send += packet.MarshalSize() - _ = localTrack.WriteRTP(payloadType, packet) + buf = append(buf, packet.Payload...) + + for len(buf) >= frameSize { + payload := buf[:frameSize] + + pkt := &rtp.Packet{ + Header: rtp.Header{ + Version: 2, + Marker: true, + PayloadType: payloadType, + SequenceNumber: seq, + Timestamp: ts, + SSRC: packet.SSRC, + }, + Payload: payload, + } + + seq++ + ts += uint32(frameSize) + buf = buf[frameSize:] + + c.conn.Send += pkt.MarshalSize() + _ = localTrack.WriteRTP(payloadType, pkt) + } } sender.HandleRTP(track) From 56d7a6fee4d1ef2494134c3820cb8bd5da01843b Mon Sep 17 00:00:00 2001 From: seydx Date: Tue, 28 Oct 2025 14:54:54 +0100 Subject: [PATCH 56/60] Add comments and improve repackaging --- pkg/tuya/client.go | 75 ++++++++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index a010243b..3043a8d2 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -161,9 +161,8 @@ func Dial(rawURL string) (core.Producer, error) { } if client.isHEVC { - // We need to replace the SDP codecs with the real ones from Skill. - // The actual media comes via DataChannel, not RTP tracks. - + // Tuya responds with H264/90000 even for HEVC streams + // So we need to replace video codecs with HEVC ones from API for _, media := range client.conn.Medias { if media.Kind == core.KindVideo { codecs := client.api.GetVideoCodecs() @@ -173,6 +172,9 @@ func Dial(rawURL string) (core.Producer, error) { } } + // Audio codecs from API as well + // Tuya responds with multiple audio codecs (PCMU, PCMA) + // But the quality is bad if we use PCMU and skill only has PCMA for _, media := range client.conn.Medias { if media.Kind == core.KindAudio { codecs := client.api.GetAudioCodecs() @@ -356,41 +358,50 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece sender := core.NewSender(media, codec) - // Frame size affects audio delay with Tuya cameras: - // Browser sends standard 20ms frames (160 bytes for G.711), but this causes - // up to 4s delay on some Tuya cameras. Increasing to 240 bytes (30ms) reduces - // delay to ~2s. Higher values (320+ bytes) don't work and cause issues. - // Using 240 bytes (30ms) as optimal balance between latency and stability. - frameSize := 240 + switch track.Codec.Name { + case core.CodecPCMA, core.CodecPCMU, core.CodecPCM, core.CodecPCML: + // Frame size affects audio delay with Tuya cameras: + // Browser sends standard 20ms frames (160 bytes for G.711), but this causes + // up to 4s delay on some Tuya cameras. Increasing to 240 bytes (30ms) reduces + // delay to ~2s. Higher values (320+ bytes) don't work and cause issues. + // Using 240 bytes (30ms) as optimal balance between latency and stability. + frameSize := 240 - var buf []byte - var seq uint16 - var ts uint32 + var buf []byte + var seq uint16 + var ts uint32 - sender.Handler = func(packet *rtp.Packet) { - buf = append(buf, packet.Payload...) + sender.Handler = func(packet *rtp.Packet) { + buf = append(buf, packet.Payload...) - for len(buf) >= frameSize { - payload := buf[:frameSize] + for len(buf) >= frameSize { + payload := buf[:frameSize] - pkt := &rtp.Packet{ - Header: rtp.Header{ - Version: 2, - Marker: true, - PayloadType: payloadType, - SequenceNumber: seq, - Timestamp: ts, - SSRC: packet.SSRC, - }, - Payload: payload, + pkt := &rtp.Packet{ + Header: rtp.Header{ + Version: 2, + Marker: true, + PayloadType: payloadType, + SequenceNumber: seq, + Timestamp: ts, + SSRC: packet.SSRC, + }, + Payload: payload, + } + + seq++ + ts += uint32(frameSize) + buf = buf[frameSize:] + + c.conn.Send += pkt.MarshalSize() + _ = localTrack.WriteRTP(payloadType, pkt) } + } - seq++ - ts += uint32(frameSize) - buf = buf[frameSize:] - - c.conn.Send += pkt.MarshalSize() - _ = localTrack.WriteRTP(payloadType, pkt) + default: + sender.Handler = func(packet *rtp.Packet) { + c.conn.Send += packet.MarshalSize() + _ = localTrack.WriteRTP(payloadType, packet) } } From 319dbf2c6362a35ca35ca46a9c4e3ba2ee09b86b Mon Sep 17 00:00:00 2001 From: seydx Date: Wed, 19 Nov 2025 00:10:13 +0100 Subject: [PATCH 57/60] Refactor after merge --- main.go | 2 ++ www/add.html | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/main.go b/main.go index 95e59ddd..a1c6aa88 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,7 @@ import ( "github.com/AlexxIT/go2rtc/internal/srtp" "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/internal/tapo" + "github.com/AlexxIT/go2rtc/internal/tuya" "github.com/AlexxIT/go2rtc/internal/v4l2" "github.com/AlexxIT/go2rtc/internal/webrtc" "github.com/AlexxIT/go2rtc/internal/webtorrent" @@ -95,6 +96,7 @@ func main() { {"ring", ring.Init}, {"roborock", roborock.Init}, {"tapo", tapo.Init}, + {"tuya", tuya.Init}, {"yandex", yandex.Init}, // Helper modules {"debug", debug.Init}, diff --git a/www/add.html b/www/add.html index 325df646..f8ef46b8 100644 --- a/www/add.html +++ b/www/add.html @@ -330,6 +330,53 @@ + +
+
+ + + + +
+
+
+ + +
From 94df080bf710198c75535ff265d4c24fb146610f Mon Sep 17 00:00:00 2001 From: Alex X Date: Tue, 25 Nov 2025 20:13:22 +0300 Subject: [PATCH 58/60] Fix MQTT url for tuya source for some regions --- pkg/tuya/smart_api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tuya/smart_api.go b/pkg/tuya/smart_api.go index 3c96fe98..09615db4 100644 --- a/pkg/tuya/smart_api.go +++ b/pkg/tuya/smart_api.go @@ -468,7 +468,7 @@ func (c *TuyaSmartApiClient) initToken() error { return errors.New(loginResp.ErrorMsg) } - c.mqttsUrl = fmt.Sprintf("wss://%s/mqtt", loginResp.Result.Domain.MobileMqttsUrl) + c.mqttsUrl = fmt.Sprintf("ssl://%s:%d", loginResp.Result.Domain.MobileMqttsUrl, loginResp.Result.Domain.MqttsPort) c.expireTime = time.Now().Unix() + 2*24*60*60 // 2 days in seconds return nil From 4ef6a147a6ecd334c2cac786fa5bafe2f9763c9b Mon Sep 17 00:00:00 2001 From: Alex X Date: Tue, 25 Nov 2025 20:13:45 +0300 Subject: [PATCH 59/60] Fix panic on login for tuya source --- internal/tuya/tuya.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/tuya/tuya.go b/internal/tuya/tuya.go index c44bdbbc..9dcf2721 100644 --- a/internal/tuya/tuya.go +++ b/internal/tuya/tuya.go @@ -196,12 +196,12 @@ func getLoginToken(client *http.Client, serverHost, username, countryCode string defer resp.Body.Close() var tokenResp tuya.LoginTokenResponse - if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { + if err = json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { return nil, err } if !tokenResp.Success { - return nil, err + return nil, errors.New("tuya: " + tokenResp.Msg) } return &tokenResp, nil From 4ec28490083d6bc09814f64d88aa2863323c05ea Mon Sep 17 00:00:00 2001 From: Alex X Date: Tue, 25 Nov 2025 20:14:00 +0300 Subject: [PATCH 60/60] Fix WebUI for tuya source --- www/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/main.js b/www/main.js index c901f300..d5629178 100644 --- a/www/main.js +++ b/www/main.js @@ -58,7 +58,7 @@ document.head.innerHTML += ` gap: 10px; } - input[type="text"] { + input[type="text"], input[type="email"], input[type="password"], select { padding: 10px; border: 1px solid #ccc; border-radius: 4px;