diff --git a/pkg/tuya/api.go b/pkg/tuya/api.go index 2f7cd194..13e8265e 100644 --- a/pkg/tuya/api.go +++ b/pkg/tuya/api.go @@ -76,10 +76,10 @@ type Skill struct { SampleRate int `json:"sampleRate"` } `json:"audios"` Videos []struct { - StreamType int `json:"streamType"` // streamType = 2 => main stream - streamType = 4 => sub stream + StreamType int `json:"streamType"` // 2 = main stream, 4 = sub stream ProfileId string `json:"profileId"` Width int `json:"width"` - CodecType int `json:"codecType"` + CodecType int `json:"codecType"` // 2 = H264, 4 = H265 SampleRate int `json:"sampleRate"` Height int `json:"height"` } `json:"videos"` @@ -325,24 +325,24 @@ func (c *TuyaClient) InitDevice() (err error) { c.medias = make([]*core.Media, 0) if len(c.skill.Audios) > 0 { - // Use the first Audio-Codec - audio := c.skill.Audios[0] - direction := core.DirectionRecvonly if c.hasBackchannel { direction = core.DirectionSendRecv } + codecs := make([]*core.Codec, 0) + for _, audio := range c.skill.Audios { + codecs = append(codecs, &core.Codec{ + Name: getAudioCodec(audio.CodecType), + ClockRate: uint32(audio.SampleRate), + Channels: uint8(audio.Channels), + }) + } + c.medias = append(c.medias, &core.Media{ Kind: core.KindAudio, Direction: direction, - Codecs: []*core.Codec{ - { - Name: "PCMU", - ClockRate: uint32(audio.SampleRate), - Channels: uint8(audio.Channels), - }, - }, + Codecs: codecs, }) } else { // Use default values for Audio @@ -351,7 +351,7 @@ func (c *TuyaClient) InitDevice() (err error) { Direction: core.DirectionRecvonly, Codecs: []*core.Codec{ { - Name: "PCMU", + Name: core.CodecPCMU, ClockRate: uint32(8000), Channels: uint8(1), }, @@ -360,24 +360,27 @@ func (c *TuyaClient) InitDevice() (err error) { } if len(c.skill.Videos) > 0 { - // Use the first Video-Codec - video := c.skill.Videos[0] + codecs := make([]*core.Codec, 0) + for _, video := range c.skill.Videos { + if video.CodecType == 2 { + codecs = append(codecs, &core.Codec{ + Name: core.CodecH264, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }) + } else if video.CodecType == 4 { + codecs = append(codecs, &core.Codec{ + Name: core.CodecH265, + ClockRate: uint32(video.SampleRate), + PayloadType: 96, + }) + } + } c.medias = append(c.medias, &core.Media{ Kind: core.KindVideo, Direction: core.DirectionRecvonly, - Codecs: []*core.Codec{ - { - Name: core.CodecH265, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, - }, - { - Name: core.CodecH264, - ClockRate: uint32(video.SampleRate), - PayloadType: 96, - }, - }, + Codecs: codecs, }) } else { // Use default values for Video @@ -469,19 +472,19 @@ func (c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) { return &openIoTHubConfigResponse.Result, nil } -func (c *TuyaClient) getStreamType(streamChoice string) uint32 { +func (c *TuyaClient) getStreamType(streamChoice string) int { // Default streamType if nothing is found - defaultStreamType := uint32(1) + defaultStreamType := 1 if c.skill == nil || len(c.skill.Videos) == 0 { return defaultStreamType } // Find the highest and lowest resolution - var highestResType uint32 = defaultStreamType - var highestRes int = 0 - var lowestResType uint32 = defaultStreamType - var lowestRes int = 0 + var highestResType = defaultStreamType + var highestRes = 0 + var lowestResType = defaultStreamType + var lowestRes = 0 for _, video := range c.skill.Videos { res := video.Width * video.Height @@ -489,13 +492,13 @@ func (c *TuyaClient) getStreamType(streamChoice string) uint32 { // Highest Resolution if res > highestRes { highestRes = res - highestResType = uint32(video.StreamType) + highestResType = video.StreamType } // Lower Resolution (or first if not set yet) if lowestRes == 0 || res < lowestRes { lowestRes = res - lowestResType = uint32(video.StreamType) + lowestResType = video.StreamType } } @@ -510,6 +513,29 @@ func (c *TuyaClient) getStreamType(streamChoice string) uint32 { } } +func getAudioCodec(codecType int) string { + switch codecType { + // case 100: + // return "ADPCM" + case 101: + return core.CodecPCM + case 102, 103, 104: + return core.CodecAAC + case 105: + return core.CodecPCMU + case 106: + return core.CodecPCMA + // case 107: + // return "G726-32" + // case 108: + // return "SPEEX" + case 109: + return core.CodecMP3 + default: + return core.CodecPCMU + } +} + func (c *TuyaClient) calBusinessSign(ts int64) string { data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts) val := md5.Sum([]byte(data)) diff --git a/pkg/tuya/client.go b/pkg/tuya/client.go index 8a0cc221..3fb7bf07 100644 --- a/pkg/tuya/client.go +++ b/pkg/tuya/client.go @@ -95,7 +95,7 @@ func Dial(rawURL string) (core.Producer, error) { conf := pion.Configuration{ ICEServers: client.api.iceServers, ICETransportPolicy: pion.ICETransportPolicyAll, - BundlePolicy: pion.BundlePolicyBalanced, + BundlePolicy: pion.BundlePolicyMaxBundle, } api, err := webrtc.NewAPI() @@ -148,6 +148,8 @@ func Dial(rawURL string) (core.Producer, error) { } client.api.mqtt.handleCandidate = func(candidate CandidateFrame) { + // fmt.Printf("tuya: candidate: %s\n", candidate.Candidate) + if candidate.Candidate != "" { client.conn.AddCandidate(candidate.Candidate) if err != nil { diff --git a/pkg/tuya/mqtt.go b/pkg/tuya/mqtt.go index bf9badd7..eb1c719e 100644 --- a/pkg/tuya/mqtt.go +++ b/pkg/tuya/mqtt.go @@ -39,10 +39,11 @@ type MqttFrame struct { } type OfferFrame struct { - Mode string `json:"mode"` - Sdp string `json:"sdp"` - StreamType uint32 `json:"stream_type"` - Auth string `json:"auth"` + Mode string `json:"mode"` + Sdp string `json:"sdp"` + StreamType int `json:"stream_type"` + Auth string `json:"auth"` + DatachannelEnable bool `json:"datachannel_enable"` } type AnswerFrame struct { @@ -57,7 +58,12 @@ type CandidateFrame struct { type ResolutionFrame struct { Mode string `json:"mode"` - Value int `json:"value"` + Value int `json:"value"` // 0: HD, 1: SD +} + +type SpeakerFrame struct { + Mode string `json:"mode"` + Value int `json:"value"` // 0: off, 1: on } type DisconnectFrame struct { @@ -202,12 +208,61 @@ func (c *TuyaMQTT) onError(err error) { } } -func (c *TuyaClient) sendOffer(sdp string, streamType uint32) { +func (c *TuyaClient) sendOffer(sdp string, streamType int) { + // H265 is currently not supported because Tuya does not send H265 data, and therefore also no audio over the normal WebRTC connection. + // The WebRTC connection is used only for sending audio back to the device (backchannel). + // Tuya expects a separate WebRTC DataChannel for H265 data and sends the H265 video and audio data packaged as fMP4 data back. + // These must then be processed separately (WIP - Work In Progress) + + // Example Answer (H265/PCMU with backchannel): + + /* + v=0 + o=- 1747174385 1 IN IP4 127.0.0.1 + s=- + t=0 0 + a=group:BUNDLE 0 1 + a=msid-semantic: WMS UMSklk + m=audio 9 UDP/TLS/RTP/SAVPF 0 + c=IN IP4 0.0.0.0 + a=rtcp:9 IN IP4 0.0.0.0 + a=ice-ufrag:zuRr + a=ice-pwd:EDeWXz847P810fyDyKxbmTdX + a=ice-options:trickle + a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14 + a=setup:active + a=mid:0 + a=sendrecv + a=msid:UMSklk NiNNboEn1rJWoQYtpguoKr1GBwpvPST + a=rtcp-mux + a=rtpmap:0 PCMU/8000 + a=ssrc:832759612 cname:bfa87264438073154dhdek + m=video 9 UDP/TLS/RTP/SAVPF 0 + c=IN IP4 0.0.0.0 + a=rtcp:9 IN IP4 0.0.0.0 + a=ice-ufrag:zuRr + a=ice-pwd:EDeWXz847P810fyDyKxbmTdX + a=ice-options:trickle + a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14 + a=setup:active + a=mid:1 + a=sendonly + a=msid:UMSklk l9o6icIVb7n7vDdp0KhocYnsijhd774 + a=rtcp-mux + a=rtpmap:0 /0 + a=rtcp-fb:0 ccm fir + a=rtcp-fb:0 nack + a=rtcp-fb:0 nack pli + a=fmtp:0 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id= + a=ssrc:0 cname:bfa87264438073154dhdek + */ + c.sendMqttMessage("offer", 302, "", OfferFrame{ - Mode: "webrtc", - Sdp: sdp, - StreamType: streamType, - Auth: c.auth, + Mode: "webrtc", + Sdp: sdp, + StreamType: streamType, + Auth: c.auth, + DatachannelEnable: c.isHEVC(streamType), }) } @@ -219,12 +274,23 @@ func (c *TuyaClient) sendCandidate(candidate string) { } func (c *TuyaClient) sendResolution(resolution int) { + if !c.isClaritySupported(resolution) { + return + } + c.sendMqttMessage("resolution", 302, "", ResolutionFrame{ Mode: "webrtc", Value: resolution, }) } +func (c *TuyaClient) sendSpeaker(speaker int) { + c.sendMqttMessage("speaker", 302, "", SpeakerFrame{ + Mode: "webrtc", + Value: speaker, + }) +} + func (c *TuyaClient) sendDisconnect() { c.sendMqttMessage("disconnect", 302, "", DisconnectFrame{ Mode: "webrtc", @@ -271,3 +337,17 @@ func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transacti c.mqtt.onError(fmt.Errorf("mqtt publish fail: %s, topic: %s", token.Error().Error(), c.mqtt.publishTopic)) } } + +func (c *TuyaClient) isHEVC(streamType int) bool { + for _, video := range c.skill.Videos { + if video.StreamType == streamType { + return video.CodecType == 4 + } + } + + return false +} + +func (c *TuyaClient) isClaritySupported(webrtcValue int) bool { + return (webrtcValue & (1 << 5)) != 0 +}