gilles/go2rtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip h265 datachannel

Browse Source
This commit is contained in:
seydx
2025-05-13 22:25:35 +02:00
parent 5ec942cb5e
commit e7bd3d401f
3 changed files with 154 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tuya/api.go
+61 -35
View File
@@ -76,10 +76,10 @@ type Skill struct {
SampleRate int `json:"sampleRate"`
} `json:"audios"`
Videos []struct {
StreamType int `json:"streamType"` // streamType = 2 => main stream - streamType = 4 => sub stream
StreamType int `json:"streamType"` // 2 = main stream, 4 = sub stream
ProfileId string `json:"profileId"`
Width int `json:"width"`
CodecType int `json:"codecType"`
CodecType int `json:"codecType"` // 2 = H264, 4 = H265
SampleRate int `json:"sampleRate"`
Height int `json:"height"`
} `json:"videos"`
@@ -325,24 +325,24 @@ func (c *TuyaClient) InitDevice() (err error) {
c.medias = make([]*core.Media, 0)
if len(c.skill.Audios) > 0 {
// Use the first Audio-Codec
audio := c.skill.Audios[0]
direction := core.DirectionRecvonly
if c.hasBackchannel {
direction = core.DirectionSendRecv
}
codecs := make([]*core.Codec, 0)
for _, audio := range c.skill.Audios {
codecs = append(codecs, &core.Codec{
Name: getAudioCodec(audio.CodecType),
ClockRate: uint32(audio.SampleRate),
Channels: uint8(audio.Channels),
})
}
c.medias = append(c.medias, &core.Media{
Kind: core.KindAudio,
Direction: direction,
Codecs: []*core.Codec{
{
Name: "PCMU",
ClockRate: uint32(audio.SampleRate),
Channels: uint8(audio.Channels),
},
},
Codecs: codecs,
})
} else {
// Use default values for Audio
@@ -351,7 +351,7 @@ func (c *TuyaClient) InitDevice() (err error) {
Direction: core.DirectionRecvonly,
Codecs: []*core.Codec{
{
Name: "PCMU",
Name: core.CodecPCMU,
ClockRate: uint32(8000),
Channels: uint8(1),
},
@@ -360,24 +360,27 @@ func (c *TuyaClient) InitDevice() (err error) {
}
if len(c.skill.Videos) > 0 {
// Use the first Video-Codec
video := c.skill.Videos[0]
codecs := make([]*core.Codec, 0)
for _, video := range c.skill.Videos {
if video.CodecType == 2 {
codecs = append(codecs, &core.Codec{
Name: core.CodecH264,
ClockRate: uint32(video.SampleRate),
PayloadType: 96,
})
} else if video.CodecType == 4 {
codecs = append(codecs, &core.Codec{
Name: core.CodecH265,
ClockRate: uint32(video.SampleRate),
PayloadType: 96,
})
}
}
c.medias = append(c.medias, &core.Media{
Kind: core.KindVideo,
Direction: core.DirectionRecvonly,
Codecs: []*core.Codec{
{
Name: core.CodecH265,
ClockRate: uint32(video.SampleRate),
PayloadType: 96,
},
{
Name: core.CodecH264,
ClockRate: uint32(video.SampleRate),
PayloadType: 96,
},
},
Codecs: codecs,
})
} else {
// Use default values for Video
@@ -469,19 +472,19 @@ func (c *TuyaClient) LoadHubConfig() (config *OpenIoTHubConfig, err error) {
return &openIoTHubConfigResponse.Result, nil
}
func (c *TuyaClient) getStreamType(streamChoice string) uint32 {
func (c *TuyaClient) getStreamType(streamChoice string) int {
// Default streamType if nothing is found
defaultStreamType := uint32(1)
defaultStreamType := 1
if c.skill == nil || len(c.skill.Videos) == 0 {
return defaultStreamType
}
// Find the highest and lowest resolution
var highestResType uint32 = defaultStreamType
var highestRes int = 0
var lowestResType uint32 = defaultStreamType
var lowestRes int = 0
var highestResType = defaultStreamType
var highestRes = 0
var lowestResType = defaultStreamType
var lowestRes = 0
for _, video := range c.skill.Videos {
res := video.Width * video.Height
@@ -489,13 +492,13 @@ func (c *TuyaClient) getStreamType(streamChoice string) uint32 {
// Highest Resolution
if res > highestRes {
highestRes = res
highestResType = uint32(video.StreamType)
highestResType = video.StreamType
}
// Lower Resolution (or first if not set yet)
if lowestRes == 0 || res < lowestRes {
lowestRes = res
lowestResType = uint32(video.StreamType)
lowestResType = video.StreamType
}
}
@@ -510,6 +513,29 @@ func (c *TuyaClient) getStreamType(streamChoice string) uint32 {
}
}
func getAudioCodec(codecType int) string {
switch codecType {
// case 100:
// return "ADPCM"
case 101:
return core.CodecPCM
case 102, 103, 104:
return core.CodecAAC
case 105:
return core.CodecPCMU
case 106:
return core.CodecPCMA
// case 107:
// return "G726-32"
// case 108:
// return "SPEEX"
case 109:
return core.CodecMP3
default:
return core.CodecPCMU
}
}
func (c *TuyaClient) calBusinessSign(ts int64) string {
data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts)
val := md5.Sum([]byte(data))
pkg/tuya/client.go
+3 -1
View File
@@ -95,7 +95,7 @@ func Dial(rawURL string) (core.Producer, error) {
conf := pion.Configuration{
ICEServers: client.api.iceServers,
ICETransportPolicy: pion.ICETransportPolicyAll,
BundlePolicy: pion.BundlePolicyBalanced,
BundlePolicy: pion.BundlePolicyMaxBundle,
}
api, err := webrtc.NewAPI()
@@ -148,6 +148,8 @@ func Dial(rawURL string) (core.Producer, error) {
}
client.api.mqtt.handleCandidate = func(candidate CandidateFrame) {
// fmt.Printf("tuya: candidate: %s\n", candidate.Candidate)
if candidate.Candidate != "" {
client.conn.AddCandidate(candidate.Candidate)
if err != nil {
pkg/tuya/mqtt.go
+90 -10
View File
@@ -39,10 +39,11 @@ type MqttFrame struct {
}
type OfferFrame struct {
Mode string `json:"mode"`
Sdp string `json:"sdp"`
StreamType uint32 `json:"stream_type"`
Auth string `json:"auth"`
Mode string `json:"mode"`
Sdp string `json:"sdp"`
StreamType int `json:"stream_type"`
Auth string `json:"auth"`
DatachannelEnable bool `json:"datachannel_enable"`
}
type AnswerFrame struct {
@@ -57,7 +58,12 @@ type CandidateFrame struct {
type ResolutionFrame struct {
Mode string `json:"mode"`
Value int `json:"value"`
Value int `json:"value"` // 0: HD, 1: SD
}
type SpeakerFrame struct {
Mode string `json:"mode"`
Value int `json:"value"` // 0: off, 1: on
}
type DisconnectFrame struct {
@@ -202,12 +208,61 @@ func (c *TuyaMQTT) onError(err error) {
}
}
func (c *TuyaClient) sendOffer(sdp string, streamType uint32) {
func (c *TuyaClient) sendOffer(sdp string, streamType int) {
// H265 is currently not supported because Tuya does not send H265 data, and therefore also no audio over the normal WebRTC connection.
// The WebRTC connection is used only for sending audio back to the device (backchannel).
// Tuya expects a separate WebRTC DataChannel for H265 data and sends the H265 video and audio data packaged as fMP4 data back.
// These must then be processed separately (WIP - Work In Progress)
// Example Answer (H265/PCMU with backchannel):
/*
v=0
o=- 1747174385 1 IN IP4 127.0.0.1
s=-
t=0 0
a=group:BUNDLE 0 1
a=msid-semantic: WMS UMSklk
m=audio 9 UDP/TLS/RTP/SAVPF 0
c=IN IP4 0.0.0.0
a=rtcp:9 IN IP4 0.0.0.0
a=ice-ufrag:zuRr
a=ice-pwd:EDeWXz847P810fyDyKxbmTdX
a=ice-options:trickle
a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14
a=setup:active
a=mid:0
a=sendrecv
a=msid:UMSklk NiNNboEn1rJWoQYtpguoKr1GBwpvPST
a=rtcp-mux
a=rtpmap:0 PCMU/8000
a=ssrc:832759612 cname:bfa87264438073154dhdek
m=video 9 UDP/TLS/RTP/SAVPF 0
c=IN IP4 0.0.0.0
a=rtcp:9 IN IP4 0.0.0.0
a=ice-ufrag:zuRr
a=ice-pwd:EDeWXz847P810fyDyKxbmTdX
a=ice-options:trickle
a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14
a=setup:active
a=mid:1
a=sendonly
a=msid:UMSklk l9o6icIVb7n7vDdp0KhocYnsijhd774
a=rtcp-mux
a=rtpmap:0 /0
a=rtcp-fb:0 ccm fir
a=rtcp-fb:0 nack
a=rtcp-fb:0 nack pli
a=fmtp:0 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=
a=ssrc:0 cname:bfa87264438073154dhdek
*/
c.sendMqttMessage("offer", 302, "", OfferFrame{
Mode: "webrtc",
Sdp: sdp,
StreamType: streamType,
Auth: c.auth,
Mode: "webrtc",
Sdp: sdp,
StreamType: streamType,
Auth: c.auth,
DatachannelEnable: c.isHEVC(streamType),
})
}
@@ -219,12 +274,23 @@ func (c *TuyaClient) sendCandidate(candidate string) {
}
func (c *TuyaClient) sendResolution(resolution int) {
if !c.isClaritySupported(resolution) {
return
}
c.sendMqttMessage("resolution", 302, "", ResolutionFrame{
Mode: "webrtc",
Value: resolution,
})
}
func (c *TuyaClient) sendSpeaker(speaker int) {
c.sendMqttMessage("speaker", 302, "", SpeakerFrame{
Mode: "webrtc",
Value: speaker,
})
}
func (c *TuyaClient) sendDisconnect() {
c.sendMqttMessage("disconnect", 302, "", DisconnectFrame{
Mode: "webrtc",
@@ -271,3 +337,17 @@ func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transacti
c.mqtt.onError(fmt.Errorf("mqtt publish fail: %s, topic: %s", token.Error().Error(), c.mqtt.publishTopic))
}
}
func (c *TuyaClient) isHEVC(streamType int) bool {
for _, video := range c.skill.Videos {
if video.StreamType == streamType {
return video.CodecType == 4
}
}
return false
}
func (c *TuyaClient) isClaritySupported(webrtcValue int) bool {
return (webrtcValue & (1 << 5)) != 0
}