support h265

pkg/tuya/api.go

@@ -67,22 +67,26 @@ type P2PConfig struct {
 	Ices []OpenApiICE `json:"ices"`
 }
 
+type AudioSkill struct {
+	Channels   int `json:"channels"`
+	DataBit    int `json:"dataBit"`
+	CodecType  int `json:"codecType"`
+	SampleRate int `json:"sampleRate"`
+}
+
+type VideoSkill struct {
+	StreamType int    `json:"streamType"` // 2 = main stream, 4 = sub stream
+	ProfileId  string `json:"profileId,omitempty"`
+	CodecType  int    `json:"codecType"` // 2 = H264, 4 = H265
+	Width      int    `json:"width"`
+	Height     int    `json:"height"`
+	SampleRate int    `json:"sampleRate"`
+}
+
 type Skill struct {
-	WebRTC int `json:"webrtc"`
+	WebRTC int          `json:"webrtc"`
-	Audios []struct {
+	Audios []AudioSkill `json:"audios"`
-		Channels   int `json:"channels"`
+	Videos []VideoSkill `json:"videos"`
-		DataBit    int `json:"dataBit"`
-		CodecType  int `json:"codecType"`
-		SampleRate int `json:"sampleRate"`
-	} `json:"audios"`
-	Videos []struct {
-		StreamType int    `json:"streamType"` // 2 = main stream, 4 = sub stream
-		ProfileId  string `json:"profileId"`
-		Width      int    `json:"width"`
-		CodecType  int    `json:"codecType"` // 2 = H264, 4 = H265
-		SampleRate int    `json:"sampleRate"`
-		Height     int    `json:"height"`
-	} `json:"videos"`
 }
 
 type WebRTConfig struct {
@@ -299,20 +303,9 @@ func (c *TuyaClient) InitDevice() (err error) {
 	c.auth = webRTCConfigResponse.Result.Auth
 
 	c.skill = &Skill{
-		Audios: []struct {
+		WebRTC: 3, // basic webrtc
-			Channels   int `json:"channels"`
+		Audios: make([]AudioSkill, 0),
-			DataBit    int `json:"dataBit"`
+		Videos: make([]VideoSkill, 0),
-			CodecType  int `json:"codecType"`
-			SampleRate int `json:"sampleRate"`
-		}{},
-		Videos: []struct {
-			StreamType int    `json:"streamType"`
-			ProfileId  string `json:"profileId"`
-			Width      int    `json:"width"`
-			CodecType  int    `json:"codecType"`
-			SampleRate int    `json:"sampleRate"`
-			Height     int    `json:"height"`
-		}{},
 	}
 
 	if webRTCConfigResponse.Result.Skill != "" {
@@ -393,6 +386,11 @@ func (c *TuyaClient) InitDevice() (err error) {
 					ClockRate:   uint32(90000),
 					PayloadType: 96,
 				},
+				{
+					Name:        core.CodecH265,
+					ClockRate:   uint32(90000),
+					PayloadType: 96,
+				},
 			},
 		})
 	}
@@ -536,6 +534,20 @@ func getAudioCodec(codecType int) string {
 	}
 }
 
+func (c *TuyaClient) isHEVC(streamType int) bool {
+	for _, video := range c.skill.Videos {
+		if video.StreamType == streamType {
+			return video.CodecType == 4
+		}
+	}
+
+	return false
+}
+
+func (c *TuyaClient) isClaritySupported(webrtcValue int) bool {
+	return (webrtcValue & (1 << 5)) != 0
+}
+
 func (c *TuyaClient) calBusinessSign(ts int64) string {
 	data := fmt.Sprintf("%s%s%s%d", c.clientId, c.accessToken, c.clientSecret, ts)
 	val := md5.Sum([]byte(data))

pkg/tuya/client.go

@@ -14,9 +14,10 @@ import (
 )
 
 type Client struct {
-	api  *TuyaClient
+	api    *TuyaClient
-	conn *webrtc.Conn
+	conn   *webrtc.Conn
-	done chan struct{}
+	dcConn *DCConn
+	done   chan struct{}
 }
 
 const (
@@ -92,6 +93,8 @@ func Dial(rawURL string) (core.Producer, error) {
 		}
 		return streams.GetProducer(client.api.hlsURL)
 	} else {
+		isHEVC := client.api.isHEVC(client.api.getStreamType(streamType))
+
 		conf := pion.Configuration{
 			ICEServers:         client.api.iceServers,
 			ICETransportPolicy: pion.ICETransportPolicyAll,
@@ -116,7 +119,7 @@ func Dial(rawURL string) (core.Producer, error) {
 		// protect from blocking on errors
 		defer sendOffer.Done(nil)
 
-		// waiter will wait PC error or WS error or nil (connection OK)
+		// waiter will wait PC error
 		var connState core.Waiter
 
 		client.conn = webrtc.NewConn(pc)
@@ -167,6 +170,15 @@ func Dial(rawURL string) (core.Producer, error) {
 			client.Stop()
 		}
 
+		// Set up data channel for HEVC
+		if isHEVC {
+			client.dcConn, err = NewDCConn(pc, client)
+			if err != nil {
+				client.api.Close()
+				return nil, err
+			}
+		}
+
 		client.conn.Listen(func(msg any) {
 			switch msg := msg.(type) {
 			case *pion.ICECandidate:
@@ -182,6 +194,7 @@ func Dial(rawURL string) (core.Producer, error) {
 				case pion.PeerConnectionStateConnected:
 					connState.Done(nil)
 				default:
+					client.Stop()
 					connState.Done(errors.New("webrtc: " + msg.String()))
 				}
 			}
@@ -200,9 +213,18 @@ func Dial(rawURL string) (core.Producer, error) {
 		offer = re.ReplaceAllString(offer, "")
 
 		// Send offer
-		client.api.sendOffer(offer, tuyaAPI.getStreamType(streamType))
+		client.api.sendOffer(offer, streamType)
 		sendOffer.Done(nil)
 
+		if client.dcConn != nil {
+			if err = client.dcConn.connected.Wait(); err != nil {
+				client.Stop()
+				return nil, err
+			}
+
+			return client.dcConn, nil
+		}
+
 		if err = connState.Wait(); err != nil {
 			return nil, err
 		}
@@ -223,12 +245,7 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece
 	// RepackG711 will not work, so add default logic without repacking
 
 	payloadType := codec.PayloadType
-
 	localTrack := c.conn.GetSenderTrack(media.ID)
-	if localTrack == nil {
-		return errors.New("webrtc: can't get track")
-	}
-
 	sender := core.NewSender(media, codec)
 	sender.Handler = func(packet *rtp.Packet) {
 		c.conn.Send += packet.MarshalSize()
@@ -243,6 +260,10 @@ func (c *Client) AddTrack(media *core.Media, codec *core.Codec, track *core.Rece
 }
 
 func (c *Client) Start() error {
+	if c.dcConn != nil {
+		c.dcConn.Start()
+	}
+
 	return c.conn.Start()
 }
 
@@ -258,6 +279,10 @@ func (c *Client) Stop() error {
 		_ = c.conn.Stop()
 	}
 
+	if c.dcConn != nil {
+		_ = c.dcConn.Stop()
+	}
+
 	if c.api != nil {
 		c.api.Close()
 	}

pkg/tuya/dc.go

@@ -0,0 +1,253 @@
+package tuya
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/AlexxIT/go2rtc/pkg/core"
+	"github.com/AlexxIT/go2rtc/pkg/mp4"
+	"github.com/pion/rtp"
+	pion "github.com/pion/webrtc/v4"
+)
+
+type DCConn struct {
+	core.Connection
+
+	client      *Client
+	dc          *pion.DataChannel
+	dem         *mp4.Demuxer
+	queue       *FrameBufferQueue
+	msgs        chan pion.DataChannelMessage
+	connected   core.Waiter
+	closed      core.Waiter
+	initialized bool
+}
+
+type DataChannelMessage struct {
+	Type string `json:"type"`
+	Msg  string `json:"msg"`
+}
+
+func NewDCConn(pc *pion.PeerConnection, c *Client) (*DCConn, error) {
+	maxRetransmits := uint16(5)
+	ordered := true
+	dc, err := pc.CreateDataChannel("fmp4Stream", &pion.DataChannelInit{
+		MaxRetransmits: &maxRetransmits,
+		Ordered:        &ordered,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	conn := &DCConn{
+		Connection: core.Connection{
+			ID:         core.NewID(),
+			FormatName: "webrtc/fmp4",
+			Transport:  dc,
+		},
+		client:      c,
+		dc:          dc,
+		dem:         &mp4.Demuxer{},
+		queue:       NewFrameBufferQueue(),
+		msgs:        make(chan pion.DataChannelMessage, 10), // Saw max 4 messages in a row, 10 should be enough
+		initialized: false,
+	}
+
+	dc.OnMessage(func(msg pion.DataChannelMessage) {
+		conn.msgs <- msg
+	})
+
+	dc.OnError(func(err error) {
+		conn.connected.Done(err)
+	})
+
+	dc.OnClose(func() {
+		close(conn.msgs)
+		conn.connected.Done(errors.New("datachannel: closed"))
+	})
+
+	go conn.initializationLoop()
+
+	return conn, nil
+}
+
+func (c *DCConn) initializationLoop() {
+	for msg := range c.msgs {
+		if c.initialized {
+			return
+		}
+
+		err := c.probe(msg)
+		if err != nil {
+			c.connected.Done(err)
+			return
+		}
+
+		if c.initialized {
+			c.connected.Done(nil)
+			return
+		}
+	}
+}
+
+func (c *DCConn) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, error) {
+	if media.Direction == core.DirectionSendRecv || media.Direction == core.DirectionSendonly {
+		return c.client.GetTrack(media, codec)
+	}
+
+	for _, receiver := range c.Receivers {
+		if receiver.Codec == codec {
+			return receiver, nil
+		}
+	}
+	receiver := core.NewReceiver(media, codec)
+	c.Receivers = append(c.Receivers, receiver)
+	return receiver, nil
+}
+
+func (c *DCConn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiver) error {
+	payloadType := codec.PayloadType
+	localTrack := c.client.conn.GetSenderTrack(media.ID)
+	sender := core.NewSender(media, codec)
+	sender.Handler = func(packet *rtp.Packet) {
+		c.Send += packet.MarshalSize()
+		//important to send with remote PayloadType
+		_ = localTrack.WriteRTP(payloadType, packet)
+	}
+
+	sender.HandleRTP(track)
+	c.Senders = append(c.Senders, sender)
+
+	return nil
+}
+
+func (c *DCConn) Start() error {
+	receivers := make(map[uint32]*core.Receiver)
+	for _, receiver := range c.Receivers {
+		trackID := c.dem.GetTrackID(receiver.Codec)
+		receivers[trackID] = receiver
+	}
+
+	ch := make(chan []byte, 10)
+	defer close(ch)
+
+	go func() {
+		for data := range ch {
+			allTracks := c.dem.DemuxAll(data)
+			for _, trackData := range allTracks {
+				trackID := trackData.TrackID
+				packets := trackData.Packets
+				receiver := receivers[trackID]
+				if receiver == nil {
+					continue
+				}
+
+				for _, packet := range packets {
+					receiver.WriteRTP(packet)
+				}
+			}
+		}
+	}()
+
+	go func() {
+		for msg := range c.msgs {
+			if len(msg.Data) >= 4 {
+				segmentNum := int(msg.Data[1])
+				fragmentCount := int(msg.Data[2])
+				fragmentSeq := int(msg.Data[3])
+				mp4Data := msg.Data[4:]
+
+				c.queue.AddFragment(segmentNum, fragmentCount, fragmentSeq, mp4Data)
+
+				if c.queue.IsSegmentComplete(segmentNum, fragmentCount) {
+					b := c.queue.GetCombinedBuffer(segmentNum)
+					c.Recv += len(b)
+					ch <- b
+				}
+			}
+		}
+	}()
+
+	c.closed.Wait()
+	return nil
+}
+
+func (c *DCConn) sendMessageToDataChannel(message string) error {
+	if c.dc != nil {
+		return c.dc.SendText(message)
+	}
+
+	return nil
+}
+
+func (c *DCConn) probe(msg pion.DataChannelMessage) (err error) {
+	if msg.IsString {
+		var message DataChannelMessage
+		if err = json.Unmarshal(msg.Data, &message); err != nil {
+			return err
+		}
+
+		switch message.Type {
+		case "codec":
+			response, _ := json.Marshal(DataChannelMessage{
+				Type: "start",
+				Msg:  "fmp4",
+			})
+
+			err = c.sendMessageToDataChannel(string(response))
+			if err != nil {
+				return err
+			}
+
+		case "recv":
+			response, _ := json.Marshal(DataChannelMessage{
+				Type: "complete",
+				Msg:  "",
+			})
+
+			err = c.sendMessageToDataChannel(string(response))
+			if err != nil {
+				return err
+			}
+		}
+
+	} else {
+		if len(msg.Data) >= 4 {
+			messageType := msg.Data[0]
+			segmentNum := int(msg.Data[1])
+			fragmentCount := int(msg.Data[2])
+			fragmentSeq := int(msg.Data[3])
+			mp4Data := msg.Data[4:]
+
+			// initialization segment
+			if messageType == 0 && segmentNum == 1 && fragmentCount == 1 && fragmentSeq == 1 {
+				medias := c.dem.Probe(mp4Data)
+				c.Medias = append(c.Medias, medias...)
+
+				// Add backchannel
+				webrtcMedias := c.client.GetMedias()
+				for _, media := range webrtcMedias {
+					if media.Kind == core.KindAudio {
+						if media.Direction == core.DirectionSendRecv || media.Direction == core.DirectionSendonly {
+							c.Medias = append(c.Medias, media)
+						}
+					}
+				}
+
+				c.initialized = true
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *DCConn) Stop() error {
+	if c.dc != nil && c.dc.ReadyState() == pion.DataChannelStateOpen {
+		_ = c.dc.Close()
+	}
+
+	c.closed.Done(nil)
+	return nil
+}

pkg/tuya/frameBuffer.go

@@ -0,0 +1,86 @@
+package tuya
+
+import (
+	"sort"
+	"sync"
+)
+
+type FrameBufferQueue struct {
+	segments map[int]map[int][]byte // segNum -> fragSeq -> data
+	mu       sync.Mutex
+}
+
+func NewFrameBufferQueue() *FrameBufferQueue {
+	return &FrameBufferQueue{
+		segments: make(map[int]map[int][]byte),
+	}
+}
+
+func (q *FrameBufferQueue) AddFragment(segmentNum, fragmentCount, fragmentSeq int, data []byte) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	if _, ok := q.segments[segmentNum]; !ok {
+		q.segments[segmentNum] = make(map[int][]byte)
+	}
+
+	q.segments[segmentNum][fragmentSeq] = data
+}
+
+func (q *FrameBufferQueue) IsSegmentComplete(segmentNum, fragmentCount int) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	if frags, ok := q.segments[segmentNum]; ok {
+		// Make sure we have the right number of fragments
+		if len(frags) != fragmentCount {
+			return false
+		}
+
+		// Check if we have all sequences from 1 to fragmentCount
+		for i := 1; i <= fragmentCount; i++ {
+			if _, ok := frags[i]; !ok {
+				return false
+			}
+		}
+
+		return true
+	}
+
+	return false
+}
+
+func (q *FrameBufferQueue) GetCombinedBuffer(segNum int) []byte {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	if frags, ok := q.segments[segNum]; ok {
+		// Sort fragments by sequence number
+		var keys []int
+		for k := range frags {
+			keys = append(keys, k)
+		}
+		sort.Ints(keys)
+
+		// Calculate total size for pre-allocation
+		totalSize := 0
+		for _, k := range keys {
+			totalSize += len(frags[k])
+		}
+
+		// Pre-allocate buffer for better performance
+		combined := make([]byte, 0, totalSize)
+
+		// Combine fragments in sequence order
+		for _, k := range keys {
+			combined = append(combined, frags[k]...)
+		}
+
+		// Remove this segment to free memory
+		delete(q.segments, segNum)
+
+		return combined
+	}
+
+	return nil
+}

pkg/tuya/mqtt.go

@@ -208,68 +208,25 @@ func (c *TuyaMQTT) onError(err error) {
 	}
 }
 
-func (c *TuyaClient) sendOffer(sdp string, streamType int) {
+func (c *TuyaClient) sendOffer(sdp string, streamType string) {
-	// Note:
+	fixedStreamType := c.getStreamType(streamType)
-	// H265 is currently not supported because Tuya does not send H265 data, and therefore also no audio over the normal WebRTC connection.
+	isHEVC := c.isHEVC(fixedStreamType)
-	// The WebRTC connection is used only for sending audio back to the device (backchannel).
-	// Tuya expects a separate WebRTC DataChannel for H265 data and sends the H265 video and audio data packaged as fMP4 data back.
-	// These must then be processed separately (WIP - Work In Progress)
 
-	// Note 2:
+	if isHEVC {
-	// Even if we don't receive any data, the peer connection is correctly established and connected
+		// On HEVC we use streamType 0 for main stream and 1 for sub stream
-
+		if streamType == "main" {
-	// Note 3:
+			fixedStreamType = 0
-	// It seems that if even one stream is HEVC, we also need to use the datachannel for the substream, even if that substream is using H264.
+		} else {
-
+			fixedStreamType = 1
-	// Example Answer (H265/PCMU with backchannel):
+		}
-
+	}
-	/*
-	   v=0
-	   o=- 1747174385 1 IN IP4 127.0.0.1
-	   s=-
-	   t=0 0
-	   a=group:BUNDLE 0 1
-	   a=msid-semantic: WMS UMSklk
-	   m=audio 9 UDP/TLS/RTP/SAVPF 0
-	   c=IN IP4 0.0.0.0
-	   a=rtcp:9 IN IP4 0.0.0.0
-	   a=ice-ufrag:zuRr
-	   a=ice-pwd:EDeWXz847P810fyDyKxbmTdX
-	   a=ice-options:trickle
-	   a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14
-	   a=setup:active
-	   a=mid:0
-	   a=sendrecv
-	   a=msid:UMSklk NiNNboEn1rJWoQYtpguoKr1GBwpvPST
-	   a=rtcp-mux
-	   a=rtpmap:0 PCMU/8000
-	   a=ssrc:832759612 cname:bfa87264438073154dhdek
-	   m=video 9 UDP/TLS/RTP/SAVPF 0
-	   c=IN IP4 0.0.0.0
-	   a=rtcp:9 IN IP4 0.0.0.0
-	   a=ice-ufrag:zuRr
-	   a=ice-pwd:EDeWXz847P810fyDyKxbmTdX
-	   a=ice-options:trickle
-	   a=fingerprint:sha-256 02:f5:44:8e:c6:5d:5c:59:49:50:a3:84:d5:e5:b9:35:bb:51:5a:0c:4d:a5:60:89:0f:e6:cb:0e:57:21:a0:14
-	   a=setup:active
-	   a=mid:1
-	   a=sendonly
-	   a=msid:UMSklk l9o6icIVb7n7vDdp0KhocYnsijhd774
-	   a=rtcp-mux
-	   a=rtpmap:0 /0
-	   a=rtcp-fb:0 ccm fir
-	   a=rtcp-fb:0 nack
-	   a=rtcp-fb:0 nack pli
-	   a=fmtp:0 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=
-	   a=ssrc:0 cname:bfa87264438073154dhdek
-	*/
 
 	c.sendMqttMessage("offer", 302, "", OfferFrame{
 		Mode:              "webrtc",
 		Sdp:               sdp,
-		StreamType:        streamType,
+		StreamType:        fixedStreamType,
 		Auth:              c.auth,
-		DatachannelEnable: c.isHEVC(streamType),
+		DatachannelEnable: isHEVC,
 	})
 }
 
@@ -344,17 +301,3 @@ func (c *TuyaClient) sendMqttMessage(messageType string, protocol int, transacti
 		c.mqtt.onError(fmt.Errorf("mqtt publish fail: %s, topic: %s", token.Error().Error(), c.mqtt.publishTopic))
 	}
 }
-
-func (c *TuyaClient) isHEVC(streamType int) bool {
-	for _, video := range c.skill.Videos {
-		if video.StreamType == streamType {
-			return video.CodecType == 4
-		}
-	}
-
-	return false
-}
-
-func (c *TuyaClient) isClaritySupported(webrtcValue int) bool {
-	return (webrtcValue & (1 << 5)) != 0
-}