From 13c426e2a96ad7d287161f452cad04a7d62cf5fc Mon Sep 17 00:00:00 2001 From: Alexey Khit Date: Sat, 11 Mar 2023 20:52:56 +0300 Subject: [PATCH] Update WebRTC passive producer handling --- pkg/webrtc/conn.go | 20 +++++++++++----- pkg/webrtc/producer.go | 52 ++++++++++++++++++------------------------ pkg/webrtc/server.go | 25 ++++++++++++++++++++ www/webrtc.html | 14 ++++++------ 4 files changed, 68 insertions(+), 43 deletions(-) diff --git a/pkg/webrtc/conn.go b/pkg/webrtc/conn.go index 07866420..5b314e74 100644 --- a/pkg/webrtc/conn.go +++ b/pkg/webrtc/conn.go @@ -120,8 +120,12 @@ func (c *Conn) getRecvTrack(remote *webrtc.TrackRemote) *streamer.Track { payloadType := uint8(remote.PayloadType()) switch c.Mode { - // browser microphone (backchannel) case streamer.ModePassiveConsumer: + // Situation: + // - Browser (passive consumer) connects to go2rtc for receiving AV from IP-camera + // - Video and audio tracks marked as local "sendonly" + // - Browser sends microphone remote track to go2rtc, this track marked as local "recvonly" + // - go2rtc should ReadRTP from this remote track and sends it to camera for _, track := range c.tracks { if track.Direction == streamer.DirectionRecvonly && track.Codec.PayloadType == payloadType { return track @@ -129,7 +133,9 @@ func (c *Conn) getRecvTrack(remote *webrtc.TrackRemote) *streamer.Track { } case streamer.ModeActiveProducer: - // remote track from WebRTC active producer (audio/video) + // Situation: + // - go2rtc (active producer) connects to remote server (ex. webtorrent) for receiving AV + // - remote server sends remote tracks, this tracks marked as remote "sendonly" for _, track := range c.tracks { if track.Direction == streamer.DirectionSendonly && track.Codec.PayloadType == payloadType { return track @@ -137,7 +143,9 @@ func (c *Conn) getRecvTrack(remote *webrtc.TrackRemote) *streamer.Track { } case streamer.ModePassiveProducer: - // remote track from WebRTC passive producer (incoming WebRTC WHIP) + // Situation: + // - OBS Studio (passive producer) connects to go2rtc for send AV + // - OBS sends remote tracks, this tracks marked as remote "sendonly" for i, media := range c.medias { // check only tracks with same kind if media.Kind != remote.Kind().String() { @@ -159,9 +167,9 @@ func (c *Conn) getRecvTrack(remote *webrtc.TrackRemote) *streamer.Track { c.medias[i].Codecs = []*streamer.Codec{codec} } - track := streamer.NewTrack(media, codec) - c.tracks = append(c.tracks, track) - return track + // forward request to passive producer GetTrack + // will create NewTrack for sendonly media + return c.GetTrack(media, codec) } } diff --git a/pkg/webrtc/producer.go b/pkg/webrtc/producer.go index c9b7e486..164867a8 100644 --- a/pkg/webrtc/producer.go +++ b/pkg/webrtc/producer.go @@ -7,38 +7,25 @@ import ( ) func (c *Conn) GetTrack(media *streamer.Media, codec *streamer.Codec) *streamer.Track { - switch c.Mode { - case streamer.ModeActiveProducer: - // active producer (webrtc source, webtorrent source): - // - creates empty track for remote sendonly media - // - bind go2rtc with pion track for remote recv media (backchannel) - for _, track := range c.tracks { - if track.Codec == codec { - return track - } - } - - var track *streamer.Track - if media.Direction == streamer.DirectionSendonly { - track = streamer.NewTrack(media, codec) - } else { - track = c.getProducerSendTrack(media, codec) - } - - c.tracks = append(c.tracks, track) - return track - - case streamer.ModePassiveProducer: - // passive producer (WHIP) - for _, track := range c.tracks { - if track.Codec == codec { - return track - } - } - return nil + if c.Mode != streamer.ModeActiveProducer && c.Mode != streamer.ModePassiveProducer { + panic("not implemented") } - panic("not implemented") + for _, track := range c.tracks { + if track.Codec == codec { + return track + } + } + + var track *streamer.Track + if media.Direction == streamer.DirectionSendonly { + track = streamer.NewTrack(media, codec) + } else { + track = c.getProducerSendTrack(media, codec) + } + + c.tracks = append(c.tracks, track) + return track } func (c *Conn) Start() error { @@ -98,6 +85,7 @@ type Track struct { rid string streamID string payloadType byte + sequence uint16 ssrc uint32 writer webrtc.TrackLocalWriter } @@ -136,9 +124,13 @@ func (t *Track) Kind() webrtc.RTPCodecType { } func (t *Track) WriteRTP(packet *rtp.Packet) error { + // important to have internal counter if input packets from different sources + t.sequence++ + header := packet.Header header.SSRC = t.ssrc header.PayloadType = t.payloadType + header.SequenceNumber = t.sequence _, err := t.writer.WriteRTP(&header, packet.Payload) return err } diff --git a/pkg/webrtc/server.go b/pkg/webrtc/server.go index 7021f078..2fc1004d 100644 --- a/pkg/webrtc/server.go +++ b/pkg/webrtc/server.go @@ -33,6 +33,31 @@ func (c *Conn) SetOffer(offer string) (err error) { } func (c *Conn) GetAnswer() (answer string, err error) { + if c.Mode == streamer.ModePassiveProducer { + // init all Sender(s) for passive producer or they will be nil + // sender for passive producer is backchannel + sd := &sdp.SessionDescription{} + if err = sd.Unmarshal([]byte(c.offer)); err != nil { + return + } + + for _, md := range sd.MediaDescriptions { + for _, attr := range md.Attributes { + switch attr.Key { + case "recvonly": + _, _ = c.pc.AddTransceiverFromKind( + webrtc.NewRTPCodecType(md.MediaName.Media), + webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly}, + ) + case "sendrecv": + _, _ = c.pc.AddTransceiverFromKind( + webrtc.NewRTPCodecType(md.MediaName.Media), + ) + } + } + } + } + // we need to process remote offer after we create transeivers desc := webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: c.offer} if err = c.pc.SetRemoteDescription(desc); err != nil { diff --git a/www/webrtc.html b/www/webrtc.html index 5263a563..2eda9541 100644 --- a/www/webrtc.html +++ b/www/webrtc.html @@ -26,13 +26,6 @@ const localTracks = [] - if (/video|audio/.test(media)) { - const tracks = ['video', 'audio'] - .filter(kind => media.indexOf(kind) >= 0) - .map(kind => pc.addTransceiver(kind, {direction: 'recvonly'}).receiver.track) - localTracks.push(...tracks) - } - if (/camera|microphone/.test(media)) { const tracks = await getMediaTracks('user', { video: media.indexOf('camera') >= 0, @@ -55,6 +48,13 @@ }) } + if (/video|audio/.test(media)) { + const tracks = ['video', 'audio'] + .filter(kind => media.indexOf(kind) >= 0) + .map(kind => pc.addTransceiver(kind, {direction: 'recvonly'}).receiver.track) + localTracks.push(...tracks) + } + document.getElementById('video').srcObject = new MediaStream(localTracks) return pc