gilles/go2rtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BIG core logic rewrite

Browse Source
This commit is contained in:
Alexey Khit
2023-03-17 06:48:02 +03:00
parent 2146ea470b
commit 12a7b96289
107 changed files with 3000 additions and 3024 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/webrtc/api.go
+4 -2
View File
@@ -8,6 +8,9 @@ import (
"strings"
)
// ReceiveMTU = Ethernet MTU (1500) - IP Header (20) - UDP Header (8)
const ReceiveMTU = 1472
func NewAPI(address string) (*webrtc.API, error) {
// for debug logs add to env: `PION_LOG_DEBUG=all`
m := &webrtc.MediaEngine{}
@@ -41,8 +44,7 @@ func NewAPI(address string) (*webrtc.API, error) {
// fix https://github.com/pion/webrtc/pull/2407
s.SetDTLSInsecureSkipHelloVerify(true)
// Ethernet MTU (1500) - IP Header (20) - UDP Header (8)
s.SetReceiveMTU(1472)
s.SetReceiveMTU(ReceiveMTU)
if address != "" {
address, network, _ := strings.Cut(address, "/")
pkg/webrtc/client.go
+7 -21
View File
@@ -1,27 +1,27 @@
package webrtc
import (
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/pion/sdp/v3"
"github.com/pion/webrtc/v3"
)
func (c *Conn) CreateOffer(medias []*streamer.Media) (string, error) {
func (c *Conn) CreateOffer(medias []*core.Media) (string, error) {
// 1. Create transeivers with proper kind and direction
for _, media := range medias {
var err error
switch media.Direction {
case streamer.DirectionRecvonly:
case core.DirectionRecvonly:
_, err = c.pc.AddTransceiverFromKind(
webrtc.NewRTPCodecType(media.Kind),
webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionRecvonly},
)
case streamer.DirectionSendonly:
case core.DirectionSendonly:
_, err = c.pc.AddTransceiverFromTrack(
NewTrack(media.Kind),
webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly},
)
case streamer.DirectionSendRecv:
case core.DirectionSendRecv:
// default transceiver is sendrecv
_, err = c.pc.AddTransceiverFromTrack(NewTrack(media.Kind))
}
@@ -45,7 +45,7 @@ func (c *Conn) CreateOffer(medias []*streamer.Media) (string, error) {
return c.pc.LocalDescription().SDP, nil
}
func (c *Conn) CreateCompleteOffer(medias []*streamer.Media) (string, error) {
func (c *Conn) CreateCompleteOffer(medias []*core.Media) (string, error) {
if _, err := c.CreateOffer(medias); err != nil {
return "", err
}
@@ -68,21 +68,7 @@ func (c *Conn) SetAnswer(answer string) (err error) {
return
}
medias := streamer.UnmarshalMedias(sd.MediaDescriptions)
// sort medias, so video will always be before audio
// and ignore application media from Hass default lovelace card
// ignore media without direction (inactive media)
for _, media := range medias {
if media.Kind == streamer.KindVideo && media.Direction != "" {
c.medias = append(c.medias, media)
}
}
for _, media := range medias {
if media.Kind == streamer.KindAudio && media.Direction != "" {
c.medias = append(c.medias, media)
}
}
c.medias = UnmarshalMedias(sd.MediaDescriptions)
return nil
}
pkg/webrtc/client_test.go
+5 -5
View File
@@ -1,7 +1,7 @@
package webrtc
import (
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/pion/webrtc/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -17,10 +17,10 @@ func TestClient(t *testing.T) {
prod := NewConn(pc)
medias := []*streamer.Media{
{Kind: streamer.KindVideo, Direction: streamer.DirectionRecvonly},
{Kind: streamer.KindAudio, Direction: streamer.DirectionRecvonly},
{Kind: streamer.KindAudio, Direction: streamer.DirectionSendonly},
medias := []*core.Media{
{Kind: core.KindVideo, Direction: core.DirectionRecvonly},
{Kind: core.KindAudio, Direction: core.DirectionRecvonly},
{Kind: core.KindAudio, Direction: core.DirectionSendonly},
}
offer, err := prod.CreateOffer(medias)
pkg/webrtc/conn.go
+52 -108
View File
@@ -2,9 +2,6 @@ package webrtc
import (
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/AlexxIT/go2rtc/pkg/h265"
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/pion/rtcp"
"github.com/pion/rtp"
"github.com/pion/webrtc/v3"
@@ -12,19 +9,20 @@ import (
)
type Conn struct {
streamer.Element
core.Listener
UserAgent string
Desc string
Mode streamer.Mode
Mode core.Mode
pc *webrtc.PeerConnection
medias []*streamer.Media
tracks []*streamer.Track
medias []*core.Media
receivers []*core.Receiver
senders []*core.Sender
receive int
send int
recv int
send int
offer string
remote string
@@ -56,13 +54,26 @@ func NewConn(pc *webrtc.PeerConnection) *Conn {
)
})
pc.OnTrack(func(remote *webrtc.TrackRemote, _ *webrtc.RTPReceiver) {
track := c.getRecvTrack(remote)
if track == nil {
return // it's OK when we not need, for example, audio from producer
pc.OnTrack(func(remote *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
media, codec := c.getMediaCodec(remote)
if media == nil {
return
}
if c.Mode == streamer.ModePassiveProducer && remote.Kind() == webrtc.RTPCodecTypeVideo {
track, err := c.GetTrack(media, codec)
if err != nil {
return
}
switch c.Mode {
case core.ModePassiveProducer, core.ModeActiveProducer:
// replace the theoretical list of codecs with the actual list of codecs
if len(media.Codecs) > 1 {
media.Codecs = []*core.Codec{codec}
}
}
if c.Mode == core.ModePassiveProducer && remote.Kind() == webrtc.RTPCodecTypeVideo {
go func() {
pkts := []rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(remote.SSRC())}}
for range time.NewTicker(time.Second * 2).C {
@@ -74,15 +85,20 @@ func NewConn(pc *webrtc.PeerConnection) *Conn {
}
for {
packet, _, err := remote.ReadRTP()
b := make([]byte, ReceiveMTU)
n, _, err := remote.Read(b)
if err != nil {
return
}
if len(packet.Payload) == 0 {
continue
c.recv += n
packet := &rtp.Packet{}
if err := packet.Unmarshal(b[:n]); err != nil {
return
}
c.receive += len(packet.Payload)
_ = track.WriteRTP(packet)
track.WriteRTP(packet)
}
})
@@ -127,106 +143,34 @@ func (c *Conn) getTranseiver(mid string) *webrtc.RTPTransceiver {
}
return nil
}
func (c *Conn) addSendTrack(media *streamer.Media, track *streamer.Track) *streamer.Track {
tr := c.getTranseiver(media.MID)
sender := tr.Sender()
localTrack := sender.Track().(*Track)
codec := track.Codec
// important to get remote PayloadType
payloadType := media.MatchCodec(codec).PayloadType
push := func(packet *rtp.Packet) error {
c.send += packet.MarshalSize()
return localTrack.WriteRTP(payloadType, packet)
}
switch codec.Name {
case streamer.CodecH264:
wrapper := h264.RTPPay(1200)
push = wrapper(push)
if codec.IsRTP() {
wrapper = h264.RTPDepay(track)
} else {
wrapper = h264.RepairAVC(track)
}
push = wrapper(push)
case streamer.CodecH265:
// SafariPay because it is the only browser in the world
// that supports WebRTC + H265
wrapper := h265.SafariPay(1200)
push = wrapper(push)
wrapper = h265.RTPDepay(track)
push = wrapper(push)
}
return track.Bind(push)
}
func (c *Conn) getRecvTrack(remote *webrtc.TrackRemote) *streamer.Track {
payloadType := uint8(remote.PayloadType())
switch c.Mode {
case streamer.ModePassiveConsumer:
// Situation:
// - Browser (passive consumer) connects to go2rtc for receiving AV from IP-camera
// - Video and audio tracks marked as local "sendonly"
// - Browser sends microphone remote track to go2rtc, this track marked as local "recvonly"
// - go2rtc should ReadRTP from this remote track and sends it to camera
for _, track := range c.tracks {
if track.Direction == streamer.DirectionRecvonly && track.Codec.PayloadType == payloadType {
return track
}
func (c *Conn) getMediaCodec(remote *webrtc.TrackRemote) (*core.Media, *core.Codec) {
for _, tr := range c.pc.GetTransceivers() {
// search Transeiver for this TrackRemote
if tr.Receiver() == nil || tr.Receiver().Track() != remote {
continue
}
case streamer.ModeActiveProducer:
// Situation:
// - go2rtc (active producer) connects to remote server (ex. webtorrent) for receiving AV
// - remote server sends remote tracks, this tracks marked as remote "sendonly"
for _, track := range c.tracks {
if track.Direction == streamer.DirectionSendonly && track.Codec.PayloadType == payloadType {
return track
}
}
case streamer.ModePassiveProducer:
// Situation:
// - OBS Studio (passive producer) connects to go2rtc for send AV
// - OBS sends remote tracks, this tracks marked as remote "sendonly"
for i, media := range c.medias {
// check only tracks with same kind
if media.Kind != remote.Kind().String() {
continue
}
// check only incoming tracks (remote media "sendonly")
if media.Direction != streamer.DirectionSendonly {
// search Media for this MID
for _, media := range c.medias {
if media.ID != tr.Mid() || media.Direction != core.DirectionRecvonly {
continue
}
// search codec for this PayloadType
for _, codec := range media.Codecs {
if codec.PayloadType != payloadType {
if codec.PayloadType != uint8(remote.PayloadType()) {
continue
}
// leave only one codec in supported media list
if len(media.Codecs) > 1 {
c.medias[i].Codecs = []*streamer.Codec{codec}
}
// forward request to passive producer GetTrack
// will create NewTrack for sendonly media
return c.GetTrack(media, codec)
return media, codec
}
}
default:
panic("not implemented")
}
return nil
// fix moment when core.ModePassiveProducer or core.ModeActiveProducer
// sends new codec with new payload type to same media
// check GetTrack
panic(core.Caller())
return nil, nil
}
pkg/webrtc/consumer.go
+49 -44
View File
@@ -2,72 +2,77 @@ package webrtc
import (
"encoding/json"
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/AlexxIT/go2rtc/pkg/h265"
"github.com/pion/rtp"
"github.com/pion/webrtc/v3"
)
func (c *Conn) GetMedias() []*streamer.Media {
func (c *Conn) GetMedias() []*core.Media {
return c.medias
}
func (c *Conn) AddTrack(media *streamer.Media, track *streamer.Track) *streamer.Track {
switch c.Mode {
case streamer.ModePassiveConsumer:
switch track.Direction {
case streamer.DirectionSendonly:
// send our track to WebRTC consumer
return c.addSendTrack(media, track)
func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiver) error {
core.Assert(media.Direction == core.DirectionSendonly)
case streamer.DirectionRecvonly:
// receive track from WebRTC consumer (microphone, backchannel, two way audio)
return c.addConsumerRecvTrack(media, track)
}
case streamer.ModePassiveProducer:
// "Stream to camera" function
consCodec := media.MatchCodec(track.Codec)
consTrack := c.GetTrack(media, consCodec)
if consTrack == nil {
for _, sender := range c.senders {
if sender.Codec == codec {
sender.HandleRTP(track)
return nil
}
return track.Bind(func(packet *rtp.Packet) error {
return consTrack.WriteRTP(packet)
})
}
panic("not implemented")
}
func (c *Conn) addConsumerRecvTrack(media *streamer.Media, track *streamer.Track) *streamer.Track {
params := webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: MimeType(track.Codec),
ClockRate: track.Codec.ClockRate,
Channels: track.Codec.Channels,
},
PayloadType: 0, // don't know if this necessary
switch c.Mode {
case core.ModePassiveConsumer: // video/audio for browser
case core.ModeActiveProducer: // go2rtc as WebRTC client (backchannel)
case core.ModePassiveProducer: // WebRTC/WHIP
default:
panic(core.Caller())
}
tr := c.getTranseiver(media.MID)
localTrack := c.getTranseiver(media.ID).Sender().Track().(*Track)
// set codec for consumer recv track so remote peer should send media with this codec
_ = tr.SetCodecPreferences([]webrtc.RTPCodecParameters{params})
sender := core.NewSender(media, track.Codec)
sender.Handler = func(packet *rtp.Packet) {
c.send += packet.MarshalSize()
//important to send with remote PayloadType
_ = localTrack.WriteRTP(codec.PayloadType, packet)
}
c.tracks = append(c.tracks, track)
return track
switch codec.Name {
case core.CodecH264:
sender.Handler = h264.RTPPay(1200, sender.Handler)
if track.Codec.IsRTP() {
sender.Handler = h264.RTPDepay(track.Codec, sender.Handler)
} else {
sender.Handler = h264.RepairAVC(track.Codec, sender.Handler)
}
case core.CodecH265:
// SafariPay because it is the only browser in the world
// that supports WebRTC + H265
sender.Handler = h265.SafariPay(1200, sender.Handler)
if track.Codec.IsRTP() {
sender.Handler = h265.RTPDepay(track.Codec, sender.Handler)
}
}
sender.HandleRTP(track)
c.senders = append(c.senders, sender)
return nil
}
func (c *Conn) MarshalJSON() ([]byte, error) {
info := &streamer.Info{
info := &core.Info{
Type: c.Desc + " " + c.Mode.String(),
RemoteAddr: c.remote,
UserAgent: c.UserAgent,
Medias: c.medias,
Tracks: c.tracks,
Recv: uint32(c.receive),
Send: uint32(c.send),
Receivers: c.receivers,
Senders: c.senders,
Recv: c.recv,
Send: c.send,
}
return json.Marshal(info)
}
pkg/webrtc/helpers.go
+49 -11
View File
@@ -3,8 +3,9 @@ package webrtc
import (
"errors"
"fmt"
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/pion/ice/v2"
"github.com/pion/sdp/v3"
"github.com/pion/stun"
"github.com/pion/webrtc/v3"
"hash/crc32"
@@ -14,6 +15,43 @@ import (
"time"
)
func UnmarshalMedias(descriptions []*sdp.MediaDescription) (medias []*core.Media) {
// 1. Sort medias, so video will always be before audio
// 2. Ignore application media from Hass default lovelace card
// 3. Ignore media without direction (inactive media)
// 4. Inverse media direction (because it is remote peer medias list)
for _, kind := range []string{core.KindVideo, core.KindAudio} {
for _, md := range descriptions {
if md.MediaName.Media != kind {
continue
}
media := core.UnmarshalMedia(md)
switch media.Direction {
case core.DirectionSendRecv:
media.Direction = core.DirectionRecvonly
medias = append(medias, media)
media = media.Clone()
media.Direction = core.DirectionSendonly
case core.DirectionRecvonly:
media.Direction = core.DirectionSendonly
case core.DirectionSendonly:
media.Direction = core.DirectionRecvonly
case "":
continue
}
medias = append(medias, media)
}
}
return
}
func NewCandidate(network, address string) (string, error) {
i := strings.LastIndexByte(address, ':')
if i < 0 {
@@ -135,25 +173,25 @@ func IsIP(host string) bool {
return true
}
func MimeType(codec *streamer.Codec) string {
func MimeType(codec *core.Codec) string {
switch codec.Name {
case streamer.CodecH264:
case core.CodecH264:
return webrtc.MimeTypeH264
case streamer.CodecH265:
case core.CodecH265:
return webrtc.MimeTypeH265
case streamer.CodecVP8:
case core.CodecVP8:
return webrtc.MimeTypeVP8
case streamer.CodecVP9:
case core.CodecVP9:
return webrtc.MimeTypeVP9
case streamer.CodecAV1:
case core.CodecAV1:
return webrtc.MimeTypeAV1
case streamer.CodecPCMU:
case core.CodecPCMU:
return webrtc.MimeTypePCMU
case streamer.CodecPCMA:
case core.CodecPCMA:
return webrtc.MimeTypePCMA
case streamer.CodecOpus:
case core.CodecOpus:
return webrtc.MimeTypeOpus
case streamer.CodecG722:
case core.CodecG722:
return webrtc.MimeTypeG722
}
panic("not implemented")
pkg/webrtc/producer.go
+36 -12
View File
@@ -1,28 +1,46 @@
package webrtc
import (
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/pion/webrtc/v3"
)
func (c *Conn) GetTrack(media *streamer.Media, codec *streamer.Codec) *streamer.Track {
if c.Mode != streamer.ModeActiveProducer && c.Mode != streamer.ModePassiveProducer {
panic("not implemented")
}
func (c *Conn) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, error) {
core.Assert(media.Direction == core.DirectionRecvonly)
for _, track := range c.tracks {
for _, track := range c.receivers {
if track.Codec == codec {
return track
return track, nil
}
}
track := streamer.NewTrack(media, codec)
switch c.Mode {
case core.ModePassiveConsumer: // backchannel from browser
// set codec for consumer recv track so remote peer should send media with this codec
params := webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: MimeType(codec),
ClockRate: codec.ClockRate,
Channels: codec.Channels,
},
PayloadType: 0, // don't know if this necessary
}
if media.Direction == streamer.DirectionRecvonly {
track = c.addSendTrack(media, track)
tr := c.getTranseiver(media.ID)
_ = tr.SetCodecPreferences([]webrtc.RTPCodecParameters{params})
case core.ModePassiveProducer, core.ModeActiveProducer:
// Passive producers: OBS Studio via WHIP or Browser
// Active producers: go2rtc as WebRTC client or WebTorrent
default:
panic(core.Caller())
}
c.tracks = append(c.tracks, track)
return track
track := core.NewReceiver(media, codec)
c.receivers = append(c.receivers, track)
return track, nil
}
func (c *Conn) Start() error {
@@ -31,5 +49,11 @@ func (c *Conn) Start() error {
}
func (c *Conn) Stop() error {
for _, receiver := range c.receivers {
receiver.Close()
}
for _, sender := range c.senders {
sender.Close()
}
return c.pc.Close()
}
pkg/webrtc/server.go
+17 -24
View File
@@ -1,7 +1,7 @@
package webrtc
import (
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/pion/sdp/v3"
"github.com/pion/webrtc/v3"
)
@@ -20,14 +20,14 @@ func (c *Conn) SetOffer(offer string) (err error) {
var tr *webrtc.RTPTransceiver
for _, attr := range md.Attributes {
switch attr.Key {
case streamer.DirectionSendRecv:
case core.DirectionSendRecv:
tr, _ = c.pc.AddTransceiverFromTrack(NewTrack(md.MediaName.Media))
case streamer.DirectionSendonly:
case core.DirectionSendonly:
tr, _ = c.pc.AddTransceiverFromKind(
webrtc.NewRTPCodecType(md.MediaName.Media),
webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionRecvonly},
)
case streamer.DirectionRecvonly:
case core.DirectionRecvonly:
tr, _ = c.pc.AddTransceiverFromTrack(
NewTrack(md.MediaName.Media),
webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly},
@@ -42,20 +42,7 @@ func (c *Conn) SetOffer(offer string) (err error) {
}
}
medias := streamer.UnmarshalMedias(sd.MediaDescriptions)
// sort medias, so video will always be before audio
// and ignore application media from Hass default lovelace card
for _, media := range medias {
if media.Kind == streamer.KindVideo {
c.medias = append(c.medias, media)
}
}
for _, media := range medias {
if media.Kind == streamer.KindAudio {
c.medias = append(c.medias, media)
}
}
c.medias = UnmarshalMedias(sd.MediaDescriptions)
return
}
@@ -67,15 +54,21 @@ func (c *Conn) GetAnswer() (answer string, err error) {
return "", err
}
// disable transceivers if we don't have track
// make direction=inactive
// don't really necessary, but anyway
// disable transceivers if we don't have track, make direction=inactive
transeivers:
for _, tr := range c.pc.GetTransceivers() {
if tr.Direction() == webrtc.RTPTransceiverDirectionSendonly && tr.Sender() == nil {
if err = tr.Stop(); err != nil {
return
for _, sender := range c.senders {
if sender.Media.ID == tr.Mid() {
continue transeivers
}
}
switch tr.Direction() {
case webrtc.RTPTransceiverDirectionSendrecv:
_ = tr.Sender().Stop()
case webrtc.RTPTransceiverDirectionSendonly:
_ = tr.Stop()
}
}
if desc, err = c.pc.CreateAnswer(nil); err != nil {
pkg/webrtc/track.go
+2 -3
View File
@@ -1,7 +1,6 @@
package webrtc
import (
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/pion/rtp"
"github.com/pion/webrtc/v3"
)
@@ -18,8 +17,8 @@ type Track struct {
func NewTrack(kind string) *Track {
return &Track{
kind: kind,
id: core.RandString(16),
streamID: core.RandString(16),
id: "go2rtc-" + kind,
streamID: "go2rtc",
}
}