gilles/go2rtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite MP4, HLS, MPEG-TS consumers

Browse Source
This commit is contained in:
Alexey Khit
2023-08-20 09:57:46 +03:00
parent f67f6e5b9f
commit 2e4e75e386
17 changed files with 492 additions and 635 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/core/core.go
+15 -5
View File
@@ -138,11 +138,13 @@ func (s *SuperProducer) Close() error {
}
type SuperConsumer struct {
Type string `json:"type,omitempty"`
URL string `json:"url,omitempty"`
Medias []*Media `json:"medias,omitempty"`
Senders []*Sender `json:"receivers,omitempty"`
Send int `json:"recv,omitempty"`
Type string `json:"type,omitempty"`
URL string `json:"url,omitempty"`
RemoteAddr string `json:"remote_addr,omitempty"`
UserAgent string `json:"user_agent,omitempty"`
Medias []*Media `json:"medias,omitempty"`
Senders []*Sender `json:"receivers,omitempty"`
Send int `json:"recv,omitempty"`
}
func (s *SuperConsumer) GetMedias() []*Media {
@@ -163,3 +165,11 @@ func (s *SuperConsumer) Close() error {
}
return nil
}
func (s *SuperConsumer) Codecs() []*Codec {
codecs := make([]*Codec, len(s.Senders))
for i, sender := range s.Senders {
codecs[i] = sender.Codec
}
return codecs
}
pkg/mp4/consumer.go
+67 -94
View File
@@ -1,7 +1,8 @@
package mp4
import (
"encoding/json"
"errors"
"io"
"sync"
"github.com/AlexxIT/go2rtc/pkg/aac"
@@ -13,27 +14,21 @@ import (
)
type Consumer struct {
core.Listener
Medias []*core.Media
Desc string
UserAgent string
RemoteAddr string
senders []*core.Sender
core.SuperConsumer
wr *core.WriteBuffer
muxer *Muxer
mu sync.Mutex
state byte
start bool
send int
Rotate int `json:"-"`
ScaleX int `json:"-"`
ScaleY int `json:"-"`
}
func (c *Consumer) GetMedias() []*core.Media {
if c.Medias == nil {
func NewConsumer(medias []*core.Media) *Consumer {
if medias == nil {
// default local medias
c.Medias = []*core.Media{
medias = []*core.Media{
{
Kind: core.KindVideo,
Direction: core.DirectionSendonly,
@@ -52,11 +47,16 @@ func (c *Consumer) GetMedias() []*core.Media {
}
}
return c.Medias
cons := &Consumer{
muxer: &Muxer{},
wr: core.NewWriteBuffer(nil),
}
cons.Medias = medias
return cons
}
func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiver) error {
trackID := byte(len(c.senders))
trackID := byte(len(c.Senders))
codec := track.Codec.Clone()
handler := core.NewSender(media, codec)
@@ -64,22 +64,19 @@ func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiv
switch track.Codec.Name {
case core.CodecH264:
handler.Handler = func(packet *rtp.Packet) {
if packet.Version != h264.RTPPacketVersionAVC {
return
}
if c.state != stateStart {
if c.state != stateInit || !h264.IsKeyframe(packet.Payload) {
if !c.start {
if !h264.IsKeyframe(packet.Payload) {
return
}
c.state = stateStart
c.start = true
}
// important to use Mutex because right fragment order
c.mu.Lock()
buf := c.muxer.Marshal(trackID, packet)
c.Fire(buf)
c.send += len(buf)
b := c.muxer.GetPayload(trackID, packet)
if n, err := c.wr.Write(b); err == nil {
c.Send += n
}
c.mu.Unlock()
}
@@ -91,21 +88,19 @@ func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiv
case core.CodecH265:
handler.Handler = func(packet *rtp.Packet) {
if packet.Version != h264.RTPPacketVersionAVC {
return
}
if c.state != stateStart {
if c.state != stateInit || !h265.IsKeyframe(packet.Payload) {
if !c.start {
if !h265.IsKeyframe(packet.Payload) {
return
}
c.state = stateStart
c.start = true
}
// important to use Mutex because right fragment order
c.mu.Lock()
buf := c.muxer.Marshal(trackID, packet)
c.Fire(buf)
c.send += len(buf)
b := c.muxer.GetPayload(trackID, packet)
if n, err := c.wr.Write(b); err == nil {
c.Send += n
}
c.mu.Unlock()
}
@@ -115,14 +110,16 @@ func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiv
default:
handler.Handler = func(packet *rtp.Packet) {
if c.state != stateStart {
if !c.start {
return
}
// important to use Mutex because right fragment order
c.mu.Lock()
buf := c.muxer.Marshal(trackID, packet)
c.Fire(buf)
c.send += len(buf)
b := c.muxer.GetPayload(trackID, packet)
if n, err := c.wr.Write(b); err == nil {
c.Send += n
}
c.mu.Unlock()
}
@@ -142,64 +139,40 @@ func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiv
}
if handler.Handler == nil {
println("ERROR: MP4 unsupported codec: " + track.Codec.String())
return nil
s := "mp4: unsupported codec: " + track.Codec.String()
println(s)
return errors.New(s)
}
c.muxer.AddTrack(codec)
handler.HandleRTP(track)
c.senders = append(c.senders, handler)
c.Senders = append(c.Senders, handler)
return nil
}
func (c *Consumer) WriteTo(wr io.Writer) (int64, error) {
init, err := c.muxer.GetInit()
if err != nil {
return 0, err
}
if c.Rotate != 0 {
PatchVideoRotate(init, c.Rotate)
}
if c.ScaleX != 0 && c.ScaleY != 0 {
PatchVideoScale(init, c.ScaleX, c.ScaleY)
}
if _, err = wr.Write(init); err != nil {
return 0, err
}
return c.wr.WriteTo(wr)
}
func (c *Consumer) Stop() error {
for _, sender := range c.senders {
sender.Close()
}
return nil
}
func (c *Consumer) Codecs() []*core.Codec {
codecs := make([]*core.Codec, len(c.senders))
for i, sender := range c.senders {
codecs[i] = sender.Codec
}
return codecs
}
func (c *Consumer) MimeCodecs() string {
return c.muxer.MimeCodecs(c.Codecs())
}
func (c *Consumer) MimeType() string {
return `video/mp4; codecs="` + c.MimeCodecs() + `"`
}
func (c *Consumer) Init() ([]byte, error) {
c.muxer = &Muxer{}
return c.muxer.GetInit(c.Codecs())
}
func (c *Consumer) Start() {
for _, sender := range c.senders {
switch sender.Codec.Name {
case core.CodecH264, core.CodecH265:
c.state = stateInit
return
}
}
c.state = stateStart
}
func (c *Consumer) MarshalJSON() ([]byte, error) {
info := &core.Info{
Type: c.Desc + " passive consumer",
RemoteAddr: c.RemoteAddr,
UserAgent: c.UserAgent,
Medias: c.Medias,
Senders: c.senders,
Send: c.send,
}
return json.Marshal(info)
_ = c.SuperConsumer.Close()
return c.wr.Close()
}
pkg/mp4/consumer_test.go
-77
View File
@@ -1,77 +0,0 @@
package mp4
import (
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/pion/rtp"
"github.com/stretchr/testify/require"
"testing"
"time"
)
func TestStartH264(t *testing.T) {
codec := &core.Codec{Name: core.CodecH264}
track := core.NewReceiver(nil, codec)
packetKey := &rtp.Packet{
Header: rtp.Header{Marker: true},
Payload: []byte{h264.NALUTypeIFrame, 0, 0},
}
packetNotKey := &rtp.Packet{
Header: rtp.Header{Marker: true},
Payload: []byte{h264.NALUTypePFrame, 0, 0},
}
cons := &Consumer{}
err := cons.AddTrack(nil, nil, track)
require.Nil(t, err)
track.WriteRTP(packetKey)
time.Sleep(time.Millisecond)
_, err = cons.Init()
require.Nil(t, err)
cons.Start()
track.WriteRTP(packetNotKey)
time.Sleep(time.Millisecond)
require.Zero(t, cons.send)
track.WriteRTP(packetKey)
time.Sleep(time.Millisecond)
require.NotZero(t, cons.send)
}
func TestStartOPUS(t *testing.T) {
// Test for fix this issue
// https://github.com/AlexxIT/go2rtc/issues/404
codec := &core.Codec{Name: core.CodecOpus}
track := core.NewReceiver(nil, codec)
cons := &Consumer{}
err := cons.AddTrack(nil, nil, track)
require.Nil(t, err)
track.WriteRTP(&rtp.Packet{
Payload: []byte{0},
})
time.Sleep(time.Millisecond)
require.Zero(t, cons.send)
_, err = cons.Init()
require.Nil(t, err)
cons.Start()
track.WriteRTP(&rtp.Packet{
Payload: []byte{0},
})
time.Sleep(time.Millisecond)
require.NotZero(t, cons.send)
}
pkg/mp4/keyframe.go
+102
View File
@@ -0,0 +1,102 @@
package mp4
import (
"io"
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/AlexxIT/go2rtc/pkg/h265"
"github.com/pion/rtp"
)
type Keyframe struct {
core.SuperConsumer
wr *core.WriteBuffer
muxer *Muxer
}
func NewKeyframe(medias []*core.Media) *Keyframe {
if medias == nil {
medias = []*core.Media{
{
Kind: core.KindVideo,
Direction: core.DirectionSendonly,
Codecs: []*core.Codec{
{Name: core.CodecH264},
{Name: core.CodecH265},
},
},
}
}
cons := &Keyframe{
muxer: &Muxer{},
wr: core.NewWriteBuffer(nil),
}
cons.Medias = medias
return cons
}
func (c *Keyframe) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiver) error {
c.muxer.AddTrack(track.Codec)
init, err := c.muxer.GetInit()
if err != nil {
return err
}
handler := core.NewSender(media, track.Codec)
switch track.Codec.Name {
case core.CodecH264:
handler.Handler = func(packet *rtp.Packet) {
if !h264.IsKeyframe(packet.Payload) {
return
}
// important to use Mutex because right fragment order
b := c.muxer.GetPayload(0, packet)
b = append(init, b...)
if n, err := c.wr.Write(b); err == nil {
c.Send += n
}
}
if track.Codec.IsRTP() {
handler.Handler = h264.RTPDepay(track.Codec, handler.Handler)
} else {
handler.Handler = h264.RepairAVCC(track.Codec, handler.Handler)
}
case core.CodecH265:
handler.Handler = func(packet *rtp.Packet) {
if !h265.IsKeyframe(packet.Payload) {
return
}
// important to use Mutex because right fragment order
b := c.muxer.GetPayload(0, packet)
b = append(init, b...)
if n, err := c.wr.Write(b); err == nil {
c.Send += n
}
}
if track.Codec.IsRTP() {
handler.Handler = h265.RTPDepay(track.Codec, handler.Handler)
}
}
handler.HandleRTP(track)
c.Senders = append(c.Senders, handler)
return nil
}
func (c *Keyframe) WriteTo(wr io.Writer) (int64, error) {
return c.wr.WriteTo(wr)
}
func (c *Keyframe) Stop() error {
_ = c.SuperConsumer.Close()
return c.wr.Close()
}
pkg/mp4/mime.go
+45
View File
@@ -0,0 +1,45 @@
package mp4
import (
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/AlexxIT/go2rtc/pkg/h264"
)
const (
MimeH264 = "avc1.640029"
MimeH265 = "hvc1.1.6.L153.B0"
MimeAAC = "mp4a.40.2"
MimeFlac = "flac"
MimeOpus = "opus"
)
func MimeCodecs(codecs []*core.Codec) string {
var s string
for i, codec := range codecs {
if i > 0 {
s += ","
}
switch codec.Name {
case core.CodecH264:
s += "avc1." + h264.GetProfileLevelID(codec.FmtpLine)
case core.CodecH265:
// H.265 profile=main level=5.1
// hvc1 - supported in Safari, hev1 - doesn't, both supported in Chrome
s += MimeH265
case core.CodecAAC:
s += MimeAAC
case core.CodecOpus:
s += MimeOpus
case core.CodecFLAC:
s += MimeFlac
}
}
return s
}
func ContentType(codecs []*core.Codec) string {
return `video/mp4; codecs="` + MimeCodecs(codecs) + `"`
}
pkg/mp4/muxer.go
+16 -49
View File
@@ -12,55 +12,26 @@ import (
)
type Muxer struct {
fragIndex uint32
dts []uint64
pts []uint32
codecs []*core.Codec
index uint32
dts []uint64
pts []uint32
codecs []*core.Codec
}
const (
MimeH264 = "avc1.640029"
MimeH265 = "hvc1.1.6.L153.B0"
MimeAAC = "mp4a.40.2"
MimeFlac = "flac"
MimeOpus = "opus"
)
func (m *Muxer) MimeCodecs(codecs []*core.Codec) string {
var s string
for i, codec := range codecs {
if i > 0 {
s += ","
}
switch codec.Name {
case core.CodecH264:
s += "avc1." + h264.GetProfileLevelID(codec.FmtpLine)
case core.CodecH265:
// H.265 profile=main level=5.1
// hvc1 - supported in Safari, hev1 - doesn't, both supported in Chrome
s += MimeH265
case core.CodecAAC:
s += MimeAAC
case core.CodecOpus:
s += MimeOpus
case core.CodecFLAC:
s += MimeFlac
}
}
return s
func (m *Muxer) AddTrack(codec *core.Codec) {
m.dts = append(m.dts, 0)
m.pts = append(m.pts, 0)
m.codecs = append(m.codecs, codec)
}
func (m *Muxer) GetInit(codecs []*core.Codec) ([]byte, error) {
func (m *Muxer) GetInit() ([]byte, error) {
mv := iso.NewMovie(1024)
mv.WriteFileType()
mv.StartAtom(iso.Moov)
mv.WriteMovieHeader()
for i, codec := range codecs {
for i, codec := range m.codecs {
switch codec.Name {
case core.CodecH264:
sps, pps := h264.GetParameterSet(codec.FmtpLine)
@@ -119,14 +90,10 @@ func (m *Muxer) GetInit(codecs []*core.Codec) ([]byte, error) {
uint32(i+1), codec.Name, codec.ClockRate, codec.Channels, nil,
)
}
m.dts = append(m.dts, 0)
m.pts = append(m.pts, 0)
m.codecs = append(m.codecs, codec)
}
mv.StartAtom(iso.MoovMvex)
for i := range codecs {
for i := range m.codecs {
mv.WriteTrackExtend(uint32(i + 1))
}
mv.EndAtom() // MVEX
@@ -137,17 +104,17 @@ func (m *Muxer) GetInit(codecs []*core.Codec) ([]byte, error) {
}
func (m *Muxer) Reset() {
m.fragIndex = 0
m.index = 0
for i := range m.dts {
m.dts[i] = 0
m.pts[i] = 0
}
}
func (m *Muxer) Marshal(trackID byte, packet *rtp.Packet) []byte {
func (m *Muxer) GetPayload(trackID byte, packet *rtp.Packet) []byte {
codec := m.codecs[trackID]
m.fragIndex++
m.index++
duration := packet.Timestamp - m.pts[trackID]
m.pts[trackID] = packet.Timestamp
@@ -185,11 +152,11 @@ func (m *Muxer) Marshal(trackID byte, packet *rtp.Packet) []byte {
mv := iso.NewMovie(1024 + size)
mv.WriteMovieFragment(
m.fragIndex, uint32(trackID+1), duration, uint32(size), flags, m.dts[trackID],
m.index, uint32(trackID+1), duration, uint32(size), flags, m.dts[trackID],
)
mv.WriteData(packet.Payload)
//log.Printf("[MP4] track=%d ts=%6d dur=%5d idx=%3d len=%d", trackID+1, m.dts[trackID], duration, m.fragIndex, len(packet.Payload))
//log.Printf("[MP4] track=%d ts=%6d dur=%5d idx=%3d len=%d", trackID+1, m.dts[trackID], duration, m.index, len(packet.Payload))
m.dts[trackID] += uint64(duration)
pkg/mp4/segment.go
-151
View File
@@ -1,151 +0,0 @@
package mp4
import (
"encoding/json"
"github.com/AlexxIT/go2rtc/pkg/core"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/AlexxIT/go2rtc/pkg/h265"
"github.com/pion/rtp"
)
type Segment struct {
core.Listener
Medias []*core.Media
UserAgent string
RemoteAddr string
senders []*core.Sender
MimeType string
OnlyKeyframe bool
send int
}
func (c *Segment) GetMedias() []*core.Media {
if c.Medias != nil {
return c.Medias
}
// default local medias
return []*core.Media{
{
Kind: core.KindVideo,
Direction: core.DirectionSendonly,
Codecs: []*core.Codec{
{Name: core.CodecH264},
{Name: core.CodecH265},
},
},
}
}
func (c *Segment) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiver) error {
muxer := &Muxer{}
codecs := []*core.Codec{track.Codec}
init, err := muxer.GetInit(codecs)
if err != nil {
return nil
}
c.MimeType = `video/mp4; codecs="` + muxer.MimeCodecs(codecs) + `"`
handler := core.NewSender(media, track.Codec)
switch track.Codec.Name {
case core.CodecH264:
if c.OnlyKeyframe {
handler.Handler = func(packet *rtp.Packet) {
if !h264.IsKeyframe(packet.Payload) {
return
}
buf := muxer.Marshal(0, packet)
c.Fire(append(init, buf...))
c.send += len(buf)
}
} else {
var buf []byte
handler.Handler = func(packet *rtp.Packet) {
if h264.IsKeyframe(packet.Payload) {
// fist frame - send only IFrame
// other frames - send IFrame and all PFrames
if buf == nil {
buf = append(buf, init...)
b := muxer.Marshal(0, packet)
buf = append(buf, b...)
}
c.Fire(buf)
c.send += len(buf)
buf = buf[:0]
buf = append(buf, init...)
muxer.Reset()
}
if buf != nil {
b := muxer.Marshal(0, packet)
buf = append(buf, b...)
}
}
}
if track.Codec.IsRTP() {
handler.Handler = h264.RTPDepay(track.Codec, handler.Handler)
} else {
handler.Handler = h264.RepairAVCC(track.Codec, handler.Handler)
}
case core.CodecH265:
handler.Handler = func(packet *rtp.Packet) {
if !h265.IsKeyframe(packet.Payload) {
return
}
buf := muxer.Marshal(0, packet)
c.Fire(append(init, buf...))
c.send += len(buf)
}
if track.Codec.IsRTP() {
handler.Handler = h265.RTPDepay(track.Codec, handler.Handler)
}
default:
panic(core.UnsupportedCodec)
}
handler.HandleRTP(track)
c.senders = append(c.senders, handler)
return nil
}
func (c *Segment) Stop() error {
for _, sender := range c.senders {
sender.Close()
}
return nil
}
func (c *Segment) MarshalJSON() ([]byte, error) {
info := &core.Info{
Type: "MP4/WebSocket passive consumer",
RemoteAddr: c.RemoteAddr,
UserAgent: c.UserAgent,
Medias: c.Medias,
Senders: c.senders,
Send: c.send,
}
return json.Marshal(info)
}
pkg/mpegts/consumer.go
+1 -1
View File
@@ -107,7 +107,7 @@ func (c *Consumer) WriteTo(wr io.Writer) (int64, error) {
return c.wr.WriteTo(wr)
}
func (c *Consumer) Close() error {
func (c *Consumer) Stop() error {
_ = c.SuperConsumer.Close()
return c.wr.Close()
}
pkg/mpegts/muxer.go
+10 -1
View File
@@ -45,17 +45,26 @@ func (m *Muxer) GetHeader() []byte {
func (m *Muxer) GetPayload(pid uint16, pts uint32, payload []byte) []byte {
pes := m.pes[pid]
size := 8 + len(payload)
b := make([]byte, 14+len(payload))
_ = b[14] // bounds
b[0] = 0
b[1] = 0
b[2] = 1
b[3] = pes.StreamID
binary.BigEndian.PutUint16(b[4:], 8+uint16(len(payload)))
b[6] = 0x80 // Marker bits (binary)
b[7] = 0x80 // PTS indicator
b[8] = 5 // PES header length
// zero size is OK for video stream
if size <= 0xFFFF {
binary.BigEndian.PutUint16(b[4:], uint16(size))
}
WriteTime(b[9:], pts)
copy(b[14:], payload)
switch pes.StreamType {