gilles/go2rtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support OPUS audio for MSE/MP4

Browse Source
This commit is contained in:
Alexey Khit
2023-01-27 12:36:36 +03:00
parent 073acdfec9
commit a1fec1c6f6
9 changed files with 98 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/mov/atoms.go → pkg/iso/atoms.go
+5 -5
View File
@@ -1,4 +1,4 @@
package mov
package iso
const (
Ftyp = "ftyp"
@@ -210,7 +210,7 @@ func (m *Movie) WriteTrackExtend(id uint32) {
m.EndAtom()
}
func (m *Movie) WriteVideoTrack(id, timescale uint32, width, height uint16, conf []byte, h264 bool) {
func (m *Movie) WriteVideoTrack(id uint32, codec string, timescale uint32, width, height uint16, conf []byte) {
m.StartAtom(MoovTrak)
m.WriteTrackHeader(id, width, height)
@@ -222,7 +222,7 @@ func (m *Movie) WriteVideoTrack(id, timescale uint32, width, height uint16, conf
m.WriteVideoMediaInfo()
m.WriteDataInfo()
m.WriteSampleTable(func() {
m.WriteH26X(width, height, conf, h264)
m.WriteVideo(codec, width, height, conf)
})
m.EndAtom() // MINF
@@ -230,7 +230,7 @@ func (m *Movie) WriteVideoTrack(id, timescale uint32, width, height uint16, conf
m.EndAtom() // TRAK
}
func (m *Movie) WriteAudioTrack(id uint32, timescale uint32, channels, sampleSize uint16, conf []byte) {
func (m *Movie) WriteAudioTrack(id uint32, codec string, timescale uint32, channels uint16, conf []byte) {
m.StartAtom(MoovTrak)
m.WriteTrackHeader(id, 0, 0)
@@ -242,7 +242,7 @@ func (m *Movie) WriteAudioTrack(id uint32, timescale uint32, channels, sampleSiz
m.WriteAudioMediaInfo()
m.WriteDataInfo()
m.WriteSampleTable(func() {
m.WriteMP4A(channels, sampleSize, timescale, conf)
m.WriteAudio(codec, channels, timescale, conf)
})
m.EndAtom() // MINF
pkg/mov/codecs.go → pkg/iso/codecs.go
+38 -12
View File
@@ -1,4 +1,6 @@
package mov
package iso
import "github.com/AlexxIT/go2rtc/pkg/streamer"
const (
MoovTrakMdiaMinfStblStsdAvc1 = "avc1"
@@ -6,14 +8,18 @@ const (
MoovTrakMdiaMinfStblStsdHev1 = "hev1"
MoovTrakMdiaMinfStblStsdHev1HvcC = "hvcC"
MoovTrakMdiaMinfStblStsdMp4a = "mp4a"
MoovTrakMdiaMinfStblStsdOpus = "Opus"
)
func (m *Movie) WriteH26X(width, height uint16, conf []byte, h264 bool) {
func (m *Movie) WriteVideo(codec string, width, height uint16, conf []byte) {
// https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap3/qtff3.html
if h264 {
switch codec {
case streamer.CodecH264:
m.StartAtom(MoovTrakMdiaMinfStblStsdAvc1)
} else {
case streamer.CodecH265:
m.StartAtom(MoovTrakMdiaMinfStblStsdHev1)
default:
panic("unsupported iso video: " + codec)
}
m.Skip(6)
m.WriteUint16(1) // data_reference_index
@@ -32,9 +38,10 @@ func (m *Movie) WriteH26X(width, height uint16, conf []byte, h264 bool) {
m.WriteUint16(24) // depth
m.WriteUint16(0xFFFF) // color table id (-1)
if h264 {
switch codec {
case streamer.CodecH264:
m.StartAtom(MoovTrakMdiaMinfStblStsdAvc1AvcC)
} else {
case streamer.CodecH265:
m.StartAtom(MoovTrakMdiaMinfStblStsdHev1HvcC)
}
m.Write(conf)
@@ -43,25 +50,37 @@ func (m *Movie) WriteH26X(width, height uint16, conf []byte, h264 bool) {
m.EndAtom() // AVC1
}
func (m *Movie) WriteMP4A(channels, sampleSize uint16, sampleRate uint32, conf []byte) {
m.StartAtom(MoovTrakMdiaMinfStblStsdMp4a)
func (m *Movie) WriteAudio(codec string, channels uint16, sampleRate uint32, conf []byte) {
switch codec {
case streamer.CodecAAC:
m.StartAtom(MoovTrakMdiaMinfStblStsdMp4a)
case streamer.CodecOpus:
m.StartAtom(MoovTrakMdiaMinfStblStsdOpus)
default:
panic("unsupported iso audio: " + codec)
}
m.Skip(6)
m.WriteUint16(1) // data_reference_index
m.Skip(2) // version
m.Skip(2) // revision
m.Skip(4) // vendor
m.WriteUint16(channels) // channel_count
m.WriteUint16(sampleSize) // sample_size
m.WriteUint16(16) // sample_size
m.Skip(2) // compression id
m.Skip(2) // reserved
m.WriteFloat32(float64(sampleRate)) // sample_rate
m.WriteESDS(conf)
switch codec {
case streamer.CodecAAC:
m.WriteEsdsAAC(conf)
case streamer.CodecOpus:
m.WriteDops()
}
m.EndAtom() // MP4A
m.EndAtom() // MP4A/OPUS
}
func (m *Movie) WriteESDS(conf []byte) {
func (m *Movie) WriteEsdsAAC(conf []byte) {
m.StartAtom("esds")
m.Skip(1) // version
m.Skip(3) // flags
@@ -95,3 +114,10 @@ func (m *Movie) WriteESDS(conf []byte) {
m.EndAtom() // ESDS
}
func (m *Movie) WriteDops() {
// don't know what means this magic
m.StartAtom("dOps")
m.WriteBytes(0, 0x02, 0x01, 0x38, 0, 0, 0xBB, 0x80, 0, 0, 0)
m.EndAtom()
}
pkg/mov/mov.go → pkg/iso/iso.go
+1 -1
View File
@@ -1,4 +1,4 @@
package mov
package iso
import (
"encoding/binary"
pkg/mp4/consumer.go
+16
View File
@@ -50,6 +50,7 @@ func (c *Consumer) GetMedias() []*streamer.Media {
Direction: streamer.DirectionRecvonly,
Codecs: []*streamer.Codec{
{Name: streamer.CodecAAC},
{Name: streamer.CodecOpus},
},
},
}
@@ -140,6 +141,21 @@ func (c *Consumer) AddTrack(media *streamer.Media, track *streamer.Track) *strea
push = wrapper(push)
}
return track.Bind(push)
case streamer.CodecOpus:
push := func(packet *rtp.Packet) error {
if c.wait != waitNone {
return nil
}
buf := c.muxer.Marshal(trackID, packet)
atomic.AddUint32(&c.send, uint32(len(buf)))
c.Fire(buf)
return nil
}
return track.Bind(push)
}
pkg/mp4/muxer.go
+28 -12
View File
@@ -4,7 +4,7 @@ import (
"encoding/hex"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/AlexxIT/go2rtc/pkg/h265"
"github.com/AlexxIT/go2rtc/pkg/mov"
"github.com/AlexxIT/go2rtc/pkg/iso"
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/deepch/vdk/codec/h264parser"
"github.com/deepch/vdk/codec/h265parser"
@@ -18,6 +18,13 @@ type Muxer struct {
pts []uint32
}
const (
MimeH264 = "avc1.640029"
MimeH265 = "hvc1.1.6.L153.B0"
MimeAAC = "mp4a.40.2"
MimeOpus = "opus"
)
func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
s := `video/mp4; codecs="`
@@ -32,9 +39,11 @@ func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
case streamer.CodecH265:
// H.265 profile=main level=5.1
// hvc1 - supported in Safari, hev1 - doesn't, both supported in Chrome
s += "hvc1.1.6.L153.B0"
s += MimeH265
case streamer.CodecAAC:
s += "mp4a.40.2"
s += MimeAAC
case streamer.CodecOpus:
s += MimeOpus
}
}
@@ -42,10 +51,10 @@ func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
}
func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
mv := mov.NewMovie(1024)
mv := iso.NewMovie(1024)
mv.WriteFileType()
mv.StartAtom(mov.Moov)
mv.StartAtom(iso.Moov)
mv.WriteMovieHeader()
for i, codec := range codecs {
@@ -64,9 +73,9 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
}
mv.WriteVideoTrack(
uint32(i+1), codec.ClockRate,
uint32(i+1), codec.Name, codec.ClockRate,
uint16(codecData.Width()), uint16(codecData.Height()),
codecData.AVCDecoderConfRecordBytes(), true,
codecData.AVCDecoderConfRecordBytes(),
)
m.flags = append(m.flags, 0x1010000)
@@ -86,9 +95,9 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
}
mv.WriteVideoTrack(
uint32(i+1), codec.ClockRate,
uint32(i+1), codec.Name, codec.ClockRate,
uint16(codecData.Width()), uint16(codecData.Height()),
codecData.AVCDecoderConfRecordBytes(), false,
codecData.AVCDecoderConfRecordBytes(),
)
m.flags = append(m.flags, 0x1010000)
@@ -101,7 +110,14 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
}
mv.WriteAudioTrack(
uint32(i+1), codec.ClockRate, codec.Channels, 16, b,
uint32(i+1), codec.Name, codec.ClockRate, codec.Channels, b,
)
m.flags = append(m.flags, 0x2000000)
case streamer.CodecOpus:
mv.WriteAudioTrack(
uint32(i+1), codec.Name, codec.ClockRate, codec.Channels, nil,
)
m.flags = append(m.flags, 0x2000000)
@@ -111,7 +127,7 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
m.dts = append(m.dts, 0)
}
mv.StartAtom(mov.MoovMvex)
mv.StartAtom(iso.MoovMvex)
for i := range codecs {
mv.WriteTrackExtend(uint32(i + 1))
}
@@ -147,7 +163,7 @@ func (m *Muxer) Marshal(trackID byte, packet *rtp.Packet) []byte {
}
m.pts[trackID] = newTime
mv := mov.NewMovie(1024 + len(packet.Payload))
mv := iso.NewMovie(1024 + len(packet.Payload))
mv.WriteMovieFragment(
m.fragIndex, uint32(trackID+1), duration,
uint32(len(packet.Payload)),