Add support OPUS audio for MSE/MP4

cmd/mp4/ws.go

@@ -106,15 +106,18 @@ func parseMedias(codecs string, parseAudio bool) (medias []*streamer.Media) {
 
 	for _, name := range strings.Split(codecs, ",") {
 		switch name {
-		case "avc1.640029":
+		case mp4.MimeH264:
 			codec := &streamer.Codec{Name: streamer.CodecH264}
 			videos = append(videos, codec)
-		case "hvc1.1.6.L153.B0":
+		case mp4.MimeH265:
 			codec := &streamer.Codec{Name: streamer.CodecH265}
 			videos = append(videos, codec)
-		case "mp4a.40.2":
+		case mp4.MimeAAC:
 			codec := &streamer.Codec{Name: streamer.CodecAAC}
 			audios = append(audios, codec)
+		case mp4.MimeOpus:
+			codec := &streamer.Codec{Name: streamer.CodecOpus}
+			audios = append(audios, codec)
 		}
 	}
 

pkg/iso/atoms.go

@@ -1,4 +1,4 @@
-package mov
+package iso
 
 const (
 	Ftyp                        = "ftyp"
@@ -210,7 +210,7 @@ func (m *Movie) WriteTrackExtend(id uint32) {
 	m.EndAtom()
 }
 
-func (m *Movie) WriteVideoTrack(id, timescale uint32, width, height uint16, conf []byte, h264 bool) {
+func (m *Movie) WriteVideoTrack(id uint32, codec string, timescale uint32, width, height uint16, conf []byte) {
 	m.StartAtom(MoovTrak)
 	m.WriteTrackHeader(id, width, height)
 
@@ -222,7 +222,7 @@ func (m *Movie) WriteVideoTrack(id, timescale uint32, width, height uint16, conf
 	m.WriteVideoMediaInfo()
 	m.WriteDataInfo()
 	m.WriteSampleTable(func() {
-		m.WriteH26X(width, height, conf, h264)
+		m.WriteVideo(codec, width, height, conf)
 	})
 	m.EndAtom() // MINF
 
@@ -230,7 +230,7 @@ func (m *Movie) WriteVideoTrack(id, timescale uint32, width, height uint16, conf
 	m.EndAtom() // TRAK
 }
 
-func (m *Movie) WriteAudioTrack(id uint32, timescale uint32, channels, sampleSize uint16, conf []byte) {
+func (m *Movie) WriteAudioTrack(id uint32, codec string, timescale uint32, channels uint16, conf []byte) {
 	m.StartAtom(MoovTrak)
 	m.WriteTrackHeader(id, 0, 0)
 
@@ -242,7 +242,7 @@ func (m *Movie) WriteAudioTrack(id uint32, timescale uint32, channels, sampleSiz
 	m.WriteAudioMediaInfo()
 	m.WriteDataInfo()
 	m.WriteSampleTable(func() {
-		m.WriteMP4A(channels, sampleSize, timescale, conf)
+		m.WriteAudio(codec, channels, timescale, conf)
 	})
 	m.EndAtom() // MINF
 

pkg/iso/codecs.go

@@ -1,4 +1,6 @@
-package mov
+package iso
+
+import "github.com/AlexxIT/go2rtc/pkg/streamer"
 
 const (
 	MoovTrakMdiaMinfStblStsdAvc1     = "avc1"
@@ -6,14 +8,18 @@ const (
 	MoovTrakMdiaMinfStblStsdHev1     = "hev1"
 	MoovTrakMdiaMinfStblStsdHev1HvcC = "hvcC"
 	MoovTrakMdiaMinfStblStsdMp4a     = "mp4a"
+	MoovTrakMdiaMinfStblStsdOpus     = "Opus"
 )
 
-func (m *Movie) WriteH26X(width, height uint16, conf []byte, h264 bool) {
+func (m *Movie) WriteVideo(codec string, width, height uint16, conf []byte) {
 	// https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap3/qtff3.html
-	if h264 {
+	switch codec {
+	case streamer.CodecH264:
 		m.StartAtom(MoovTrakMdiaMinfStblStsdAvc1)
-	} else {
+	case streamer.CodecH265:
 		m.StartAtom(MoovTrakMdiaMinfStblStsdHev1)
+	default:
+		panic("unsupported iso video: " + codec)
 	}
 	m.Skip(6)
 	m.WriteUint16(1)      // data_reference_index
@@ -32,9 +38,10 @@ func (m *Movie) WriteH26X(width, height uint16, conf []byte, h264 bool) {
 	m.WriteUint16(24)     // depth
 	m.WriteUint16(0xFFFF) // color table id (-1)
 
-	if h264 {
+	switch codec {
+	case streamer.CodecH264:
 		m.StartAtom(MoovTrakMdiaMinfStblStsdAvc1AvcC)
-	} else {
+	case streamer.CodecH265:
 		m.StartAtom(MoovTrakMdiaMinfStblStsdHev1HvcC)
 	}
 	m.Write(conf)
@@ -43,25 +50,37 @@ func (m *Movie) WriteH26X(width, height uint16, conf []byte, h264 bool) {
 	m.EndAtom() // AVC1
 }
 
-func (m *Movie) WriteMP4A(channels, sampleSize uint16, sampleRate uint32, conf []byte) {
+func (m *Movie) WriteAudio(codec string, channels uint16, sampleRate uint32, conf []byte) {
-	m.StartAtom(MoovTrakMdiaMinfStblStsdMp4a)
+	switch codec {
+	case streamer.CodecAAC:
+		m.StartAtom(MoovTrakMdiaMinfStblStsdMp4a)
+	case streamer.CodecOpus:
+		m.StartAtom(MoovTrakMdiaMinfStblStsdOpus)
+	default:
+		panic("unsupported iso audio: " + codec)
+	}
 	m.Skip(6)
 	m.WriteUint16(1)                    // data_reference_index
 	m.Skip(2)                           // version
 	m.Skip(2)                           // revision
 	m.Skip(4)                           // vendor
 	m.WriteUint16(channels)             // channel_count
-	m.WriteUint16(sampleSize)           // sample_size
+	m.WriteUint16(16)                   // sample_size
 	m.Skip(2)                           // compression id
 	m.Skip(2)                           // reserved
 	m.WriteFloat32(float64(sampleRate)) // sample_rate
 
-	m.WriteESDS(conf)
+	switch codec {
+	case streamer.CodecAAC:
+		m.WriteEsdsAAC(conf)
+	case streamer.CodecOpus:
+		m.WriteDops()
+	}
 
-	m.EndAtom() // MP4A
+	m.EndAtom() // MP4A/OPUS
 }
 
-func (m *Movie) WriteESDS(conf []byte) {
+func (m *Movie) WriteEsdsAAC(conf []byte) {
 	m.StartAtom("esds")
 	m.Skip(1) // version
 	m.Skip(3) // flags
@@ -95,3 +114,10 @@ func (m *Movie) WriteESDS(conf []byte) {
 
 	m.EndAtom() // ESDS
 }
+
+func (m *Movie) WriteDops() {
+	// don't know what means this magic
+	m.StartAtom("dOps")
+	m.WriteBytes(0, 0x02, 0x01, 0x38, 0, 0, 0xBB, 0x80, 0, 0, 0)
+	m.EndAtom()
+}

pkg/iso/iso.go

@@ -1,4 +1,4 @@
-package mov
+package iso
 
 import (
 	"encoding/binary"

pkg/mp4/consumer.go

@@ -50,6 +50,7 @@ func (c *Consumer) GetMedias() []*streamer.Media {
 			Direction: streamer.DirectionRecvonly,
 			Codecs: []*streamer.Codec{
 				{Name: streamer.CodecAAC},
+				{Name: streamer.CodecOpus},
 			},
 		},
 	}
@@ -140,6 +141,21 @@ func (c *Consumer) AddTrack(media *streamer.Media, track *streamer.Track) *strea
 			push = wrapper(push)
 		}
 
+		return track.Bind(push)
+
+	case streamer.CodecOpus:
+		push := func(packet *rtp.Packet) error {
+			if c.wait != waitNone {
+				return nil
+			}
+
+			buf := c.muxer.Marshal(trackID, packet)
+			atomic.AddUint32(&c.send, uint32(len(buf)))
+			c.Fire(buf)
+
+			return nil
+		}
+
 		return track.Bind(push)
 	}
 

pkg/mp4/muxer.go

@@ -4,7 +4,7 @@ import (
 	"encoding/hex"
 	"github.com/AlexxIT/go2rtc/pkg/h264"
 	"github.com/AlexxIT/go2rtc/pkg/h265"
-	"github.com/AlexxIT/go2rtc/pkg/mov"
+	"github.com/AlexxIT/go2rtc/pkg/iso"
 	"github.com/AlexxIT/go2rtc/pkg/streamer"
 	"github.com/deepch/vdk/codec/h264parser"
 	"github.com/deepch/vdk/codec/h265parser"
@@ -18,6 +18,13 @@ type Muxer struct {
 	pts       []uint32
 }
 
+const (
+	MimeH264 = "avc1.640029"
+	MimeH265 = "hvc1.1.6.L153.B0"
+	MimeAAC  = "mp4a.40.2"
+	MimeOpus = "opus"
+)
+
 func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
 	s := `video/mp4; codecs="`
 
@@ -32,9 +39,11 @@ func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
 		case streamer.CodecH265:
 			// H.265 profile=main level=5.1
 			// hvc1 - supported in Safari, hev1 - doesn't, both supported in Chrome
-			s += "hvc1.1.6.L153.B0"
+			s += MimeH265
 		case streamer.CodecAAC:
-			s += "mp4a.40.2"
+			s += MimeAAC
+		case streamer.CodecOpus:
+			s += MimeOpus
 		}
 	}
 
@@ -42,10 +51,10 @@ func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
 }
 
 func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
-	mv := mov.NewMovie(1024)
+	mv := iso.NewMovie(1024)
 	mv.WriteFileType()
 
-	mv.StartAtom(mov.Moov)
+	mv.StartAtom(iso.Moov)
 	mv.WriteMovieHeader()
 
 	for i, codec := range codecs {
@@ -64,9 +73,9 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
 			}
 
 			mv.WriteVideoTrack(
-				uint32(i+1), codec.ClockRate,
+				uint32(i+1), codec.Name, codec.ClockRate,
 				uint16(codecData.Width()), uint16(codecData.Height()),
-				codecData.AVCDecoderConfRecordBytes(), true,
+				codecData.AVCDecoderConfRecordBytes(),
 			)
 
 			m.flags = append(m.flags, 0x1010000)
@@ -86,9 +95,9 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
 			}
 
 			mv.WriteVideoTrack(
-				uint32(i+1), codec.ClockRate,
+				uint32(i+1), codec.Name, codec.ClockRate,
 				uint16(codecData.Width()), uint16(codecData.Height()),
-				codecData.AVCDecoderConfRecordBytes(), false,
+				codecData.AVCDecoderConfRecordBytes(),
 			)
 
 			m.flags = append(m.flags, 0x1010000)
@@ -101,7 +110,14 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
 			}
 
 			mv.WriteAudioTrack(
-				uint32(i+1), codec.ClockRate, codec.Channels, 16, b,
+				uint32(i+1), codec.Name, codec.ClockRate, codec.Channels, b,
+			)
+
+			m.flags = append(m.flags, 0x2000000)
+
+		case streamer.CodecOpus:
+			mv.WriteAudioTrack(
+				uint32(i+1), codec.Name, codec.ClockRate, codec.Channels, nil,
 			)
 
 			m.flags = append(m.flags, 0x2000000)
@@ -111,7 +127,7 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
 		m.dts = append(m.dts, 0)
 	}
 
-	mv.StartAtom(mov.MoovMvex)
+	mv.StartAtom(iso.MoovMvex)
 	for i := range codecs {
 		mv.WriteTrackExtend(uint32(i + 1))
 	}
@@ -147,7 +163,7 @@ func (m *Muxer) Marshal(trackID byte, packet *rtp.Packet) []byte {
 	}
 	m.pts[trackID] = newTime
 
-	mv := mov.NewMovie(1024 + len(packet.Payload))
+	mv := iso.NewMovie(1024 + len(packet.Payload))
 	mv.WriteMovieFragment(
 		m.fragIndex, uint32(trackID+1), duration,
 		uint32(len(packet.Payload)),

www/README.md

@@ -56,3 +56,5 @@ pc.ontrack = ev => {
 - https://web.dev/i18n/en/fast-playback-with-preload/#manual_buffering
 - https://developer.mozilla.org/en-US/docs/Web/API/Media_Source_Extensions_API
 - https://chromium.googlesource.com/external/w3c/web-platform-tests/+/refs/heads/master/media-source/mediasource-is-type-supported.html
+- https://googlechrome.github.io/samples/media/sourcebuffer-changetype.html
+- https://chromestatus.com/feature/5100845653819392

www/codecs.html

@@ -53,7 +53,7 @@
     const video = document.createElement("video");
     out.innerText += "video.canPlayType\n";
     types.forEach(type => {
-        out.innerText += type + "=" + (video.canPlayType(type) ? "true" : "false") + "\n";
+        out.innerText += `${type} = ${MediaSource.isTypeSupported(type)} / ${video.canPlayType(type)}\n`;
     })
 
 </script>

www/video-rtc.js

@@ -26,8 +26,7 @@ export class VideoRTC extends HTMLElement {
             "hvc1.1.6.L153.B0", // H.265 main 5.1 (Chromecast Ultra)
             "mp4a.40.2",        // AAC LC
             "mp4a.40.5",        // AAC HE
-            "mp4a.69",          // MP3
+            "opus",             // OPUS Chrome
-            "mp4a.6B",          // MP3
         ];
 
         /**