Fix OPUS for HomeKit server #667 #843

internal/ffmpeg/ffmpeg.go

@@ -61,8 +61,9 @@ var defaults = map[string]string{
 	// https://ffmpeg.org/ffmpeg-codecs.html#libopus-1
 	// https://github.com/pion/webrtc/issues/1514
 	// https://ffmpeg.org/ffmpeg-resampler.html
-	// `-async 1` or `-min_comp 0` - force frame_size=960, important for WebRTC audio quality
+	// `-async 1` or `-min_comp 0` - force resampling for static timestamp inc, important for WebRTC audio quality
-	"opus":       "-c:a libopus -application:a lowdelay -frame_duration 20 -min_comp 0",
+	"opus":       "-c:a libopus -application:a lowdelay -min_comp 0",
+	"opus/16000": "-c:a libopus -application:a lowdelay -min_comp 0 -ar:a 16000 -ac:a 1",
 	"pcmu":       "-c:a pcm_mulaw -ar:a 8000 -ac:a 1",
 	"pcmu/8000":  "-c:a pcm_mulaw -ar:a 8000 -ac:a 1",
 	"pcmu/16000": "-c:a pcm_mulaw -ar:a 16000 -ac:a 1",

pkg/hap/README.md

@@ -50,4 +50,5 @@ Requires ffmpeg built with `--enable-libfdk-aac`
 - [Extracting HomeKit Pairing Keys](https://pvieito.com/2019/12/extract-homekit-pairing-keys)
 - [HAP in AirPlay2 receiver](https://github.com/openairplay/airplay2-receiver/blob/master/ap2/pairing/hap.py)
 - [HomeKit Secure Video Unofficial Specification](https://github.com/Supereg/secure-video-specification)
-- [Homebridge Camera FFmpeg](https://sunoo.github.io/homebridge-camera-ffmpeg/configs/)
+- [Homebridge Camera FFmpeg](https://sunoo.github.io/homebridge-camera-ffmpeg/configs/)
+- https://github.com/ljezny/Particle-HAP/blob/master/HAP-Specification-Non-Commercial-Version.pdf

pkg/homekit/consumer.go

@@ -10,6 +10,7 @@ import (
 	"github.com/AlexxIT/go2rtc/pkg/core"
 	"github.com/AlexxIT/go2rtc/pkg/h264"
 	"github.com/AlexxIT/go2rtc/pkg/hap/camera"
+	"github.com/AlexxIT/go2rtc/pkg/opus"
 	"github.com/AlexxIT/go2rtc/pkg/srtp"
 	"github.com/pion/rtp"
 )
@@ -24,6 +25,7 @@ type Consumer struct {
 	sessionID    string
 	videoSession *srtp.Session
 	audioSession *srtp.Session
+	audioRTPTime byte
 }
 
 func NewConsumer(conn net.Conn, server *srtp.Server) *Consumer {
@@ -113,6 +115,7 @@ func (c *Consumer) SetConfig(conf *camera.SelectedStreamConfig) bool {
 	c.audioSession.Remote.SSRC = conf.AudioCodec.RTPParams[0].SSRC
 	c.audioSession.PayloadType = conf.AudioCodec.RTPParams[0].PayloadType
 	c.audioSession.RTCPInterval = toDuration(conf.AudioCodec.RTPParams[0].RTCPInterval)
+	c.audioRTPTime = conf.AudioCodec.CodecParams[0].RTPTime[0]
 
 	c.srtp.AddSession(c.videoSession)
 	c.srtp.AddSession(c.audioSession)
@@ -155,6 +158,8 @@ func (c *Consumer) AddTrack(media *core.Media, codec *core.Codec, track *core.Re
 		} else {
 			sender.Handler = h264.RepairAVCC(track.Codec, sender.Handler)
 		}
+	case core.CodecOpus:
+		sender.Handler = opus.RepackToHAP(c.audioRTPTime, sender.Handler)
 	}
 
 	sender.HandleRTP(track)

pkg/opus/README.md

@@ -0,0 +1,5 @@
+## Useful links
+
+- [RFC 3550: RTP: A Transport Protocol for Real-Time Applications](https://datatracker.ietf.org/doc/html/rfc3550)
+- [RFC 6716: Definition of the Opus Audio Codec](https://datatracker.ietf.org/doc/html/rfc6716)
+- [RFC 7587: RTP Payload Format for the Opus Speech and Audio Codec](https://datatracker.ietf.org/doc/html/rfc7587)

pkg/opus/homekit.go

@@ -0,0 +1,96 @@
+package opus
+
+import (
+	"github.com/AlexxIT/go2rtc/pkg/core"
+	"github.com/pion/rtp"
+)
+
+// Some info about this magic:
+// - Apple has no respect for RFC 7587 standard and using RFC 3550 for RTP timestamps
+// - Apple can request packets with 20ms duration over LAN connection and 60ms over LTE
+// - FFmpeg produce packets with 20ms duration by default and only one frame per packet
+// - FFmpeg should use "-min_comp 0" option, so every packet will be same duration
+// - Apple doesn't care about real sample rate of track
+// - Apple only cares about proper timestamp based on REQUESTED sample rate
+
+// RepackToHAP - convert standart RTP packet with OPUS to HAP packet
+// We expect that:
+// - incoming packet will be 20ms duration and only one frame per packet
+// - outgouing packet will be 20ms or 60ms duration
+// - incoming sample rate will be any (but not very big if we needs 60ms packets for output)
+// - outgouing sample rate will be 16000
+// https://github.com/AlexxIT/go2rtc/issues/667
+func RepackToHAP(rtpTime byte, handler core.HandlerFunc) core.HandlerFunc {
+	switch rtpTime {
+	case 20:
+		return repackToHAP20(handler)
+	case 60:
+		return repackToHAP60(handler)
+	}
+	return handler
+}
+
+// we using only one sample rate in the pkg/hap/camera/accessory.go
+const (
+	timestamp20 = 16000 * 0.020
+	timestamp60 = 16000 * 0.060
+)
+
+// repackToHAP20 - just fix RTP timestamp from RFC 7587 to RFC 3550
+func repackToHAP20(handler core.HandlerFunc) core.HandlerFunc {
+	var timestamp uint32
+
+	return func(pkt *rtp.Packet) {
+		timestamp += timestamp20
+
+		clone := *pkt
+		clone.Timestamp = timestamp
+		handler(&clone)
+	}
+}
+
+// repackToHAP60 - collect 20ms frames to single 60ms packet
+// thanks to @civita idea https://github.com/AlexxIT/go2rtc/pull/843
+func repackToHAP60(handler core.HandlerFunc) core.HandlerFunc {
+	var sequence uint16
+	var timestamp uint32
+
+	var framesCount byte
+	var framesSize []byte
+	var framesData []byte
+
+	return func(pkt *rtp.Packet) {
+		framesData = append(framesData, pkt.Payload[1:]...)
+
+		if framesCount++; framesCount < 3 {
+			if frameSize := len(pkt.Payload) - 1; frameSize >= 252 {
+				b0 := 252 + byte(frameSize)&0b11
+				framesSize = append(framesSize, b0, byte(frameSize/4)-b0)
+			} else {
+				framesSize = append(framesSize, byte(frameSize))
+			}
+			return
+		}
+
+		toc := pkt.Payload[0]
+
+		payload := make([]byte, 2, len(framesSize)+len(framesData))
+		payload[0] = toc | 0b11  // code 3 (multiple frames per packet)
+		payload[1] = 0b1000_0011 // VBR, no padding, 3 frames
+		payload = append(payload, framesSize...)
+		payload = append(payload, framesData...)
+
+		sequence++
+		timestamp += timestamp60
+
+		clone := *pkt
+		clone.Payload = payload
+		clone.SequenceNumber = sequence
+		clone.Timestamp = timestamp
+		handler(&clone)
+
+		framesCount = 0
+		framesSize = framesSize[:0]
+		framesData = framesData[:0]
+	}
+}

pkg/opus/opus.go

@@ -0,0 +1,69 @@
+package opus
+
+import (
+	"time"
+
+	"github.com/AlexxIT/go2rtc/pkg/core"
+	"github.com/pion/rtp"
+	"github.com/rs/zerolog/log"
+)
+
+func Log(handler core.HandlerFunc) core.HandlerFunc {
+	var ts uint32
+
+	return func(pkt *rtp.Packet) {
+		if ts == 0 {
+			ts = pkt.Timestamp
+		}
+
+		toc := pkt.Payload[0]
+		//config := toc >> 3
+		code := toc & 0b11
+
+		frame := parseFrameSize(toc)
+		rate := parseSampleRate(toc)
+
+		log.Printf(
+			"[RTP/OPUS] frame=%s rate=%5d code=%d size=%6d ts=%10d dt=%5d pt=%2d ssrc=%d seq=%d mark=%t",
+			frame, rate, code, len(pkt.Payload), pkt.Timestamp, pkt.Timestamp-ts, pkt.PayloadType, pkt.SSRC, pkt.SequenceNumber, pkt.Marker,
+		)
+
+		ts = pkt.Timestamp
+
+		handler(pkt)
+	}
+}
+
+func parseFrameSize(toc byte) time.Duration {
+	switch toc >> 3 {
+	case 0, 4, 8, 12, 14, 18, 22, 26, 30:
+		return 10_000_000
+	case 1, 5, 9, 13, 15, 19, 23, 27, 31:
+		return 20_000_000
+	case 2, 6, 10:
+		return 40_000_000
+	case 3, 7, 11:
+		return 60_000_000
+	case 16, 20, 24, 28:
+		return 2_500_000
+	case 17, 21, 25, 29:
+		return 5_000_000
+	}
+	return 0
+}
+
+func parseSampleRate(toc byte) uint16 {
+	switch toc >> 3 {
+	case 0, 1, 2, 3, 16, 17, 18, 19:
+		return 8000
+	case 4, 5, 6, 7:
+		return 12000
+	case 8, 9, 10, 11, 20, 21, 22, 23:
+		return 16000
+	case 12, 13, 24, 25, 26, 27:
+		return 24000
+	case 14, 15, 28, 29, 30, 31:
+		return 48000
+	}
+	return 0
+}