diff --git a/internal/ffmpeg/ffmpeg.go b/internal/ffmpeg/ffmpeg.go index 25101c59..5d94d3c1 100644 --- a/internal/ffmpeg/ffmpeg.go +++ b/internal/ffmpeg/ffmpeg.go @@ -61,8 +61,9 @@ var defaults = map[string]string{ // https://ffmpeg.org/ffmpeg-codecs.html#libopus-1 // https://github.com/pion/webrtc/issues/1514 // https://ffmpeg.org/ffmpeg-resampler.html - // `-async 1` or `-min_comp 0` - force frame_size=960, important for WebRTC audio quality - "opus": "-c:a libopus -application:a lowdelay -frame_duration 20 -min_comp 0", + // `-async 1` or `-min_comp 0` - force resampling for static timestamp inc, important for WebRTC audio quality + "opus": "-c:a libopus -application:a lowdelay -min_comp 0", + "opus/16000": "-c:a libopus -application:a lowdelay -min_comp 0 -ar:a 16000 -ac:a 1", "pcmu": "-c:a pcm_mulaw -ar:a 8000 -ac:a 1", "pcmu/8000": "-c:a pcm_mulaw -ar:a 8000 -ac:a 1", "pcmu/16000": "-c:a pcm_mulaw -ar:a 16000 -ac:a 1", diff --git a/pkg/hap/README.md b/pkg/hap/README.md index 33341fea..9d7fbf5b 100644 --- a/pkg/hap/README.md +++ b/pkg/hap/README.md @@ -50,4 +50,5 @@ Requires ffmpeg built with `--enable-libfdk-aac` - [Extracting HomeKit Pairing Keys](https://pvieito.com/2019/12/extract-homekit-pairing-keys) - [HAP in AirPlay2 receiver](https://github.com/openairplay/airplay2-receiver/blob/master/ap2/pairing/hap.py) - [HomeKit Secure Video Unofficial Specification](https://github.com/Supereg/secure-video-specification) -- [Homebridge Camera FFmpeg](https://sunoo.github.io/homebridge-camera-ffmpeg/configs/) \ No newline at end of file +- [Homebridge Camera FFmpeg](https://sunoo.github.io/homebridge-camera-ffmpeg/configs/) +- https://github.com/ljezny/Particle-HAP/blob/master/HAP-Specification-Non-Commercial-Version.pdf \ No newline at end of file diff --git a/pkg/homekit/consumer.go b/pkg/homekit/consumer.go index 88b977b9..1e04fedf 100644 --- a/pkg/homekit/consumer.go +++ b/pkg/homekit/consumer.go @@ -10,6 +10,7 @@ import ( "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/h264" "github.com/AlexxIT/go2rtc/pkg/hap/camera" + "github.com/AlexxIT/go2rtc/pkg/opus" "github.com/AlexxIT/go2rtc/pkg/srtp" "github.com/pion/rtp" ) @@ -24,6 +25,7 @@ type Consumer struct { sessionID string videoSession *srtp.Session audioSession *srtp.Session + audioRTPTime byte } func NewConsumer(conn net.Conn, server *srtp.Server) *Consumer { @@ -113,6 +115,7 @@ func (c *Consumer) SetConfig(conf *camera.SelectedStreamConfig) bool { c.audioSession.Remote.SSRC = conf.AudioCodec.RTPParams[0].SSRC c.audioSession.PayloadType = conf.AudioCodec.RTPParams[0].PayloadType c.audioSession.RTCPInterval = toDuration(conf.AudioCodec.RTPParams[0].RTCPInterval) + c.audioRTPTime = conf.AudioCodec.CodecParams[0].RTPTime[0] c.srtp.AddSession(c.videoSession) c.srtp.AddSession(c.audioSession) @@ -155,6 +158,8 @@ func (c *Consumer) AddTrack(media *core.Media, codec *core.Codec, track *core.Re } else { sender.Handler = h264.RepairAVCC(track.Codec, sender.Handler) } + case core.CodecOpus: + sender.Handler = opus.RepackToHAP(c.audioRTPTime, sender.Handler) } sender.HandleRTP(track) diff --git a/pkg/opus/README.md b/pkg/opus/README.md new file mode 100644 index 00000000..15e3cc86 --- /dev/null +++ b/pkg/opus/README.md @@ -0,0 +1,5 @@ +## Useful links + +- [RFC 3550: RTP: A Transport Protocol for Real-Time Applications](https://datatracker.ietf.org/doc/html/rfc3550) +- [RFC 6716: Definition of the Opus Audio Codec](https://datatracker.ietf.org/doc/html/rfc6716) +- [RFC 7587: RTP Payload Format for the Opus Speech and Audio Codec](https://datatracker.ietf.org/doc/html/rfc7587) diff --git a/pkg/opus/homekit.go b/pkg/opus/homekit.go new file mode 100644 index 00000000..1199464d --- /dev/null +++ b/pkg/opus/homekit.go @@ -0,0 +1,96 @@ +package opus + +import ( + "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/pion/rtp" +) + +// Some info about this magic: +// - Apple has no respect for RFC 7587 standard and using RFC 3550 for RTP timestamps +// - Apple can request packets with 20ms duration over LAN connection and 60ms over LTE +// - FFmpeg produce packets with 20ms duration by default and only one frame per packet +// - FFmpeg should use "-min_comp 0" option, so every packet will be same duration +// - Apple doesn't care about real sample rate of track +// - Apple only cares about proper timestamp based on REQUESTED sample rate + +// RepackToHAP - convert standart RTP packet with OPUS to HAP packet +// We expect that: +// - incoming packet will be 20ms duration and only one frame per packet +// - outgouing packet will be 20ms or 60ms duration +// - incoming sample rate will be any (but not very big if we needs 60ms packets for output) +// - outgouing sample rate will be 16000 +// https://github.com/AlexxIT/go2rtc/issues/667 +func RepackToHAP(rtpTime byte, handler core.HandlerFunc) core.HandlerFunc { + switch rtpTime { + case 20: + return repackToHAP20(handler) + case 60: + return repackToHAP60(handler) + } + return handler +} + +// we using only one sample rate in the pkg/hap/camera/accessory.go +const ( + timestamp20 = 16000 * 0.020 + timestamp60 = 16000 * 0.060 +) + +// repackToHAP20 - just fix RTP timestamp from RFC 7587 to RFC 3550 +func repackToHAP20(handler core.HandlerFunc) core.HandlerFunc { + var timestamp uint32 + + return func(pkt *rtp.Packet) { + timestamp += timestamp20 + + clone := *pkt + clone.Timestamp = timestamp + handler(&clone) + } +} + +// repackToHAP60 - collect 20ms frames to single 60ms packet +// thanks to @civita idea https://github.com/AlexxIT/go2rtc/pull/843 +func repackToHAP60(handler core.HandlerFunc) core.HandlerFunc { + var sequence uint16 + var timestamp uint32 + + var framesCount byte + var framesSize []byte + var framesData []byte + + return func(pkt *rtp.Packet) { + framesData = append(framesData, pkt.Payload[1:]...) + + if framesCount++; framesCount < 3 { + if frameSize := len(pkt.Payload) - 1; frameSize >= 252 { + b0 := 252 + byte(frameSize)&0b11 + framesSize = append(framesSize, b0, byte(frameSize/4)-b0) + } else { + framesSize = append(framesSize, byte(frameSize)) + } + return + } + + toc := pkt.Payload[0] + + payload := make([]byte, 2, len(framesSize)+len(framesData)) + payload[0] = toc | 0b11 // code 3 (multiple frames per packet) + payload[1] = 0b1000_0011 // VBR, no padding, 3 frames + payload = append(payload, framesSize...) + payload = append(payload, framesData...) + + sequence++ + timestamp += timestamp60 + + clone := *pkt + clone.Payload = payload + clone.SequenceNumber = sequence + clone.Timestamp = timestamp + handler(&clone) + + framesCount = 0 + framesSize = framesSize[:0] + framesData = framesData[:0] + } +} diff --git a/pkg/opus/opus.go b/pkg/opus/opus.go new file mode 100644 index 00000000..9fe1d8b6 --- /dev/null +++ b/pkg/opus/opus.go @@ -0,0 +1,69 @@ +package opus + +import ( + "time" + + "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/pion/rtp" + "github.com/rs/zerolog/log" +) + +func Log(handler core.HandlerFunc) core.HandlerFunc { + var ts uint32 + + return func(pkt *rtp.Packet) { + if ts == 0 { + ts = pkt.Timestamp + } + + toc := pkt.Payload[0] + //config := toc >> 3 + code := toc & 0b11 + + frame := parseFrameSize(toc) + rate := parseSampleRate(toc) + + log.Printf( + "[RTP/OPUS] frame=%s rate=%5d code=%d size=%6d ts=%10d dt=%5d pt=%2d ssrc=%d seq=%d mark=%t", + frame, rate, code, len(pkt.Payload), pkt.Timestamp, pkt.Timestamp-ts, pkt.PayloadType, pkt.SSRC, pkt.SequenceNumber, pkt.Marker, + ) + + ts = pkt.Timestamp + + handler(pkt) + } +} + +func parseFrameSize(toc byte) time.Duration { + switch toc >> 3 { + case 0, 4, 8, 12, 14, 18, 22, 26, 30: + return 10_000_000 + case 1, 5, 9, 13, 15, 19, 23, 27, 31: + return 20_000_000 + case 2, 6, 10: + return 40_000_000 + case 3, 7, 11: + return 60_000_000 + case 16, 20, 24, 28: + return 2_500_000 + case 17, 21, 25, 29: + return 5_000_000 + } + return 0 +} + +func parseSampleRate(toc byte) uint16 { + switch toc >> 3 { + case 0, 1, 2, 3, 16, 17, 18, 19: + return 8000 + case 4, 5, 6, 7: + return 12000 + case 8, 9, 10, 11, 20, 21, 22, 23: + return 16000 + case 12, 13, 24, 25, 26, 27: + return 24000 + case 14, 15, 28, 29, 30, 31: + return 48000 + } + return 0 +}