From c60767c8b03b16da4298176e53dc83abb6e342d4 Mon Sep 17 00:00:00 2001 From: Alex X Date: Sun, 31 Dec 2023 21:06:41 +0300 Subject: [PATCH] Add support H265 to FLV source #822 --- pkg/flv/producer.go | 79 +++++++++++++++++++++++++++++++++++++-------- pkg/h265/avcc.go | 18 +++++++++++ pkg/h265/mpeg4.go | 60 +++++++++++++++++++++++++++++++++- pkg/mp4/consumer.go | 2 ++ 4 files changed, 145 insertions(+), 14 deletions(-) diff --git a/pkg/flv/producer.go b/pkg/flv/producer.go index 5155e17a..3972e666 100644 --- a/pkg/flv/producer.go +++ b/pkg/flv/producer.go @@ -10,6 +10,7 @@ import ( "github.com/AlexxIT/go2rtc/pkg/aac" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/h264" + "github.com/AlexxIT/go2rtc/pkg/h265" "github.com/pion/rtp" ) @@ -40,6 +41,21 @@ const ( CodecAVC = 7 ) +const ( + PacketTypeAVCHeader = iota + PacketTypeAVCNALU + PacketTypeAVCEnd +) + +const ( + PacketTypeSequenceStart = iota + PacketTypeCodedFrames + PacketTypeSequenceEnd + PacketTypeCodedFramesX + PacketTypeMetadata + PacketTypeMPEG2TSSequenceStart +) + func (c *Producer) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, error) { receiver, _ := c.SuperProducer.GetTrack(media, codec) if media.Kind == core.KindVideo { @@ -70,13 +86,32 @@ func (c *Producer) Start() error { c.audio.WriteRTP(pkt) case TagVideo: - // frame type 4b, codecID 4b, avc packet type 8b, composition time 24b - if c.video == nil || pkt.Payload[1] == 0 { + if c.video == nil { continue } + if isExHeader(pkt.Payload) { + switch packetType := pkt.Payload[0] & 0b1111; packetType { + case PacketTypeCodedFrames: + // frame type 4b, packet type 4b, fourCC 32b, composition time 24b + pkt.Payload = pkt.Payload[8:] + case PacketTypeCodedFramesX: + // frame type 4b, packet type 4b, fourCC 32b + pkt.Payload = pkt.Payload[5:] + default: + continue + } + } else { + switch pkt.Payload[1] { + case PacketTypeAVCNALU: + // frame type 4b, codecID 4b, avc packet type 8b, composition time 24b + pkt.Payload = pkt.Payload[5:] + default: + continue + } + } + pkt.Timestamp = TimeToRTP(pkt.Timestamp, c.video.Codec.ClockRate) - pkt.Payload = pkt.Payload[5:] c.video.WriteRTP(pkt) } } @@ -145,20 +180,32 @@ func (c *Producer) probe() error { c.Medias = append(c.Medias, media) case TagVideo: - _ = pkt.Payload[1] // bounds + var codec *core.Codec - _ = pkt.Payload[0] >> 4 // FrameType - codecID := pkt.Payload[0] & 0b1111 // CodecID + if isExHeader(pkt.Payload) { + if string(pkt.Payload[1:5]) != "hvc1" { + continue + } - if codecID != CodecAVC { - continue + if packetType := pkt.Payload[0] & 0b1111; packetType != PacketTypeSequenceStart { + continue + } + + codec = h265.ConfigToCodec(pkt.Payload[5:]) + } else { + _ = pkt.Payload[0] >> 4 // FrameType + + if codecID := pkt.Payload[0] & 0b1111; codecID != CodecAVC { + continue + } + + if packetType := pkt.Payload[1]; packetType != PacketTypeAVCHeader { // check if header + continue + } + + codec = h264.ConfigToCodec(pkt.Payload[5:]) } - if pkt.Payload[1] != 0 { // check if header - continue - } - - codec := h264.ConfigToCodec(pkt.Payload[5:]) media := &core.Media{ Kind: core.KindVideo, Direction: core.DirectionRecvonly, @@ -229,9 +276,15 @@ func (c *Producer) readPacket() (*rtp.Packet, error) { return nil, err } + //log.Printf("[FLV] %d %.40x", pkt.PayloadType, pkt.Payload) + return pkt, nil } func TimeToRTP(timeMS uint32, clockRate uint32) uint32 { return timeMS * clockRate / 1000 } + +func isExHeader(data []byte) bool { + return data[0]&0b1000_0000 != 0 +} diff --git a/pkg/h265/avcc.go b/pkg/h265/avcc.go index 3c3f0e41..9c257a19 100644 --- a/pkg/h265/avcc.go +++ b/pkg/h265/avcc.go @@ -7,8 +7,26 @@ import ( "encoding/binary" "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/h264" + "github.com/pion/rtp" ) +func RepairAVCC(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc { + vds, sps, pps := GetParameterSet(codec.FmtpLine) + ps := h264.JoinNALU(vds, sps, pps) + + return func(packet *rtp.Packet) { + switch NALUType(packet.Payload) { + case NALUTypeIFrame, NALUTypeIFrame2, NALUTypeIFrame3: + clone := *packet + clone.Payload = h264.Join(ps, packet.Payload) + handler(&clone) + default: + handler(packet) + } + } +} + func AVCCToCodec(avcc []byte) *core.Codec { buf := bytes.NewBufferString("profile-id=1") diff --git a/pkg/h265/mpeg4.go b/pkg/h265/mpeg4.go index ec78b17d..e06d0836 100644 --- a/pkg/h265/mpeg4.go +++ b/pkg/h265/mpeg4.go @@ -1,7 +1,40 @@ // Package h265 - MPEG4 format related functions package h265 -import "encoding/binary" +import ( + "bytes" + "encoding/base64" + "encoding/binary" + + "github.com/AlexxIT/go2rtc/pkg/core" +) + +func DecodeConfig(conf []byte) (profile, vps, sps, pps []byte) { + profile = conf[1:4] + + b := conf[23:] + if binary.BigEndian.Uint16(b[1:]) != 1 { + return + } + vpsSize := binary.BigEndian.Uint16(b[3:]) + vps = b[5 : 5+vpsSize] + + b = conf[23+5+vpsSize:] + if binary.BigEndian.Uint16(b[1:]) != 1 { + return + } + spsSize := binary.BigEndian.Uint16(b[3:]) + sps = b[5 : 5+spsSize] + + b = conf[23+5+vpsSize+5+spsSize:] + if binary.BigEndian.Uint16(b[1:]) != 1 { + return + } + ppsSize := binary.BigEndian.Uint16(b[3:]) + pps = b[5 : 5+ppsSize] + + return +} func EncodeConfig(vps, sps, pps []byte) []byte { vpsSize := uint16(len(vps)) @@ -38,3 +71,28 @@ func EncodeConfig(vps, sps, pps []byte) []byte { return buf } + +func ConfigToCodec(conf []byte) *core.Codec { + buf := bytes.NewBufferString("profile-id=1") + + _, vps, sps, pps := DecodeConfig(conf) + if vps != nil { + buf.WriteString(";sprop-vps=") + buf.WriteString(base64.StdEncoding.EncodeToString(vps)) + } + if sps != nil { + buf.WriteString(";sprop-sps=") + buf.WriteString(base64.StdEncoding.EncodeToString(sps)) + } + if pps != nil { + buf.WriteString(";sprop-pps=") + buf.WriteString(base64.StdEncoding.EncodeToString(pps)) + } + + return &core.Codec{ + Name: core.CodecH265, + ClockRate: 90000, + FmtpLine: buf.String(), + PayloadType: core.PayloadTypeRAW, + } +} diff --git a/pkg/mp4/consumer.go b/pkg/mp4/consumer.go index 8d0beef9..83b2d2e3 100644 --- a/pkg/mp4/consumer.go +++ b/pkg/mp4/consumer.go @@ -106,6 +106,8 @@ func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiv if track.Codec.IsRTP() { handler.Handler = h265.RTPDepay(track.Codec, handler.Handler) + } else { + handler.Handler = h265.RepairAVCC(track.Codec, handler.Handler) } default: