From f8bc25d0ae5f383f9b2fcf7cb48c18aeee6038e2 Mon Sep 17 00:00:00 2001 From: Alex X Date: Sat, 25 May 2024 08:22:38 +0300 Subject: [PATCH] Add support rawvideo format --- internal/ffmpeg/ffmpeg.go | 16 ++- pkg/core/core.go | 1 + pkg/core/helpers.go | 7 ++ pkg/magic/producer.go | 4 + pkg/mjpeg/consumer.go | 3 + pkg/mjpeg/helpers.go | 21 ++++ pkg/y4m/y4m.go | 199 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 pkg/y4m/y4m.go diff --git a/internal/ffmpeg/ffmpeg.go b/internal/ffmpeg/ffmpeg.go index 26f9880f..2b24c3ce 100644 --- a/internal/ffmpeg/ffmpeg.go +++ b/internal/ffmpeg/ffmpeg.go @@ -12,6 +12,7 @@ import ( "github.com/AlexxIT/go2rtc/internal/ffmpeg/virtual" "github.com/AlexxIT/go2rtc/internal/rtsp" "github.com/AlexxIT/go2rtc/internal/streams" + "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/ffmpeg" ) @@ -61,6 +62,7 @@ var defaults = map[string]string{ // output "output": "-user_agent ffmpeg/go2rtc -rtsp_transport tcp -f rtsp {output}", "output/mjpeg": "-f mjpeg -", + "output/raw": "-f yuv4mpegpipe -", "output/aac": "-f adts -", "output/wav": "-f wav -", @@ -73,6 +75,12 @@ var defaults = map[string]string{ "mjpeg": "-c:v mjpeg", //"mjpeg": "-c:v mjpeg -force_duplicated_matrix:v 1 -huffman:v 0 -pix_fmt:v yuvj420p", + "raw": "-c:v rawvideo", + "raw/gray8": "-c:v rawvideo -pix_fmt:v gray8", + "raw/yuv420p": "-c:v rawvideo -pix_fmt:v yuv420p", + "raw/yuv422p": "-c:v rawvideo -pix_fmt:v yuv422p", + "raw/yuv444p": "-c:v rawvideo -pix_fmt:v yuv444p", + // https://ffmpeg.org/ffmpeg-codecs.html#libopus-1 // https://github.com/pion/webrtc/issues/1514 // https://ffmpeg.org/ffmpeg-resampler.html @@ -336,12 +344,14 @@ func parseArgs(s string) *ffmpeg.Args { args.Output = defaults["output/mjpeg"] } case args.Video == 1 && args.Audio == 0: - if query.Get("video") == "mjpeg" { + switch core.Before(query.Get("video"), "/") { + case "mjpeg": args.Output = defaults["output/mjpeg"] + case "raw": + args.Output = defaults["output/raw"] } case args.Video == 0 && args.Audio == 1: - codec, _, _ := strings.Cut(query.Get("audio"), "/") - switch codec { + switch core.Before(query.Get("audio"), "/") { case "aac": args.Output = defaults["output/aac"] case "pcma", "pcmu", "pcml": diff --git a/pkg/core/core.go b/pkg/core/core.go index 146533e3..bc855ccc 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -18,6 +18,7 @@ const ( CodecVP9 = "VP9" CodecAV1 = "AV1" CodecJPEG = "JPEG" // payloadType: 26 + CodecRAW = "RAW" CodecPCMU = "PCMU" // payloadType: 0 CodecPCMA = "PCMA" // payloadType: 8 diff --git a/pkg/core/helpers.go b/pkg/core/helpers.go index 0c367e2c..72afe897 100644 --- a/pkg/core/helpers.go +++ b/pkg/core/helpers.go @@ -38,6 +38,13 @@ func RandString(size, base byte) string { return string(b) } +func Before(s, sep string) string { + if i := strings.Index(s, sep); i > 0 { + return s[:i] + } + return s +} + func Between(s, sub1, sub2 string) string { i := strings.Index(s, sub1) if i < 0 { diff --git a/pkg/magic/producer.go b/pkg/magic/producer.go index c49fe8bf..9bde508d 100644 --- a/pkg/magic/producer.go +++ b/pkg/magic/producer.go @@ -15,6 +15,7 @@ import ( "github.com/AlexxIT/go2rtc/pkg/mpegts" "github.com/AlexxIT/go2rtc/pkg/multipart" "github.com/AlexxIT/go2rtc/pkg/wav" + "github.com/AlexxIT/go2rtc/pkg/y4m" ) func Open(r io.Reader) (core.Producer, error) { @@ -32,6 +33,9 @@ func Open(r io.Reader) (core.Producer, error) { case string(b) == wav.FourCC: return wav.Open(rd) + case string(b) == y4m.FourCC: + return y4m.Open(rd) + case bytes.HasPrefix(b, []byte{0xFF, 0xD8}): return mjpeg.Open(rd) diff --git a/pkg/mjpeg/consumer.go b/pkg/mjpeg/consumer.go index 444cbdcc..d5fb0d51 100644 --- a/pkg/mjpeg/consumer.go +++ b/pkg/mjpeg/consumer.go @@ -22,6 +22,7 @@ func NewConsumer() *Consumer { Direction: core.DirectionSendonly, Codecs: []*core.Codec{ {Name: core.CodecJPEG}, + {Name: core.CodecRAW}, }, }, }, @@ -40,6 +41,8 @@ func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiv if track.Codec.IsRTP() { sender.Handler = RTPDepay(sender.Handler) + } else if track.Codec.Name == core.CodecRAW { + sender.Handler = Encoder(track.Codec, sender.Handler) } sender.HandleRTP(track) diff --git a/pkg/mjpeg/helpers.go b/pkg/mjpeg/helpers.go index 21000b9b..08b4408b 100644 --- a/pkg/mjpeg/helpers.go +++ b/pkg/mjpeg/helpers.go @@ -3,6 +3,10 @@ package mjpeg import ( "bytes" "image/jpeg" + + "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/y4m" + "github.com/pion/rtp" ) // FixJPEG - reencode JPEG if it has wrong header @@ -33,3 +37,20 @@ func FixJPEG(b []byte) []byte { } return buf.Bytes() } + +func Encoder(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc { + newImage := y4m.NewImage(codec.FmtpLine) + + return func(packet *rtp.Packet) { + img := newImage(packet.Payload) + + buf := bytes.NewBuffer(nil) + if err := jpeg.Encode(buf, img, nil); err != nil { + return + } + + clone := *packet + clone.Payload = buf.Bytes() + handler(&clone) + } +} diff --git a/pkg/y4m/y4m.go b/pkg/y4m/y4m.go new file mode 100644 index 00000000..6caef1a0 --- /dev/null +++ b/pkg/y4m/y4m.go @@ -0,0 +1,199 @@ +package y4m + +import ( + "bufio" + "bytes" + "errors" + "image" + "io" + + "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/pion/rtp" +) + +const FourCC = "YUV4" + +func Open(r io.Reader) (*Producer, error) { + rd := bufio.NewReaderSize(r, core.BufferSize) + b, err := rd.ReadBytes('\n') + if err != nil { + return nil, err + } + + b = b[:len(b)-1] // remove \n + + sdp := string(b) + var fmtp string + + for b != nil { + // YUV4MPEG2 W1280 H720 F24:1 Ip A1:1 C420mpeg2 XYSCSS=420MPEG2 + // https://manned.org/yuv4mpeg.5 + // https://github.com/FFmpeg/FFmpeg/blob/master/libavformat/yuv4mpegenc.c + key := b[0] + var value string + if i := bytes.IndexByte(b, ' '); i > 0 { + value = string(b[1:i]) + b = b[i+1:] + } else { + value = string(b[1:]) + b = nil + } + + switch key { + case 'W': + fmtp = "width=" + value + case 'H': + fmtp += ";height=" + value + case 'C': + fmtp += ";colorspace=" + value + } + } + + if GetSize(fmtp) == 0 { + return nil, errors.New("y4m: unsupported format: " + sdp) + } + + prod := &Producer{rd: rd, cl: r.(io.Closer)} + prod.Type = "YUV4MPEG2 producer" + prod.SDP = sdp + prod.Medias = []*core.Media{ + { + Kind: core.KindVideo, + Direction: core.DirectionRecvonly, + Codecs: []*core.Codec{ + { + Name: core.CodecRAW, + ClockRate: 90000, + FmtpLine: fmtp, + PayloadType: core.PayloadTypeRAW, + }, + }, + }, + } + + return prod, nil +} + +type Producer struct { + core.SuperProducer + rd *bufio.Reader + cl io.Closer +} + +func (c *Producer) Start() error { + size := GetSize(c.Medias[0].Codecs[0].FmtpLine) + + for { + // FRAME\n + if _, err := c.rd.Discard(6); err != nil { + return err + } + + frame := make([]byte, size) + if _, err := io.ReadFull(c.rd, frame); err != nil { + return err + } + + c.Recv += size + + if len(c.Receivers) == 0 { + continue + } + + pkt := &rtp.Packet{ + Header: rtp.Header{Timestamp: core.Now90000()}, + Payload: frame, + } + c.Receivers[0].WriteRTP(pkt) + } +} + +func (c *Producer) Stop() error { + _ = c.SuperProducer.Close() + return c.cl.Close() +} + +func GetSize(fmtp string) int { + w := core.Atoi(core.Between(fmtp, "width=", ";")) + h := core.Atoi(core.Between(fmtp, "height=", ";")) + + switch core.Between(fmtp, "colorspace=", ";") { + case "mono": + return w * h + case "420mpeg2", "420jpeg": + return w * h * 3 / 2 + case "422": + return w * h * 2 + case "444": + return w * h * 3 + } + + return 0 +} + +func NewImage(fmtp string) func(frame []byte) image.Image { + w := core.Atoi(core.Between(fmtp, "width=", ";")) + h := core.Atoi(core.Between(fmtp, "height=", ";")) + rect := image.Rect(0, 0, w, h) + + switch core.Between(fmtp, "colorspace=", ";") { + case "mono": + return func(frame []byte) image.Image { + return &image.Gray{ + Pix: frame, + Stride: w, + Rect: rect, + } + } + case "420mpeg2", "420jpeg": + i1 := w * h + i2 := i1 + i1/4 + i3 := i2 + i1/4 + + return func(frame []byte) image.Image { + return &image.YCbCr{ + Y: frame[:i1], + Cb: frame[i1:i2], + Cr: frame[i2:i3], + YStride: w, + CStride: w / 2, + SubsampleRatio: image.YCbCrSubsampleRatio420, + Rect: rect, + } + } + case "422": + i1 := w * h + i2 := i1 + i1/2 + i3 := i2 + i1/2 + + return func(frame []byte) image.Image { + return &image.YCbCr{ + Y: frame[:i1], + Cb: frame[i1:i2], + Cr: frame[i2:i3], + YStride: w, + CStride: w / 2, + SubsampleRatio: image.YCbCrSubsampleRatio422, + Rect: rect, + } + } + case "444": + i1 := w * h + i2 := i1 + i1 + i3 := i2 + i1 + + return func(frame []byte) image.Image { + return &image.YCbCr{ + Y: frame[:i1], + Cb: frame[i1:i2], + Cr: frame[i2:i3], + YStride: w, + CStride: w, + SubsampleRatio: image.YCbCrSubsampleRatio444, + Rect: rect, + } + } + } + + return nil +}