feat(webp): add WebP streaming and snapshot APIs
- implement WebP streaming with multipart support - add snapshot endpoint for WebP format with quality options - introduce WebP encoding using pure Go library without FFmpeg - update documentation and links for new WebP features
This commit is contained in:
Sergey Krashevich
@@ -0,0 +1,61 @@
|
||||
package webp
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/AlexxIT/go2rtc/pkg/core"
|
||||
"github.com/pion/rtp"
|
||||
)
|
||||
|
||||
type Consumer struct {
|
||||
core.Connection
|
||||
wr *core.WriteBuffer
|
||||
}
|
||||
|
||||
func NewConsumer() *Consumer {
|
||||
medias := []*core.Media{
|
||||
{
|
||||
Kind: core.KindVideo,
|
||||
Direction: core.DirectionSendonly,
|
||||
Codecs: []*core.Codec{
|
||||
{Name: core.CodecJPEG},
|
||||
{Name: core.CodecRAW},
|
||||
},
|
||||
},
|
||||
}
|
||||
wr := core.NewWriteBuffer(nil)
|
||||
return &Consumer{
|
||||
Connection: core.Connection{
|
||||
ID: core.NewID(),
|
||||
FormatName: "webp",
|
||||
Medias: medias,
|
||||
Transport: wr,
|
||||
},
|
||||
wr: wr,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiver) error {
|
||||
sender := core.NewSender(media, track.Codec)
|
||||
sender.Handler = func(packet *rtp.Packet) {
|
||||
if n, err := c.wr.Write(packet.Payload); err == nil {
|
||||
c.Send += n
|
||||
}
|
||||
}
|
||||
|
||||
if track.Codec.IsRTP() {
|
||||
sender.Handler = RTPDepay(sender.Handler)
|
||||
} else if track.Codec.Name == core.CodecRAW {
|
||||
sender.Handler = Encoder(track.Codec, sender.Handler)
|
||||
} else if track.Codec.Name == core.CodecJPEG {
|
||||
sender.Handler = JPEGToWebP(sender.Handler)
|
||||
}
|
||||
|
||||
sender.HandleRTP(track)
|
||||
c.Senders = append(c.Senders, sender)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumer) WriteTo(wr io.Writer) (int64, error) {
|
||||
return c.wr.WriteTo(wr)
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package webp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
|
||||
webplib "github.com/skrashevich/go-webp"
|
||||
|
||||
"github.com/AlexxIT/go2rtc/pkg/core"
|
||||
"github.com/AlexxIT/go2rtc/pkg/mjpeg"
|
||||
"github.com/AlexxIT/go2rtc/pkg/y4m"
|
||||
"github.com/pion/rtp"
|
||||
)
|
||||
|
||||
// EncodeImage encodes any image.Image to WebP lossy bytes.
|
||||
func EncodeImage(img image.Image, quality int) ([]byte, error) {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
if err := webplib.Encode(buf, img, &webplib.Options{Lossy: true, Quality: float32(quality)}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// EncodeLossless encodes image.Image to WebP lossless bytes.
|
||||
func EncodeLossless(img image.Image) ([]byte, error) {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
if err := webplib.Encode(buf, img, &webplib.Options{Lossy: false}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// EncodeJPEG converts JPEG bytes to WebP lossy bytes.
|
||||
func EncodeJPEG(jpegData []byte, quality int) ([]byte, error) {
|
||||
img, err := jpeg.Decode(bytes.NewReader(jpegData))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return EncodeImage(img, quality)
|
||||
}
|
||||
|
||||
// Decode decodes WebP bytes to image.Image.
|
||||
func Decode(data []byte) (image.Image, error) {
|
||||
return webplib.Decode(bytes.NewReader(data))
|
||||
}
|
||||
|
||||
// FixJPEGToWebP is like mjpeg.FixJPEG but outputs WebP. Handles AVI1 MJPEG frames.
|
||||
func FixJPEGToWebP(jpegData []byte, quality int) ([]byte, error) {
|
||||
fixed := mjpeg.FixJPEG(jpegData)
|
||||
return EncodeJPEG(fixed, quality)
|
||||
}
|
||||
|
||||
// Encoder converts a RAW YUV frame to WebP.
|
||||
func Encoder(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc {
|
||||
newImage := y4m.NewImage(codec.FmtpLine)
|
||||
|
||||
return func(packet *rtp.Packet) {
|
||||
img := newImage(packet.Payload)
|
||||
|
||||
b, err := EncodeImage(img, 75)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
clone := *packet
|
||||
clone.Payload = b
|
||||
handler(&clone)
|
||||
}
|
||||
}
|
||||
|
||||
// JPEGToWebP converts a JPEG frame packet to WebP.
|
||||
func JPEGToWebP(handler core.HandlerFunc) core.HandlerFunc {
|
||||
return func(packet *rtp.Packet) {
|
||||
b, err := EncodeJPEG(packet.Payload, 75)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
clone := *packet
|
||||
clone.Payload = b
|
||||
handler(&clone)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package webp
|
||||
|
||||
import (
|
||||
"github.com/AlexxIT/go2rtc/pkg/core"
|
||||
"github.com/AlexxIT/go2rtc/pkg/mjpeg"
|
||||
)
|
||||
|
||||
// RTPDepay depayloads RTP/JPEG packets and converts the resulting JPEG frame to WebP.
|
||||
func RTPDepay(handler core.HandlerFunc) core.HandlerFunc {
|
||||
return mjpeg.RTPDepay(JPEGToWebP(handler))
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package webp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"testing"
|
||||
|
||||
"github.com/AlexxIT/go2rtc/pkg/core"
|
||||
)
|
||||
|
||||
func newTestImage(w, h int) *image.NRGBA {
|
||||
img := image.NewNRGBA(image.Rect(0, 0, w, h))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
img.SetNRGBA(x, y, color.NRGBA{R: uint8(x % 256), G: uint8(y % 256), B: 128, A: 255})
|
||||
}
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
func isWebP(data []byte) bool {
|
||||
return len(data) >= 12 &&
|
||||
bytes.Equal(data[0:4], []byte("RIFF")) &&
|
||||
bytes.Equal(data[8:12], []byte("WEBP"))
|
||||
}
|
||||
|
||||
func TestEncodeImage(t *testing.T) {
|
||||
img := newTestImage(100, 100)
|
||||
data, err := EncodeImage(img, 75)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeImage error: %v", err)
|
||||
}
|
||||
if !isWebP(data) {
|
||||
t.Fatalf("output is not valid WebP: got prefix %q", data[:min(12, len(data))])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeJPEG(t *testing.T) {
|
||||
img := newTestImage(100, 100)
|
||||
var jpegBuf bytes.Buffer
|
||||
if err := jpeg.Encode(&jpegBuf, img, &jpeg.Options{Quality: 90}); err != nil {
|
||||
t.Fatalf("jpeg.Encode error: %v", err)
|
||||
}
|
||||
data, err := EncodeJPEG(jpegBuf.Bytes(), 75)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeJPEG error: %v", err)
|
||||
}
|
||||
if !isWebP(data) {
|
||||
t.Fatalf("output is not valid WebP: got prefix %q", data[:min(12, len(data))])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecode(t *testing.T) {
|
||||
img := newTestImage(100, 80)
|
||||
data, err := EncodeImage(img, 80)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeImage error: %v", err)
|
||||
}
|
||||
decoded, err := Decode(data)
|
||||
if err != nil {
|
||||
t.Fatalf("Decode error: %v", err)
|
||||
}
|
||||
bounds := decoded.Bounds()
|
||||
if bounds.Dx() != 100 || bounds.Dy() != 80 {
|
||||
t.Fatalf("expected 100x80, got %dx%d", bounds.Dx(), bounds.Dy())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
img := newTestImage(64, 64)
|
||||
data, err := EncodeLossless(img)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeLossless error: %v", err)
|
||||
}
|
||||
decoded, err := Decode(data)
|
||||
if err != nil {
|
||||
t.Fatalf("Decode error: %v", err)
|
||||
}
|
||||
bounds := decoded.Bounds()
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
orig := img.At(x, y)
|
||||
got := decoded.At(x, y)
|
||||
or, og, ob, oa := orig.RGBA()
|
||||
gr, gg, gb, ga := got.RGBA()
|
||||
if or != gr || og != gg || ob != gb || oa != ga {
|
||||
t.Fatalf("pixel mismatch at (%d,%d): want %v got %v", x, y, orig, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeLossless(t *testing.T) {
|
||||
img := newTestImage(50, 50)
|
||||
data, err := EncodeLossless(img)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeLossless error: %v", err)
|
||||
}
|
||||
if !isWebP(data) {
|
||||
t.Fatalf("output is not valid WebP")
|
||||
}
|
||||
decoded, err := Decode(data)
|
||||
if err != nil {
|
||||
t.Fatalf("Decode error: %v", err)
|
||||
}
|
||||
bounds := decoded.Bounds()
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
orig := img.At(x, y)
|
||||
got := decoded.At(x, y)
|
||||
or, og, ob, oa := orig.RGBA()
|
||||
gr, gg, gb, ga := got.RGBA()
|
||||
if or != gr || og != gg || ob != gb || oa != ga {
|
||||
t.Fatalf("pixel mismatch at (%d,%d): want %v got %v", x, y, orig, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewConsumer(t *testing.T) {
|
||||
c := NewConsumer()
|
||||
if c == nil {
|
||||
t.Fatal("NewConsumer returned nil")
|
||||
}
|
||||
if c.FormatName != "webp" {
|
||||
t.Fatalf("expected FormatName=webp, got %q", c.FormatName)
|
||||
}
|
||||
if len(c.Medias) == 0 {
|
||||
t.Fatal("expected at least one media")
|
||||
}
|
||||
media := c.Medias[0]
|
||||
if media.Kind != core.KindVideo {
|
||||
t.Fatalf("expected KindVideo, got %v", media.Kind)
|
||||
}
|
||||
if media.Direction != core.DirectionSendonly {
|
||||
t.Fatalf("expected DirectionSendonly, got %v", media.Direction)
|
||||
}
|
||||
hasJPEG := false
|
||||
hasRAW := false
|
||||
for _, codec := range media.Codecs {
|
||||
if codec.Name == core.CodecJPEG {
|
||||
hasJPEG = true
|
||||
}
|
||||
if codec.Name == core.CodecRAW {
|
||||
hasRAW = true
|
||||
}
|
||||
}
|
||||
if !hasJPEG {
|
||||
t.Fatal("expected JPEG codec in consumer medias")
|
||||
}
|
||||
if !hasRAW {
|
||||
t.Fatal("expected RAW codec in consumer medias")
|
||||
}
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package webp
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const header = "--frame\r\nContent-Type: image/webp\r\nContent-Length: "
|
||||
|
||||
// Writer writes multipart WebP frames to an HTTP response.
|
||||
type Writer struct {
|
||||
wr io.Writer
|
||||
buf []byte
|
||||
}
|
||||
|
||||
// NewWriter creates a Writer that sets the multipart Content-Type header.
|
||||
func NewWriter(w io.Writer) *Writer {
|
||||
h := w.(http.ResponseWriter).Header()
|
||||
h.Set("Content-Type", "multipart/x-mixed-replace; boundary=frame")
|
||||
return &Writer{wr: w, buf: []byte(header)}
|
||||
}
|
||||
|
||||
func (w *Writer) Write(p []byte) (n int, err error) {
|
||||
w.buf = w.buf[:len(header)]
|
||||
w.buf = append(w.buf, strconv.Itoa(len(p))...)
|
||||
w.buf = append(w.buf, "\r\n\r\n"...)
|
||||
w.buf = append(w.buf, p...)
|
||||
w.buf = append(w.buf, "\r\n"...)
|
||||
|
||||
if _, err = w.wr.Write(w.buf); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
w.wr.(http.Flusher).Flush()
|
||||
|
||||
return len(p), nil
|
||||
}
|
||||
Reference in New Issue
Block a user