diff --git a/Dockerfile b/Dockerfile index 09e6adad..10936d45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,8 @@ FROM base # Install ffmpeg, tini (for signal handling), # and other common tools for the echo source. # alsa-plugins-pulse for ALSA support (+0MB) -RUN apk add --no-cache tini ffmpeg bash curl jq alsa-plugins-pulse +# font-droid for FFmpeg drawtext filter (+2MB) +RUN apk add --no-cache tini ffmpeg bash curl jq alsa-plugins-pulse font-droid # Hardware Acceleration for Intel CPU (+50MB) ARG TARGETARCH diff --git a/README.md b/README.md index 3f3574e9..fe41ddf6 100644 --- a/README.md +++ b/README.md @@ -582,7 +582,7 @@ POST http://localhost:1984/api/streams?dst=camera1&src=ffmpeg:http://example.com The HTTP API is the main part for interacting with the application. Default address: `http://localhost:1984/`. -Interactive [OpenAPI](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/AlexxIT/go2rtc/master/api/openapi.yaml&nocors). +Interactive [OpenAPI](https://alexxit.github.io/go2rtc/api/). go2rtc has its own JS video player (`video-rtc.js`) with: @@ -1074,6 +1074,7 @@ streams: ## Cameras experience - [Dahua](https://www.dahuasecurity.com/) - reference implementation streaming protocols, a lot of settings, high stream quality, multiple streaming clients +- [EZVIZ](https://www.ezviz.com/) - awful RTSP protocol realisation, many bugs in SDP - [Hikvision](https://www.hikvision.com/) - a lot of proprietary streaming technologies - [Reolink](https://reolink.com/) - some models has awful unusable RTSP realisation and not best HTTP-FLV alternative (I recommend that you contact Reolink support for new firmware), few settings - [Sonoff](https://sonoff.tech/) - very low stream quality, no settings, not best protocol implementation diff --git a/api/openapi.yaml b/api/openapi.yaml index 42d3d8c6..e9de7198 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -232,7 +232,7 @@ paths: /api/webrtc?src={src}: post: summary: Get stream in WebRTC format (WHEP) - description: "[Source: WebRTC](https://github.com/AlexxIT/go2rtc#source-webrtc)" + description: "[Module: WebRTC](https://github.com/AlexxIT/go2rtc#module-webrtc)" tags: [ Consume stream ] parameters: - $ref: "#/components/parameters/stream_src_path" diff --git a/hardware.Dockerfile b/hardware.Dockerfile index c6424d29..3c0bf7d8 100644 --- a/hardware.Dockerfile +++ b/hardware.Dockerfile @@ -12,7 +12,13 @@ FROM ngrok/ngrok:${NGROK_VERSION} AS ngrok # 1. Build go2rtc binary -FROM go AS build +FROM --platform=$BUILDPLATFORM go AS build +ARG TARGETPLATFORM +ARG TARGETOS +ARG TARGETARCH + +ENV GOOS=${TARGETOS} +ENV GOARCH=${TARGETARCH} WORKDIR /build diff --git a/internal/README.md b/internal/README.md deleted file mode 100644 index e200c163..00000000 --- a/internal/README.md +++ /dev/null @@ -1,11 +0,0 @@ -## Go - -``` -go mod why github.com/pion/rtcp -go list -deps .\cmd\go2rtc_rtsp\ -``` - -## Useful links - -- https://github.com/golang-standards/project-layout -- https://github.com/micro/micro diff --git a/internal/ffmpeg/device/device_darwin.go b/internal/ffmpeg/device/device_darwin.go index 7526b885..0900cf0c 100644 --- a/internal/ffmpeg/device/device_darwin.go +++ b/internal/ffmpeg/device/device_darwin.go @@ -3,24 +3,41 @@ package device import ( "github.com/AlexxIT/go2rtc/internal/api" "github.com/AlexxIT/go2rtc/pkg/core" + "net/url" "os/exec" "regexp" "strings" ) -// https://trac.ffmpeg.org/wiki/Capture/Webcam -const deviceInputPrefix = "-f avfoundation" +func queryToInput(query url.Values) string { + video := query.Get("video") + audio := query.Get("audio") -func deviceInputSuffix(video, audio string) string { - switch { - case video != "" && audio != "": - return `"` + video + `:` + audio + `"` - case video != "": - return `"` + video + `"` - case audio != "": - return `":` + audio + `"` + if video == "" && audio == "" { + return "" } - return "" + + // https://ffmpeg.org/ffmpeg-devices.html#avfoundation + input := "-f avfoundation" + + if video != "" { + video = indexToItem(videos, video) + + for key, value := range query { + switch key { + case "resolution": + input += " -video_size " + value[0] + case "pixel_format", "framerate", "video_size", "capture_cursor", "capture_mouse_clicks", "capture_raw_data": + input += " -" + key + " " + value[0] + } + } + } + + if audio != "" { + audio = indexToItem(audios, audio) + } + + return input + ` -i "` + video + `:` + audio + `"` } func initDevices() { diff --git a/internal/ffmpeg/device/device_linux.go b/internal/ffmpeg/device/device_linux.go index 1b35472c..1dd8bd74 100644 --- a/internal/ffmpeg/device/device_linux.go +++ b/internal/ffmpeg/device/device_linux.go @@ -3,19 +3,36 @@ package device import ( "github.com/AlexxIT/go2rtc/internal/api" "github.com/AlexxIT/go2rtc/pkg/core" + "net/url" "os" "os/exec" "regexp" "strings" ) -// https://trac.ffmpeg.org/wiki/Capture/Webcam -const deviceInputPrefix = "-f v4l2" +func queryToInput(query url.Values) string { + if video := query.Get("video"); video != "" { + // https://ffmpeg.org/ffmpeg-devices.html#video4linux2_002c-v4l2 + input := "-f v4l2" -func deviceInputSuffix(video, audio string) string { - if video != "" { - return video + for key, value := range query { + switch key { + case "resolution": + input += " -video_size " + value[0] + case "video_size", "pixel_format", "input_format", "framerate", "use_libv4l2": + input += " -" + key + " " + value[0] + } + } + + return input + " -i " + indexToItem(videos, video) } + + if audio := query.Get("audio"); audio != "" { + input := "-f alsa" + + return input + " -i " + indexToItem(audios, audio) + } + return "" } @@ -57,4 +74,15 @@ func initDevices() { streams = append(streams, stream) } } + + err = exec.Command(Bin, "-f", "alsa", "-i", "default", "-t", "1", "-f", "null", "-").Run() + if err == nil { + stream := api.Stream{ + Name: "ALSA default", + URL: "ffmpeg:device?audio=default#audio=opus", + } + + audios = append(audios, "default") + streams = append(streams, stream) + } } diff --git a/internal/ffmpeg/device/device_windows.go b/internal/ffmpeg/device/device_windows.go index 1f614891..8bae19c4 100644 --- a/internal/ffmpeg/device/device_windows.go +++ b/internal/ffmpeg/device/device_windows.go @@ -3,12 +3,58 @@ package device import ( "github.com/AlexxIT/go2rtc/internal/api" "github.com/AlexxIT/go2rtc/pkg/core" + "net/url" "os/exec" "regexp" ) -// https://trac.ffmpeg.org/wiki/DirectShow -const deviceInputPrefix = "-f dshow" +func queryToInput(query url.Values) string { + video := query.Get("video") + audio := query.Get("audio") + + if video == "" && audio == "" { + return "" + } + + // https://ffmpeg.org/ffmpeg-devices.html#dshow + input := "-f dshow" + + if video != "" { + video = indexToItem(videos, video) + + for key, value := range query { + switch key { + case "resolution": + input += " -video_size " + value[0] + case "video_size", "framerate", "pixel_format": + input += " -" + key + " " + value[0] + } + } + } + + if audio != "" { + audio = indexToItem(audios, audio) + + for key, value := range query { + switch key { + case "sample_rate", "sample_size", "channels", "audio_buffer_size": + input += " -" + key + " " + value[0] + } + } + } + + if video != "" { + input += ` -i video="` + video + `"` + + if audio != "" { + input += `:audio="` + audio + `"` + } + } else { + input += ` -i audio="` + audio + `"` + } + + return input +} func deviceInputSuffix(video, audio string) string { switch { diff --git a/internal/ffmpeg/device/devices.go b/internal/ffmpeg/device/devices.go index 3e657906..4eaa6e87 100644 --- a/internal/ffmpeg/device/devices.go +++ b/internal/ffmpeg/device/devices.go @@ -1,6 +1,7 @@ package device import ( + "errors" "github.com/AlexxIT/go2rtc/internal/api" "net/http" "net/url" @@ -16,45 +17,23 @@ func Init(bin string) { } func GetInput(src string) (string, error) { + i := strings.IndexByte(src, '?') + if i < 0 { + return "", errors.New("empty query: " + src) + } + + query, err := url.ParseQuery(src[i+1:]) + if err != nil { + return "", err + } + runonce.Do(initDevices) - input := deviceInputPrefix - - var video, audio string - - if i := strings.IndexByte(src, '?'); i > 0 { - query, err := url.ParseQuery(src[i+1:]) - if err != nil { - return "", err - } - for key, value := range query { - switch key { - case "video": - video = value[0] - case "audio": - audio = value[0] - case "resolution": - input += " -video_size " + value[0] - default: // "input_format", "framerate", "video_size" - input += " -" + key + " " + value[0] - } - } + if input := queryToInput(query); input != "" { + return input, nil } - if video != "" { - if i, err := strconv.Atoi(video); err == nil && i < len(videos) { - video = videos[i] - } - } - if audio != "" { - if i, err := strconv.Atoi(audio); err == nil && i < len(audios) { - audio = audios[i] - } - } - - input += " -i " + deviceInputSuffix(video, audio) - - return input, nil + return "", errors.New("wrong query: " + src) } var Bin string @@ -68,3 +47,10 @@ func apiDevices(w http.ResponseWriter, r *http.Request) { api.ResponseStreams(w, streams) } + +func indexToItem(items []string, index string) string { + if i, err := strconv.Atoi(index); err == nil && i < len(items) { + return items[i] + } + return index +} diff --git a/internal/ffmpeg/ffmpeg.go b/internal/ffmpeg/ffmpeg.go index be7e4c26..0d8e0ab4 100644 --- a/internal/ffmpeg/ffmpeg.go +++ b/internal/ffmpeg/ffmpeg.go @@ -63,7 +63,9 @@ var defaults = map[string]string{ //"mjpeg": "-c:v mjpeg -force_duplicated_matrix:v 1 -huffman:v 0 -pix_fmt:v yuvj420p", // https://ffmpeg.org/ffmpeg-codecs.html#libopus-1 - "opus": "-c:a libopus -ar:a 48000 -ac:a 2 -application:a voip -compression_level:a 0", + // https://github.com/pion/webrtc/issues/1514 + // `-af adelay=0|0` - force frame_size=960, important for WebRTC audio quality + "opus": "-c:a libopus -ar:a 48000 -ac:a 2 -application:a voip -af adelay=0|0", "pcmu": "-c:a pcm_mulaw -ar:a 8000 -ac:a 1", "pcmu/16000": "-c:a pcm_mulaw -ar:a 16000 -ac:a 1", "pcmu/48000": "-c:a pcm_mulaw -ar:a 48000 -ac:a 1", @@ -90,8 +92,8 @@ var defaults = map[string]string{ // hardware NVidia on Linux and Windows // preset=p2 - faster, tune=ll - low latency - "h264/cuda": "-c:v h264_nvenc -g 50 -profile:v high -level:v auto -preset:v p2 -tune:v ll", - "h265/cuda": "-c:v hevc_nvenc -g 50 -profile:v high -level:v auto", + "h264/cuda": "-c:v h264_nvenc -g 50 -bf 0 -profile:v high -level:v auto -preset:v p2 -tune:v ll", + "h265/cuda": "-c:v hevc_nvenc -g 50 -bf 0 -profile:v high -level:v auto", // hardware Intel on Windows "h264/dxva2": "-c:v h264_qsv -g 50 -bf 0 -profile:v high -level:v 4.1 -async_depth:v 1", @@ -103,6 +105,14 @@ var defaults = map[string]string{ "h265/videotoolbox": "-c:v hevc_videotoolbox -g 50 -bf 0 -profile:v high -level:v 5.1", } +// configTemplate - return template from config (defaults) if exist or return raw template +func configTemplate(template string) string { + if s := defaults[template]; s != "" { + return s + } + return template +} + // inputTemplate - select input template from YAML config by template name // if query has input param - select another template by this name // if there is no another template - use input param as template @@ -110,9 +120,7 @@ var defaults = map[string]string{ func inputTemplate(name, s string, query url.Values) string { var template string if input := query.Get("input"); input != "" { - if template = defaults[input]; template == "" { - template = input - } + template = configTemplate(input) } else { template = defaults[name] } @@ -199,6 +207,8 @@ func parseArgs(s string) *ffmpeg.Args { if len(query) != 0 { // 1. Process raw params for FFmpeg for _, raw := range query["raw"] { + // support templates https://github.com/AlexxIT/go2rtc/issues/487 + raw = configTemplate(raw) args.AddCodec(raw) } @@ -234,6 +244,18 @@ func parseArgs(s string) *ffmpeg.Args { } } + for _, drawtext := range query["drawtext"] { + // support templates https://github.com/AlexxIT/go2rtc/issues/487 + drawtext = configTemplate(drawtext) + + // support default timestamp format + if !strings.Contains(drawtext, "text=") { + drawtext += `:text='%{localtime\:%Y-%m-%d %X}'` + } + + args.AddFilter("drawtext=" + drawtext) + } + // 3. Process video codecs if args.Video > 0 { for _, video := range query["video"] { diff --git a/internal/ffmpeg/hardware/hardware.go b/internal/ffmpeg/hardware/hardware.go index bb8f7174..d4aa917c 100644 --- a/internal/ffmpeg/hardware/hardware.go +++ b/internal/ffmpeg/hardware/hardware.go @@ -55,33 +55,51 @@ func MakeHardware(args *ffmpeg.Args, engine string, defaults map[string]string) switch engine { case EngineVAAPI: - args.Input = "-hwaccel vaapi -hwaccel_output_format vaapi " + args.Input args.Codecs[i] = defaults[name+"/"+engine] - for i, filter := range args.Filters { - if strings.HasPrefix(filter, "scale=") { - args.Filters[i] = "scale_vaapi=" + filter[6:] - } - if strings.HasPrefix(filter, "transpose=") { - if filter == "transpose=1,transpose=1" { // 180 degrees half-turn - args.Filters[i] = "transpose_vaapi=4" // reversal - } else { - args.Filters[i] = "transpose_vaapi=" + filter[10:] + if !args.HasFilters("drawtext=") { + args.Input = "-hwaccel vaapi -hwaccel_output_format vaapi " + args.Input + + for i, filter := range args.Filters { + if strings.HasPrefix(filter, "scale=") { + args.Filters[i] = "scale_vaapi=" + filter[6:] + } + if strings.HasPrefix(filter, "transpose=") { + if filter == "transpose=1,transpose=1" { // 180 degrees half-turn + args.Filters[i] = "transpose_vaapi=4" // reversal + } else { + args.Filters[i] = "transpose_vaapi=" + filter[10:] + } } } + + // fix if input doesn't support hwaccel, do nothing when support + // insert as first filter before hardware scale and transpose + args.InsertFilter("format=vaapi|nv12,hwupload") + } else { + // enable software pixel for drawtext, scale and transpose + args.Input = "-hwaccel vaapi -hwaccel_output_format nv12 " + args.Input + + args.AddFilter("hwupload") } - // fix if input doesn't support hwaccel, do nothing when support - args.InsertFilter("format=vaapi|nv12,hwupload") - case EngineCUDA: - args.Input = "-hwaccel cuda -hwaccel_output_format cuda -extra_hw_frames 2 " + args.Input args.Codecs[i] = defaults[name+"/"+engine] - for i, filter := range args.Filters { - if strings.HasPrefix(filter, "scale=") { - args.Filters[i] = "scale_cuda=" + filter[6:] + // CUDA doesn't support hardware transpose + // https://github.com/AlexxIT/go2rtc/issues/389 + if !args.HasFilters("drawtext=", "transpose=") { + args.Input = "-hwaccel cuda -hwaccel_output_format cuda " + args.Input + + for i, filter := range args.Filters { + if strings.HasPrefix(filter, "scale=") { + args.Filters[i] = "scale_cuda=" + filter[6:] + } } + } else { + args.Input = "-hwaccel cuda -hwaccel_output_format nv12 " + args.Input + + args.AddFilter("hwupload") } case EngineDXVA2: diff --git a/internal/hls/README.md b/internal/hls/README.md new file mode 100644 index 00000000..c14f0fdb --- /dev/null +++ b/internal/hls/README.md @@ -0,0 +1,3 @@ +## Useful links + +- https://walterebert.com/playground/video/hls/ diff --git a/internal/hls/hls.go b/internal/hls/hls.go index a92b46a7..de92047a 100644 --- a/internal/hls/hls.go +++ b/internal/hls/hls.go @@ -1,8 +1,8 @@ package hls import ( - "fmt" "github.com/AlexxIT/go2rtc/internal/api" + "github.com/AlexxIT/go2rtc/internal/api/ws" "github.com/AlexxIT/go2rtc/internal/streams" "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/mp4" @@ -25,6 +25,8 @@ func Init() { // HLS (fMP4) api.HandleFunc("api/hls/init.mp4", handlerInit) api.HandleFunc("api/hls/segment.m4s", handlerSegmentMP4) + + ws.HandleFunc("hls", handlerWSHLS) } type Consumer interface { @@ -35,16 +37,6 @@ type Consumer interface { Start() } -type Session struct { - cons Consumer - playlist string - init []byte - segment []byte - seq int - alive *time.Timer - mu sync.Mutex -} - const keepalive = 5 * time.Second var sessions = map[string]*Session{} @@ -86,21 +78,21 @@ func handlerStream(w http.ResponseWriter, r *http.Request) { } } + if err := stream.AddConsumer(cons); err != nil { + log.Error().Err(err).Caller().Send() + return + } + session := &Session{cons: cons} cons.Listen(func(msg any) { if data, ok := msg.([]byte); ok { session.mu.Lock() - session.segment = append(session.segment, data...) + session.buffer = append(session.buffer, data...) session.mu.Unlock() } }) - if err := stream.AddConsumer(cons); err != nil { - log.Error().Err(err).Caller().Send() - return - } - session.alive = time.AfterFunc(keepalive, func() { stream.RemoveConsumer(cons) }) @@ -112,7 +104,7 @@ func handlerStream(w http.ResponseWriter, r *http.Request) { // two segments important for Chromecast if medias != nil { - session.playlist = `#EXTM3U + session.template = `#EXTM3U #EXT-X-VERSION:6 #EXT-X-TARGETDURATION:1 #EXT-X-MEDIA-SEQUENCE:%d @@ -122,7 +114,7 @@ segment.m4s?id=` + sid + `&n=%d #EXTINF:0.500, segment.m4s?id=` + sid + `&n=%d` } else { - session.playlist = `#EXTM3U + session.template = `#EXTM3U #EXT-X-VERSION:3 #EXT-X-TARGETDURATION:1 #EXT-X-MEDIA-SEQUENCE:%d @@ -167,9 +159,7 @@ func handlerPlaylist(w http.ResponseWriter, r *http.Request) { return } - s := fmt.Sprintf(session.playlist, session.seq, session.seq, session.seq+1) - - if _, err := w.Write([]byte(s)); err != nil { + if _, err := w.Write([]byte(session.Playlist())); err != nil { log.Error().Err(err).Caller().Send() } } @@ -194,22 +184,12 @@ func handlerSegmentTS(w http.ResponseWriter, r *http.Request) { session.alive.Reset(keepalive) - var i byte - for len(session.segment) == 0 { - if i++; i > 10 { - http.NotFound(w, r) - return - } - time.Sleep(time.Millisecond * 100) + data := session.Segment() + if data == nil { + http.NotFound(w, r) + return } - session.mu.Lock() - data := session.segment - // important to start new segment with init - session.segment = session.init - session.seq++ - session.mu.Unlock() - if _, err := w.Write(data); err != nil { log.Error().Err(err).Caller().Send() } @@ -233,7 +213,16 @@ func handlerInit(w http.ResponseWriter, r *http.Request) { return } - if _, err := w.Write(session.init); err != nil { + data := session.init + session.init = nil + + session.segment0 = session.Segment() + if session.segment0 == nil { + http.NotFound(w, r) + return + } + + if _, err := w.Write(data); err != nil { log.Error().Err(err).Caller().Send() } } @@ -243,11 +232,13 @@ func handlerSegmentMP4(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "video/iso.segment") if r.Method == "OPTIONS" { - w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS") + w.Header().Set("Access-Control-Allow-Methods", "GET") return } - sid := r.URL.Query().Get("id") + query := r.URL.Query() + + sid := query.Get("id") sessionsMu.RLock() session := sessions[sid] sessionsMu.RUnlock() @@ -258,20 +249,18 @@ func handlerSegmentMP4(w http.ResponseWriter, r *http.Request) { session.alive.Reset(keepalive) - var i byte - for len(session.segment) == 0 { - if i++; i > 10 { - http.NotFound(w, r) - return - } - time.Sleep(time.Millisecond * 100) + var data []byte + + if query.Get("n") != "0" { + data = session.Segment() + } else { + data = session.segment0 } - session.mu.Lock() - data := session.segment - session.segment = nil - session.seq++ - session.mu.Unlock() + if data == nil { + http.NotFound(w, r) + return + } if _, err := w.Write(data); err != nil { log.Error().Err(err).Caller().Send() diff --git a/internal/hls/session.go b/internal/hls/session.go new file mode 100644 index 00000000..6f4d66ed --- /dev/null +++ b/internal/hls/session.go @@ -0,0 +1,41 @@ +package hls + +import ( + "fmt" + "sync" + "time" +) + +type Session struct { + cons Consumer + template string + init []byte + segment0 []byte + buffer []byte + seq int + alive *time.Timer + mu sync.Mutex +} + +func (s *Session) Playlist() string { + return fmt.Sprintf(s.template, s.seq, s.seq, s.seq+1) +} + +func (s *Session) Segment() (segment []byte) { + for i := 0; i < 20 && segment == nil; i++ { + if i > 0 { + time.Sleep(50 * time.Millisecond) + } + + s.mu.Lock() + if len(s.buffer) > 0 { + segment = s.buffer + // for TS important to start new segment with init + s.buffer = s.init + s.seq++ + } + s.mu.Unlock() + } + + return +} diff --git a/internal/hls/ws.go b/internal/hls/ws.go new file mode 100644 index 00000000..adae0a57 --- /dev/null +++ b/internal/hls/ws.go @@ -0,0 +1,81 @@ +package hls + +import ( + "errors" + "github.com/AlexxIT/go2rtc/internal/api" + "github.com/AlexxIT/go2rtc/internal/api/ws" + "github.com/AlexxIT/go2rtc/internal/streams" + "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/mp4" + "github.com/AlexxIT/go2rtc/pkg/tcp" + "github.com/rs/zerolog/log" + "strings" + "time" +) + +func handlerWSHLS(tr *ws.Transport, msg *ws.Message) error { + src := tr.Request.URL.Query().Get("src") + stream := streams.Get(src) + if stream == nil { + return errors.New(api.StreamNotFound) + } + + codecs := msg.String() + + cons := &mp4.Consumer{ + RemoteAddr: tcp.RemoteAddr(tr.Request), + UserAgent: tr.Request.UserAgent(), + Medias: mp4.ParseCodecs(codecs, true), + } + + if err := stream.AddConsumer(cons); err != nil { + log.Error().Err(err).Caller().Send() + return err + } + + session := &Session{cons: cons} + + cons.Listen(func(msg any) { + if data, ok := msg.([]byte); ok { + session.mu.Lock() + session.buffer = append(session.buffer, data...) + session.mu.Unlock() + } + }) + + session.alive = time.AfterFunc(keepalive, func() { + stream.RemoveConsumer(cons) + }) + session.init, _ = cons.Init() + + cons.Start() + + sid := core.RandString(8, 62) + + // two segments important for Chromecast + session.template = `#EXTM3U +#EXT-X-VERSION:6 +#EXT-X-TARGETDURATION:1 +#EXT-X-MEDIA-SEQUENCE:%d +#EXT-X-MAP:URI="init.mp4?id=` + sid + `" +#EXTINF:0.500, +segment.m4s?id=` + sid + `&n=%d +#EXTINF:0.500, +segment.m4s?id=` + sid + `&n=%d` + + sessionsMu.Lock() + sessions[sid] = session + sessionsMu.Unlock() + + // Apple Safari can play FLAC codec, but fail it it in m3u8 playlist + codecs = strings.Replace(cons.MimeCodecs(), mp4.MimeFlac, mp4.MimeAAC, 1) + + // bandwidth important for Safari, codecs useful for smooth playback + data := `#EXTM3U +#EXT-X-STREAM-INF:BANDWIDTH=1000000,CODECS="` + codecs + `" +hls/playlist.m3u8?id=` + sid + + tr.Write(&ws.Message{Type: "hls", Value: data}) + + return nil +} diff --git a/internal/mp4/ws.go b/internal/mp4/ws.go index 08d7da02..5b3f2eb5 100644 --- a/internal/mp4/ws.go +++ b/internal/mp4/ws.go @@ -5,10 +5,8 @@ import ( "github.com/AlexxIT/go2rtc/internal/api" "github.com/AlexxIT/go2rtc/internal/api/ws" "github.com/AlexxIT/go2rtc/internal/streams" - "github.com/AlexxIT/go2rtc/pkg/core" "github.com/AlexxIT/go2rtc/pkg/mp4" "github.com/AlexxIT/go2rtc/pkg/tcp" - "strings" ) func handlerWSMSE(tr *ws.Transport, msg *ws.Message) error { @@ -25,7 +23,7 @@ func handlerWSMSE(tr *ws.Transport, msg *ws.Message) error { if codecs := msg.String(); codecs != "" { log.Trace().Str("codecs", codecs).Msgf("[mp4] new WS/MSE consumer") - cons.Medias = parseMedias(codecs, true) + cons.Medias = mp4.ParseCodecs(codecs, true) } cons.Listen(func(msg any) { @@ -73,7 +71,7 @@ func handlerWSMP4(tr *ws.Transport, msg *ws.Message) error { if codecs := msg.String(); codecs != "" { log.Trace().Str("codecs", codecs).Msgf("[mp4] new WS/MP4 consumer") - cons.Medias = parseMedias(codecs, false) + cons.Medias = mp4.ParseCodecs(codecs, false) } cons.Listen(func(msg any) { @@ -95,51 +93,3 @@ func handlerWSMP4(tr *ws.Transport, msg *ws.Message) error { return nil } - -func parseMedias(codecs string, parseAudio bool) (medias []*core.Media) { - var videos []*core.Codec - var audios []*core.Codec - - for _, name := range strings.Split(codecs, ",") { - switch name { - case mp4.MimeH264: - codec := &core.Codec{Name: core.CodecH264} - videos = append(videos, codec) - case mp4.MimeH265: - codec := &core.Codec{Name: core.CodecH265} - videos = append(videos, codec) - case mp4.MimeAAC: - codec := &core.Codec{Name: core.CodecAAC} - audios = append(audios, codec) - case mp4.MimeFlac: - audios = append(audios, - &core.Codec{Name: core.CodecPCMA}, - &core.Codec{Name: core.CodecPCMU}, - &core.Codec{Name: core.CodecPCM}, - ) - case mp4.MimeOpus: - codec := &core.Codec{Name: core.CodecOpus} - audios = append(audios, codec) - } - } - - if videos != nil { - media := &core.Media{ - Kind: core.KindVideo, - Direction: core.DirectionSendonly, - Codecs: videos, - } - medias = append(medias, media) - } - - if audios != nil && parseAudio { - media := &core.Media{ - Kind: core.KindAudio, - Direction: core.DirectionSendonly, - Codecs: audios, - } - medias = append(medias, media) - } - - return -} diff --git a/pkg/ffmpeg/ffmpeg.go b/pkg/ffmpeg/ffmpeg.go index 10fb5bed..a24d9201 100644 --- a/pkg/ffmpeg/ffmpeg.go +++ b/pkg/ffmpeg/ffmpeg.go @@ -29,6 +29,18 @@ func (a *Args) InsertFilter(filter string) { a.Filters = append([]string{filter}, a.Filters...) } +func (a *Args) HasFilters(filters ...string) bool { + for _, f1 := range a.Filters { + for _, f2 := range filters { + if strings.HasPrefix(f1, f2) { + return true + } + } + } + + return false +} + func (a *Args) String() string { b := bytes.NewBuffer(make([]byte, 0, 512)) @@ -65,12 +77,13 @@ func (a *Args) String() string { if a.Filters != nil { for i, filter := range a.Filters { if i == 0 { - b.WriteString(" -vf ") + b.WriteString(` -vf "`) } else { b.WriteByte(',') } b.WriteString(filter) } + b.WriteByte('"') } b.WriteByte(' ') diff --git a/pkg/mp4/helpers.go b/pkg/mp4/helpers.go index 174dc48d..fc2a9cf9 100644 --- a/pkg/mp4/helpers.go +++ b/pkg/mp4/helpers.go @@ -1,6 +1,9 @@ package mp4 -import "github.com/AlexxIT/go2rtc/pkg/core" +import ( + "github.com/AlexxIT/go2rtc/pkg/core" + "strings" +) // ParseQuery - like usual parse, but with mp4 param handler func ParseQuery(query map[string][]string) []*core.Media { @@ -48,6 +51,54 @@ func ParseQuery(query map[string][]string) []*core.Media { return core.ParseQuery(query) } +func ParseCodecs(codecs string, parseAudio bool) (medias []*core.Media) { + var videos []*core.Codec + var audios []*core.Codec + + for _, name := range strings.Split(codecs, ",") { + switch name { + case MimeH264: + codec := &core.Codec{Name: core.CodecH264} + videos = append(videos, codec) + case MimeH265: + codec := &core.Codec{Name: core.CodecH265} + videos = append(videos, codec) + case MimeAAC: + codec := &core.Codec{Name: core.CodecAAC} + audios = append(audios, codec) + case MimeFlac: + audios = append(audios, + &core.Codec{Name: core.CodecPCMA}, + &core.Codec{Name: core.CodecPCMU}, + &core.Codec{Name: core.CodecPCM}, + ) + case MimeOpus: + codec := &core.Codec{Name: core.CodecOpus} + audios = append(audios, codec) + } + } + + if videos != nil { + media := &core.Media{ + Kind: core.KindVideo, + Direction: core.DirectionSendonly, + Codecs: videos, + } + medias = append(medias, media) + } + + if audios != nil && parseAudio { + media := &core.Media{ + Kind: core.KindAudio, + Direction: core.DirectionSendonly, + Codecs: audios, + } + medias = append(medias, media) + } + + return +} + const ( stateNone byte = iota stateInit diff --git a/pkg/pcm/helpers.go b/pkg/pcm/helpers.go deleted file mode 100644 index f2e723d5..00000000 --- a/pkg/pcm/helpers.go +++ /dev/null @@ -1,44 +0,0 @@ -package pcm - -import ( - "github.com/AlexxIT/go2rtc/pkg/core" - "github.com/pion/rtp" - "sync" -) - -func RepackBackchannel(handler core.HandlerFunc) core.HandlerFunc { - var buf []byte - var seq uint16 - - // fix https://github.com/AlexxIT/go2rtc/issues/432 - var mu sync.Mutex - - return func(packet *rtp.Packet) { - mu.Lock() - - buf = append(buf, packet.Payload...) - if len(buf) < 1024 { - mu.Unlock() - return - } - - pkt := &rtp.Packet{ - Header: rtp.Header{ - Version: 2, - Marker: true, // should be true - PayloadType: packet.PayloadType, // will be owerwriten - SequenceNumber: seq, - Timestamp: 0, // should be always zero - SSRC: packet.SSRC, - }, - Payload: buf[:1024], - } - - buf = buf[1024:] - seq++ - - mu.Unlock() - - handler(pkt) - } -} diff --git a/pkg/pcm/pcm.go b/pkg/pcm/pcm.go index 717a1450..d2c08717 100644 --- a/pkg/pcm/pcm.go +++ b/pkg/pcm/pcm.go @@ -3,6 +3,7 @@ package pcm import ( "github.com/AlexxIT/go2rtc/pkg/core" "github.com/pion/rtp" + "sync" ) func Resample(codec *core.Codec, sampleRate uint32, handler core.HandlerFunc) core.HandlerFunc { @@ -114,3 +115,54 @@ func DownsamplePCM(fromPCM func(int16) byte, n float32, handler core.HandlerFunc handler(&clone) } } + +// RepackG711 - Repack G.711 PCMA/PCMU into frames of size 1024 +// 1. Fixes WebRTC audio quality issue (monotonic timestamp) +// 2. Fixes Reolink Doorbell backchannel issue (zero timestamp) +// https://github.com/AlexxIT/go2rtc/issues/331 +func RepackG711(zeroTS bool, handler core.HandlerFunc) core.HandlerFunc { + const PacketSize = 1024 + + var buf []byte + var seq uint16 + var ts uint32 + + // fix https://github.com/AlexxIT/go2rtc/issues/432 + var mu sync.Mutex + + return func(packet *rtp.Packet) { + mu.Lock() + + buf = append(buf, packet.Payload...) + if len(buf) < PacketSize { + mu.Unlock() + return + } + + pkt := &rtp.Packet{ + Header: rtp.Header{ + Version: 2, + Marker: true, // should be true + PayloadType: packet.PayloadType, // will be owerwriten + SequenceNumber: seq, + SSRC: packet.SSRC, + }, + Payload: buf[:PacketSize], + } + + seq++ + + // don't know if zero TS important for Reolink Doorbell + // don't have this strange devices for tests + if !zeroTS { + pkt.Timestamp = ts + ts += PacketSize + } + + buf = buf[PacketSize:] + + mu.Unlock() + + handler(pkt) + } +} diff --git a/pkg/rtsp/consumer.go b/pkg/rtsp/consumer.go index 903967c1..a0cc9662 100644 --- a/pkg/rtsp/consumer.go +++ b/pkg/rtsp/consumer.go @@ -62,9 +62,9 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv // important to send original codec for valid IsRTP check sender.Handler = c.packetWriter(track.Codec, channel, codec.PayloadType) - // https://github.com/AlexxIT/go2rtc/issues/331 if c.mode == core.ModeActiveProducer && track.Codec.Name == core.CodecPCMA { - sender.Handler = pcm.RepackBackchannel(sender.Handler) + // Fix Reolink Doorbell https://github.com/AlexxIT/go2rtc/issues/331 + sender.Handler = pcm.RepackG711(true, sender.Handler) } sender.HandleRTP(track) diff --git a/pkg/webrtc/consumer.go b/pkg/webrtc/consumer.go index 070573c6..dc2e29c4 100644 --- a/pkg/webrtc/consumer.go +++ b/pkg/webrtc/consumer.go @@ -71,6 +71,9 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv codec.ClockRate = 8000 sender.Handler = pcm.Resample(track.Codec, 8000, sender.Handler) } + + // Fix audio quality https://github.com/AlexxIT/WebRTC/issues/500 + sender.Handler = pcm.RepackG711(false, sender.Handler) } sender.HandleRTP(track) diff --git a/scripts/README.md b/scripts/README.md index 7346cc4a..789dab52 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -7,6 +7,15 @@ - `aarch64` = `arm64` - `armv7` = `arm` +## Go + +``` +go get -u +go mod tidy +go mod why github.com/pion/rtcp +go list -deps .\cmd\go2rtc_rtsp\ +``` + ## Virus - https://go.dev/doc/faq#virus @@ -14,6 +23,8 @@ ## Useful links +- https://github.com/golang-standards/project-layout +- https://github.com/micro/micro - https://github.com/golang/go/wiki/GoArm - https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63 - https://en.wikipedia.org/wiki/AArch64 diff --git a/website/api/index.html b/website/api/index.html new file mode 100644 index 00000000..65e87fa3 --- /dev/null +++ b/website/api/index.html @@ -0,0 +1,19 @@ + + + + go2rtc - API + + + + + + + + + + \ No newline at end of file diff --git a/www/README.md b/www/README.md index 5c95bf17..fa28938c 100644 --- a/www/README.md +++ b/www/README.md @@ -11,6 +11,8 @@ ``` +- https://developer.apple.com/documentation/webkit/delivering_video_content_for_safari/ + **2. [Safari] pc.createOffer** Don't work in Desktop Safari: diff --git a/www/index.html b/www/index.html index 96198f8c..5125c50f 100644 --- a/www/index.html +++ b/www/index.html @@ -63,7 +63,7 @@ - + diff --git a/www/video-rtc.js b/www/video-rtc.js index 11ec30be..25109eac 100644 --- a/www/video-rtc.js +++ b/www/video-rtc.js @@ -32,10 +32,10 @@ export class VideoRTC extends HTMLElement { ]; /** - * [config] Supported modes (webrtc, mse, mp4, mjpeg). + * [config] Supported modes (webrtc, webrtc/tcp, mse, hls, mp4, mjpeg). * @type {string} */ - this.mode = "webrtc,mse,mp4,mjpeg"; + this.mode = "webrtc,mse,hls,mjpeg"; /** * [config] Run stream when not displayed on the screen. Default `false`. @@ -324,6 +324,9 @@ export class VideoRTC extends HTMLElement { if (this.mode.indexOf("mse") >= 0 && "MediaSource" in window) { // iPhone modes.push("mse"); this.onmse(); + } else if (this.mode.indexOf("hls") >= 0 && this.video.canPlayType("application/vnd.apple.mpegurl")) { + modes.push("hls"); + this.onhls(); } else if (this.mode.indexOf("mp4") >= 0) { modes.push("mp4"); this.onmp4(); @@ -440,6 +443,8 @@ export class VideoRTC extends HTMLElement { video2.addEventListener("loadeddata", ev => this.onpcvideo(ev), {once: true}); pc.addEventListener("icecandidate", ev => { + if (ev.candidate && this.mode.indexOf("webrtc/tcp") >= 0 && ev.candidate.protocol === "udp") return; + const candidate = ev.candidate ? ev.candidate.toJSON().candidate : ""; this.send({type: "webrtc/candidate", value: candidate}); }); @@ -471,16 +476,12 @@ export class VideoRTC extends HTMLElement { this.onmessage["webrtc"] = msg => { switch (msg.type) { case "webrtc/candidate": - pc.addIceCandidate({ - candidate: msg.value, - sdpMid: "0" - }).catch(() => console.debug); + if (this.mode.indexOf("webrtc/tcp") >= 0 && msg.value.indexOf(" udp ") > 0) return; + + pc.addIceCandidate({candidate: msg.value, sdpMid: "0"}).catch(() => console.debug); break; case "webrtc/answer": - pc.setRemoteDescription({ - type: "answer", - sdp: msg.value - }).catch(() => console.debug); + pc.setRemoteDescription({type: "answer", sdp: msg.value}).catch(() => console.debug); break; case "error": if (msg.value.indexOf("webrtc/offer") < 0) return; @@ -554,6 +555,17 @@ export class VideoRTC extends HTMLElement { this.send({type: "mjpeg"}); } + onhls() { + this.onmessage["hls"] = msg => { + const url = "http" + this.wsURL.substring(2, this.wsURL.indexOf("/ws")) + "/hls/"; + const playlist = msg.value.replace("hls/", url); + this.video.src = "data:application/vnd.apple.mpegurl;base64," + btoa(playlist); + this.play(); + } + + this.send({type: "hls", value: this.codecs("hls")}); + } + onmp4() { /** @type {HTMLCanvasElement} **/ const canvas = document.createElement("canvas"); diff --git a/www/video-stream.js b/www/video-stream.js index 22615dc5..9347a6c2 100644 --- a/www/video-stream.js +++ b/www/video-stream.js @@ -70,6 +70,7 @@ class VideoStream extends VideoRTC { this.divError = msg.value; break; case "mse": + case "hls": case "mp4": case "mjpeg": this.divMode = msg.type.toUpperCase();