From 47285675b99149e920fe8aeda645348e1beeb487 Mon Sep 17 00:00:00 2001 From: Brendan Le Glaunec Date: Thu, 23 May 2019 01:42:04 +0200 Subject: [PATCH] Add route dictionary generator (#204) --- dictionaries/routes | 92 +++++++-- .../main.go | 188 ++++++++++++++++++ 2 files changed, 259 insertions(+), 21 deletions(-) create mode 100644 tools/automatic_route_dictionary_updater/main.go diff --git a/dictionaries/routes b/dictionaries/routes index 66a03b8..685ee80 100644 --- a/dictionaries/routes +++ b/dictionaries/routes @@ -1,76 +1,110 @@ +/live/ch01_0 +0/1:1/main +0/usrnm:pwd/main +0/video1 1 1.AMP 1/h264major 1/stream1 11 12 +125 +666 +AVStream1_1 +CAM_ID.password.mp2 +CH001.sdp +GetData.cgi +HighResolutionVideo +LowResolutionVideo +MediaInput/h264 +MediaInput/mpeg4 +ONVIF/MediaInput +ONVIF/MediaInput?profile=4_def_profile6 +StdCh1 +Streaming/Channels/1 +Streaming/Unicast/channels/101 +StreamingSetting?version=1.0&action=getRTSPStream&ChannelID=1&ChannelName=Channel1 +VideoInput/1/h264/1 +VideoInput/1/mpeg4/1 access_code access_name_for_stream_1_to_5 +api/mjpegvideo.cgi av0_0 av2 +avc avn=2 axis-media/media.amp -axis-media/media.amp +axis-media/media.amp?camera=1 axis-media/media.amp?videocodec=h264 cam -CAM_ID.password.mp2 cam/realmonitor +cam/realmonitor?channel=0&subtype=0 +cam/realmonitor?channel=1&subtype=0 +cam/realmonitor?channel=1&subtype=1&unicast=true&proto=Onvif cam0_0 cam0_1 cam1/h264 cam1/h264/multicast cam1/mjpeg cam1/mpeg4 +cam1/mpeg4?user='username'&pwd='password' cam1/onvif-h264 camera.stm ch0 +ch00/0 +ch001.sdp +ch01.264 +ch01.264? +ch01.264?ptype=tcp ch0_0.h264 ch0_unicast_firststream ch0_unicast_secondstream -ch001.sdp -ch01.264 ch1-s1 channel1 -GetData.cgi +gnz_media/main h264 -h264_vga.sdp +h264.sdp +h264/ch1/sub/av_stream h264/media.amp +h264_vga.sdp image.mpg img/media.sav +img/media.sav?channel=1 img/video.asf img/video.sav ioImage/1 -ipcam_h264.sdp ipcam.sdp +ipcam_h264.sdp +ipcam_mjpeg.sdp live -live_mpeg4.sdp live.sdp live/av0 +live/ch0 live/ch00_0 live/h264 live/main live/main0 live/mpeg4 +live1.sdp live3.sdp +live_mpeg4.sdp +live_st1 livestream livestream/ -LowResolutionVideo +main media media.amp +media.amp?streamprofile=Profile1 media/media.amp media/video1 -MediaInput/h264 -MediaInput/h264 -MediaInput/mpeg4 -MediaInput/mpeg4 medias2 mjpeg/media.smp mp4 +mpeg/media.amp mpeg4 mpeg4/1/media.amp mpeg4/media.amp -mpeg4/media.amp mpeg4/media.smp mpeg4unicast mpg4/rtsp.amp @@ -82,10 +116,14 @@ nphMpeg4/g726-640x48 nphMpeg4/g726-640x480 nphMpeg4/nil-320x240 onvif-media/media.amp -ONVIF/MediaInput +onvif1 +pass@10.0.0.5:6667/blinkhd play1.sdp play2.sdp +profile2/media.smp +profile5/media.smp rtpvideo1.sdp +rtsp://ip-addr rtsp_live0 rtsp_live1 rtsp_live2 @@ -93,12 +131,25 @@ rtsp_tunnel rtsph264 rtsph2641080p stream +stream.sdp stream1 -Streaming/Channels/1 +streaming/channels/0 +streaming/channels/1 +streaming/channels/101 +tcp/av0_0 +test +trackID=1 ucast/11 +udp/av0_0 udp/unicast/aiphone_H264 -user_defined +udpstream user.pin.mp2 +user=admin&password=&channel=1&stream=0.sdp? +user=admin&password=&channel=1&stream=0.sdp?real_stream +user=admin_password=?????_channel=1_stream=0.sdp?real_stream +user=admin_password=R5XFY888_channel=1_stream=0.sdp?real_stream +user_defined +v2 video video.3gp video.h264 @@ -107,11 +158,10 @@ video.mp4 video.pro1 video.pro2 video.pro3 +video0.sdp video1 video1+audio1 -VideoInput/1/h264/1 -VideoInput/1/h264/1 -VideoInput/1/mpeg4/1 videoMain +videoinput_1/h264_1/media.stm vis -wfov +wfov \ No newline at end of file diff --git a/tools/automatic_route_dictionary_updater/main.go b/tools/automatic_route_dictionary_updater/main.go new file mode 100644 index 0000000..607017d --- /dev/null +++ b/tools/automatic_route_dictionary_updater/main.go @@ -0,0 +1,188 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "net/http" + "sort" + "strings" + "sync" + + "github.com/ullaakut/disgo/style" + + "github.com/PuerkitoBio/goquery" + "github.com/ullaakut/disgo" + "github.com/vbauerster/mpb" + "github.com/vbauerster/mpb/decor" +) + +const dictionaryURL = "https://community.geniusvision.net/platform/cprndr/manulist" + +var rtspURLsFound sync.Map + +func main() { + if err := updateDictionary(); err != nil { + log.Fatalf(err.Error()) + } +} + +func updateDictionary() error { + disgo.SetTerminalOptions(disgo.WithColors(true), disgo.WithDebug(true)) + + disgo.StartStep("Fetching dictionary list") + resp, err := http.Get(dictionaryURL) + if err != nil { + return disgo.FailStepf("unable to download dictionaries: %v", err) + } + defer resp.Body.Close() + + disgo.StartStep("Parsing dictionary list") + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return disgo.FailStepf("unable to read from dictionary list: %v", err) + } + + var vendorURLs []string + doc.Find("td.simpletable a").Each(func(i int, s *goquery.Selection) { + url, ok := s.Attr("href") + if !ok { + return + } + + if url != "javascript:void(0)" { + vendorURLs = append(vendorURLs, url) + } + }) + + disgo.StartStep("Loading current cameradar dictionary") + currentDictionary, err := ioutil.ReadFile("dictionaries/routes") + if err != nil { + return disgo.FailStepf("unable to read current dictionary: %v", err) + } + + dictionaryEntries := bytes.Split(currentDictionary, []byte("\n")) + + for _, rtspURL := range dictionaryEntries { + rtspURLsFound.Store(string(rtspURL), struct{}{}) + } + + disgo.Debugf("Current dictionary has %d entries\n", len(dictionaryEntries)) + disgo.EndStep() + + p := mpb.New(mpb.WithWidth(64)) + name := fmt.Sprintf("Fetching default routes from %d constructors:", len(vendorURLs)) + bar := p.AddBar(int64(len(vendorURLs)), + // set custom bar style, default one is "[=>-]" + mpb.BarStyle("╢▌▌░╟"), + mpb.PrependDecorators( + // display our name with one space on the right + decor.Name(name, decor.WC{W: len(name), C: decor.DidentRight}), + ), + mpb.AppendDecorators(decor.Percentage()), + ) + + for _, url := range vendorURLs { + go loadRoutes(url, bar) + } + + p.Wait() + + disgo.StartStep("Converting found routes into proper data model") + + var rtspURLs []string + rtspURLsFound.Range(func(rtspURL, _ interface{}) bool { + disgo.Infoln("Adding URL", rtspURL.(string)) + rtspURLs = append(rtspURLs, rtspURL.(string)) + return true + }) + + sort.Slice(rtspURLs, func(a, b int) bool { + return rtspURLs[a] < rtspURLs[b] + }) + + disgo.EndStep() + + if len(dictionaryEntries) < len(rtspURLs) { + disgo.Infof("%s Saving them in cameradar default dictionary.\n", style.Success("Found ", len(rtspURLs)-len(dictionaryEntries), " new entries!")) + saveRoutes(rtspURLs) + } else { + disgo.Infoln(style.Success("No new entry found, dictionary up-to-date! :)")) + } + + return nil +} + +func loadRoutes(url string, bar *mpb.Bar) { + defer bar.IncrBy(1) + + var ( + failureCounter int + resp *http.Response + err error + ) + for failureCounter < 5 { + resp, err = http.Get(url) + if err != nil { + failureCounter++ + } else { + break + } + } + + if failureCounter == 5 { + disgo.Errorln("Request failed 5 times in a row, giving up on this vendor") + return + } + + defer resp.Body.Close() + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + disgo.Errorf("unable to read from dictionary list for URL %q: %v\n", url, err) + return + } + + doc.Find("tr.simpletable td.simpletable:nth-child(4) a").Each(func(i int, s *goquery.Selection) { + rtspURL := s.Text() + + if strings.HasPrefix(rtspURL, "(") && strings.HasSuffix(rtspURL, ")") { + return + } + + if strings.HasPrefix(rtspURL, "[") && strings.HasSuffix(rtspURL, "]") { + return + } + + if strings.HasPrefix(rtspURL, "http://") { + return + } + + // Skip the port and only get the route. + if strings.HasPrefix(rtspURL, "rtsp://ip-addr:") { + routeAndPort := strings.TrimSpace(strings.TrimPrefix(rtspURL, "rtsp://ip-addr:")) + route := strings.TrimLeft(routeAndPort, "0123456789/") + rtspURLsFound.Store(route, struct{}{}) + return + } + + switch rtspURL { + case "", "[Details]", "rtsp://ip-addr/", "http://ip-addr/": + return + default: + route := strings.TrimSpace(strings.TrimPrefix(rtspURL, "rtsp://ip-addr/")) + rtspURLsFound.Store(route, struct{}{}) + } + }) +} + +func saveRoutes(rtspURLs []string) { + contents := strings.Join(rtspURLs, "\n") + + disgo.StartStep("Writing new dictionary file") + err := ioutil.WriteFile("dictionaries/routes", []byte(contents), 0644) + if err != nil { + disgo.FailStepf("unable to write dictionnary: %v", err) + } +}