Add route dictionary generator (#204)

dictionaries/routes

@@ -1,76 +1,110 @@
 
+/live/ch01_0
+0/1:1/main
+0/usrnm:pwd/main
+0/video1
 1
 1.AMP
 1/h264major
 1/stream1
 11
 12
+125
+666
+AVStream1_1
+CAM_ID.password.mp2
+CH001.sdp
+GetData.cgi
+HighResolutionVideo
+LowResolutionVideo
+MediaInput/h264
+MediaInput/mpeg4
+ONVIF/MediaInput
+ONVIF/MediaInput?profile=4_def_profile6
+StdCh1
+Streaming/Channels/1
+Streaming/Unicast/channels/101
+StreamingSetting?version=1.0&action=getRTSPStream&ChannelID=1&ChannelName=Channel1
+VideoInput/1/h264/1
+VideoInput/1/mpeg4/1
 access_code
 access_name_for_stream_1_to_5
+api/mjpegvideo.cgi
 av0_0
 av2
+avc
 avn=2
 axis-media/media.amp
-axis-media/media.amp
+axis-media/media.amp?camera=1
 axis-media/media.amp?videocodec=h264
 cam
-CAM_ID.password.mp2
 cam/realmonitor
+cam/realmonitor?channel=0&subtype=0
+cam/realmonitor?channel=1&subtype=0
+cam/realmonitor?channel=1&subtype=1&unicast=true&proto=Onvif
 cam0_0
 cam0_1
 cam1/h264
 cam1/h264/multicast
 cam1/mjpeg
 cam1/mpeg4
+cam1/mpeg4?user='username'&pwd='password'
 cam1/onvif-h264
 camera.stm
 ch0
+ch00/0
+ch001.sdp
+ch01.264
+ch01.264?
+ch01.264?ptype=tcp
 ch0_0.h264
 ch0_unicast_firststream
 ch0_unicast_secondstream
-ch001.sdp
-ch01.264
 ch1-s1
 channel1
-GetData.cgi
+gnz_media/main
 h264
-h264_vga.sdp
+h264.sdp
+h264/ch1/sub/av_stream
 h264/media.amp
+h264_vga.sdp
 image.mpg
 img/media.sav
+img/media.sav?channel=1
 img/video.asf
 img/video.sav
 ioImage/1
-ipcam_h264.sdp
 ipcam.sdp
+ipcam_h264.sdp
+ipcam_mjpeg.sdp
 live
-live_mpeg4.sdp
 live.sdp
 live/av0
+live/ch0
 live/ch00_0
 live/h264
 live/main
 live/main0
 live/mpeg4
+live1.sdp
 live3.sdp
+live_mpeg4.sdp
+live_st1
 livestream
 livestream/
-LowResolutionVideo
+main
 media
 media.amp
+media.amp?streamprofile=Profile1
 media/media.amp
 media/video1
-MediaInput/h264
-MediaInput/h264
-MediaInput/mpeg4
-MediaInput/mpeg4
 medias2
 mjpeg/media.smp
 mp4
+mpeg/media.amp
 mpeg4
 mpeg4/1/media.amp
 mpeg4/media.amp
-mpeg4/media.amp
 mpeg4/media.smp
 mpeg4unicast
 mpg4/rtsp.amp
@@ -82,10 +116,14 @@ nphMpeg4/g726-640x48
 nphMpeg4/g726-640x480
 nphMpeg4/nil-320x240
 onvif-media/media.amp
-ONVIF/MediaInput
+onvif1
+pass@10.0.0.5:6667/blinkhd
 play1.sdp
 play2.sdp
+profile2/media.smp
+profile5/media.smp
 rtpvideo1.sdp
+rtsp://ip-addr
 rtsp_live0
 rtsp_live1
 rtsp_live2
@@ -93,12 +131,25 @@ rtsp_tunnel
 rtsph264
 rtsph2641080p
 stream
+stream.sdp
 stream1
-Streaming/Channels/1
+streaming/channels/0
+streaming/channels/1
+streaming/channels/101
+tcp/av0_0
+test
+trackID=1
 ucast/11
+udp/av0_0
 udp/unicast/aiphone_H264
-user_defined
+udpstream
 user.pin.mp2
+user=admin&password=&channel=1&stream=0.sdp?
+user=admin&password=&channel=1&stream=0.sdp?real_stream
+user=admin_password=?????_channel=1_stream=0.sdp?real_stream
+user=admin_password=R5XFY888_channel=1_stream=0.sdp?real_stream
+user_defined
+v2
 video
 video.3gp
 video.h264
@@ -107,11 +158,10 @@ video.mp4
 video.pro1
 video.pro2
 video.pro3
+video0.sdp
 video1
 video1+audio1
-VideoInput/1/h264/1
-VideoInput/1/h264/1
-VideoInput/1/mpeg4/1
 videoMain
+videoinput_1/h264_1/media.stm
 vis
 wfov

tools/automatic_route_dictionary_updater/main.go

@@ -0,0 +1,188 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/ullaakut/disgo/style"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/ullaakut/disgo"
+	"github.com/vbauerster/mpb"
+	"github.com/vbauerster/mpb/decor"
+)
+
+const dictionaryURL = "https://community.geniusvision.net/platform/cprndr/manulist"
+
+var rtspURLsFound sync.Map
+
+func main() {
+	if err := updateDictionary(); err != nil {
+		log.Fatalf(err.Error())
+	}
+}
+
+func updateDictionary() error {
+	disgo.SetTerminalOptions(disgo.WithColors(true), disgo.WithDebug(true))
+
+	disgo.StartStep("Fetching dictionary list")
+	resp, err := http.Get(dictionaryURL)
+	if err != nil {
+		return disgo.FailStepf("unable to download dictionaries: %v", err)
+	}
+	defer resp.Body.Close()
+
+	disgo.StartStep("Parsing dictionary list")
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return disgo.FailStepf("unable to read from dictionary list: %v", err)
+	}
+
+	var vendorURLs []string
+	doc.Find("td.simpletable a").Each(func(i int, s *goquery.Selection) {
+		url, ok := s.Attr("href")
+		if !ok {
+			return
+		}
+
+		if url != "javascript:void(0)" {
+			vendorURLs = append(vendorURLs, url)
+		}
+	})
+
+	disgo.StartStep("Loading current cameradar dictionary")
+	currentDictionary, err := ioutil.ReadFile("dictionaries/routes")
+	if err != nil {
+		return disgo.FailStepf("unable to read current dictionary: %v", err)
+	}
+
+	dictionaryEntries := bytes.Split(currentDictionary, []byte("\n"))
+
+	for _, rtspURL := range dictionaryEntries {
+		rtspURLsFound.Store(string(rtspURL), struct{}{})
+	}
+
+	disgo.Debugf("Current dictionary has %d entries\n", len(dictionaryEntries))
+	disgo.EndStep()
+
+	p := mpb.New(mpb.WithWidth(64))
+	name := fmt.Sprintf("Fetching default routes from %d constructors:", len(vendorURLs))
+	bar := p.AddBar(int64(len(vendorURLs)),
+		// set custom bar style, default one is "[=>-]"
+		mpb.BarStyle("╢▌▌░╟"),
+		mpb.PrependDecorators(
+			// display our name with one space on the right
+			decor.Name(name, decor.WC{W: len(name), C: decor.DidentRight}),
+		),
+		mpb.AppendDecorators(decor.Percentage()),
+	)
+
+	for _, url := range vendorURLs {
+		go loadRoutes(url, bar)
+	}
+
+	p.Wait()
+
+	disgo.StartStep("Converting found routes into proper data model")
+
+	var rtspURLs []string
+	rtspURLsFound.Range(func(rtspURL, _ interface{}) bool {
+		disgo.Infoln("Adding URL", rtspURL.(string))
+		rtspURLs = append(rtspURLs, rtspURL.(string))
+		return true
+	})
+
+	sort.Slice(rtspURLs, func(a, b int) bool {
+		return rtspURLs[a] < rtspURLs[b]
+	})
+
+	disgo.EndStep()
+
+	if len(dictionaryEntries) < len(rtspURLs) {
+		disgo.Infof("%s Saving them in cameradar default dictionary.\n", style.Success("Found ", len(rtspURLs)-len(dictionaryEntries), " new entries!"))
+		saveRoutes(rtspURLs)
+	} else {
+		disgo.Infoln(style.Success("No new entry found, dictionary up-to-date! :)"))
+	}
+
+	return nil
+}
+
+func loadRoutes(url string, bar *mpb.Bar) {
+	defer bar.IncrBy(1)
+
+	var (
+		failureCounter int
+		resp           *http.Response
+		err            error
+	)
+	for failureCounter < 5 {
+		resp, err = http.Get(url)
+		if err != nil {
+			failureCounter++
+		} else {
+			break
+		}
+	}
+
+	if failureCounter == 5 {
+		disgo.Errorln("Request failed 5 times in a row, giving up on this vendor")
+		return
+	}
+
+	defer resp.Body.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		disgo.Errorf("unable to read from dictionary list for URL %q: %v\n", url, err)
+		return
+	}
+
+	doc.Find("tr.simpletable td.simpletable:nth-child(4) a").Each(func(i int, s *goquery.Selection) {
+		rtspURL := s.Text()
+
+		if strings.HasPrefix(rtspURL, "(") && strings.HasSuffix(rtspURL, ")") {
+			return
+		}
+
+		if strings.HasPrefix(rtspURL, "[") && strings.HasSuffix(rtspURL, "]") {
+			return
+		}
+
+		if strings.HasPrefix(rtspURL, "http://") {
+			return
+		}
+
+		// Skip the port and only get the route.
+		if strings.HasPrefix(rtspURL, "rtsp://ip-addr:") {
+			routeAndPort := strings.TrimSpace(strings.TrimPrefix(rtspURL, "rtsp://ip-addr:"))
+			route := strings.TrimLeft(routeAndPort, "0123456789/")
+			rtspURLsFound.Store(route, struct{}{})
+			return
+		}
+
+		switch rtspURL {
+		case "", "[Details]", "rtsp://ip-addr/", "http://ip-addr/":
+			return
+		default:
+			route := strings.TrimSpace(strings.TrimPrefix(rtspURL, "rtsp://ip-addr/"))
+			rtspURLsFound.Store(route, struct{}{})
+		}
+	})
+}
+
+func saveRoutes(rtspURLs []string) {
+	contents := strings.Join(rtspURLs, "\n")
+
+	disgo.StartStep("Writing new dictionary file")
+	err := ioutil.WriteFile("dictionaries/routes", []byte(contents), 0644)
+	if err != nil {
+		disgo.FailStepf("unable to write dictionnary: %v", err)
+	}
+}