Initial commit

cameradar_standalone/src/tasks/brutelogs.cpp

@@ -0,0 +1,122 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <tasks/brutelogs.h>
+#include <cachemanager.h>
+
+namespace etix {
+namespace cameradar {
+
+static const std::string no_ids_warning_ =
+    "The ids.json files' default paths didn't match with the discovered "
+    "cameras. Either "
+    "they have custom ids, or your ids.json file does not contain enough "
+    "default routes. "
+    "Path bruteforce is impossible without the IDs.";
+
+//! Tries to match the detected combination of Username / Password
+//! with the camera stream. Creates a resource in the DB upon
+//! valid discovery
+bool
+brutelogs::test_ids(const etix::cameradar::stream_model& stream,
+                    const std::string& password,
+                    const std::string& username) const {
+    bool found = false;
+    std::string path = stream.service_name + "://";
+    if (username != "" || password != "") { path += username + ":" + password + "@"; }
+    path += stream.address + ":" + std::to_string(stream.port);
+    LOG_DEBUG_("Testing ids : " + path, "bruteforce");
+    try {
+        if (curl_describe(path, true)) {
+            LOG_DEBUG_("[FOUND IDS] : " + path, "bruteforce");
+            found = true;
+            stream_model newstream{
+                stream.address, stream.port,         username,       password,
+                stream.route,   stream.service_name, stream.product, stream.protocol,
+                stream.state,   stream.path_found,   true,           stream.thumbnail_path
+            };
+            (*cache)->update_stream(newstream);
+        } else {
+            stream_model newstream{ stream.address,    stream.port,     username,
+                                    password,          stream.route,    stream.service_name,
+                                    stream.product,    stream.protocol, stream.state,
+                                    stream.path_found, false,           stream.thumbnail_path };
+            (*cache)->update_stream(newstream);
+        }
+    } catch (const std::runtime_error& e) {
+        LOG_DEBUG_("Ids already tested : " + std::string(e.what()), "bruteforce");
+    }
+    return found;
+}
+
+bool
+ids_already_found(std::vector<stream_model> streams, stream_model stream) {
+    for (const auto& it : streams) {
+        if ((stream.address == it.address) && (stream.port == it.port) && it.ids_found) return true;
+    }
+    return false;
+}
+
+//! Tries to discover the right IDs on all RTSP streams in DB
+//! Uses the ids.json file to try different combinations
+bool
+brutelogs::run() const {
+    LOG_INFO_(
+        "Beginning bruteforce of the usernames and passwords task, it may "
+        "take a while.",
+        "bruteforce");
+    std::vector<etix::cameradar::stream_model> streams = (*cache)->get_streams();
+    bool doubleskip;
+    size_t found = 0;
+    for (const auto& stream : streams) {
+        doubleskip = false;
+        if (signal_handler::instance().should_stop() != etix::cameradar::stop_priority::running)
+            break;
+        if ((found < streams.size()) && ids_already_found(streams, stream)) {
+            LOG_INFO_(stream.address +
+                          " : This camera's ids were already discovered in "
+                          "the database. Skipping to "
+                          "the next camera.",
+                      "bruteforce");
+            ++found;
+        } else {
+            for (const auto& username : conf.usernames) {
+                if (doubleskip ||
+                    signal_handler::instance().should_stop() !=
+                        etix::cameradar::stop_priority::running)
+                    break;
+                for (const auto& password : conf.passwords) {
+                    if (doubleskip ||
+                        signal_handler::instance().should_stop() !=
+                            etix::cameradar::stop_priority::running)
+                        break;
+                    if (test_ids(stream, password, username)) {
+                        ++found;
+                        doubleskip = true;
+                    }
+                }
+            }
+        }
+    }
+    if (!found) {
+        LOG_WARN_(no_ids_warning_, "bruteforce");
+        return false;
+    } else
+        LOG_INFO_("Found " + std::to_string(found) + " ids for " + std::to_string(streams.size()) +
+                      " cameras",
+                  "bruteforce");
+    return true;
+}
+}
+}

cameradar_standalone/src/tasks/brutepath.cpp

@@ -0,0 +1,103 @@
+// Copyright (C) 2016 Etix Labs - All Rights Reserved.
+// All information contained herein is, and remains the property of Etix Labs
+// and its suppliers,
+// if any. The intellectual and technical concepts contained herein are
+// proprietary to Etix Labs
+// Dissemination of this information or reproduction of this material is
+// strictly forbidden unless
+// prior written permission is obtained from Etix Labs.
+
+#include <tasks/brutepath.h>
+
+namespace etix {
+namespace cameradar {
+
+static const std::string no_route_found_ =
+    "The url.json files' default paths didn't match with the discovered "
+    "cameras. Either "
+    "they have a custom path, or your url.json file does not contain enough "
+    "default "
+    "routes. Thumbnail generation is impossible without the path.";
+
+//! Tries to match the detected combination of Username / Password
+//! with a route for the camera stream. Creates a resource in the DB upon
+//! valid discovery
+bool
+brutepath::test_path(const stream_model& stream, const std::string& route) const {
+    bool found = false;
+    std::string path = stream.service_name + "://" + stream.username + ":" + stream.password + "@" +
+                       stream.address + ":" + std::to_string(stream.port);
+    if (route.front() != '/') { path += "/"; }
+    path += route;
+    LOG_DEBUG_("Testing path : " + path, "brutepath");
+    try {
+        if (curl_describe(path, false)) {
+            // insert in DB and go to the next port, print a cool message
+            found = true;
+            LOG_INFO_("Discovered a valid path : [" + path + "]", "brutepath");
+            stream_model newstream{
+                stream.address,      stream.port,          stream.username, stream.password, route,
+                stream.service_name, stream.product,       stream.protocol, stream.state,    true,
+                stream.ids_found,    stream.thumbnail_path
+            };
+            (*cache)->update_stream(newstream);
+        } else {
+            stream_model newstream{
+                stream.address,      stream.port,          stream.username, stream.password, route,
+                stream.service_name, stream.product,       stream.protocol, stream.state,    false,
+                stream.ids_found,    stream.thumbnail_path
+            };
+            (*cache)->update_stream(newstream);
+        }
+    } catch (const std::runtime_error& e) { LOG_INFO_(e.what(), "brutepath"); }
+    return found;
+}
+
+bool
+path_already_found(std::vector<stream_model> streams, stream_model model) {
+    for (const auto& stream : streams) {
+        if ((model.address == stream.address) && (model.port == stream.port) && stream.path_found)
+            return true;
+    }
+    return false;
+}
+
+//! Tries to discover a route on all RTSP streams in DB
+//! Uses the url.json file to try different routes
+bool
+brutepath::run() const {
+    LOG_INFO_("Beginning bruteforce of the camera paths task, it may take a while.", "bruteforce");
+    std::vector<stream_model> streams = (*cache)->get_streams();
+    int found = 0;
+    for (const auto& stream : streams) {
+        if (signal_handler::instance().should_stop() != etix::cameradar::stop_priority::running)
+            break;
+        if (path_already_found(streams, stream)) {
+            LOG_INFO_(stream.address +
+                          " : This camera's path was already discovered in the database."
+                          "Skipping to the next camera.",
+                      "brutepath");
+            ++found;
+        } else {
+            for (const auto& route : conf.paths) {
+                if (signal_handler::instance().should_stop() !=
+                    etix::cameradar::stop_priority::running)
+                    break;
+                if (test_path(stream, route)) {
+                    found++;
+                    break;
+                }
+            }
+        }
+    }
+    if (!found) {
+        LOG_WARN_(no_route_found_, "brutepath");
+
+    } else
+        LOG_INFO_("Found " + std::to_string(found) + " routes for " +
+                      std::to_string(streams.size()) + " cameras",
+                  "brutepath");
+    return true;
+}
+}
+}

cameradar_standalone/src/tasks/mapping.cpp

@@ -0,0 +1,59 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <tasks/mapping.h>
+
+namespace etix {
+namespace cameradar {
+
+//! The first command checks if dpkg finds nmap in the system by cutting the
+//! result and grepping
+//! nmap from it.
+//!
+//! The second command checks the version of nmap, right now it needs to be the
+//! 6.47 but this could
+//! be changed to 6 or greater depending on the needs. In a docker container
+//! this should not be a
+//! problem.
+bool
+nmap_is_ok() {
+    return (launch_command("test `dpkg -l | cut -c 5-9 | grep nmap` = nmap")
+    // && launch_command("test `nmap --version | cut -c 14-18  | head -n2 | tail -n1` = 6.47")
+    && launch_command("mkdir -p scans")); // Creates the directory in which the scans will be stored
+}
+
+//! Launches and checks the return of the nmap command
+//! Uses the subnets specified in the conf file to launch nmap
+bool
+mapping::run() const {
+    if (nmap_is_ok()) {
+        std::string subnets = this->conf.subnets;
+        std::replace(subnets.begin(), subnets.end(), ',', ' ');
+        LOG_INFO_("Nmap 6.0 or greater found", "mapping");
+        LOG_INFO_("Beginning mapping task. This may take a while.", "mapping");
+        std::string cmd =
+            "nmap -T4 -A " + subnets + " -p " + this->conf.ports + " -oX " + nmap_output;
+        bool ret = launch_command(cmd);
+        if (ret)
+            LOG_INFO_("Nmap XML output successfully generated in file: " + nmap_output, "mapping");
+        else
+            LOG_ERR_("Nmap command failed", "mapping");
+        return ret;
+    } else {
+        LOG_ERR_("Nmap 6.0 or greater is required to launch Cameradar", "mapping");
+        return false;
+    }
+}
+}
+}

cameradar_standalone/src/tasks/parsing.cpp

@@ -0,0 +1,122 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <tasks/parsing.h>
+
+namespace etix {
+namespace cameradar {
+
+static const std::string no_hosts_found_ =
+    "No hosts were discovered on your network. Please check your internet "
+    "connexion "
+    "and verify that the subnetworks you specified in your configuration file "
+    "were "
+    "accessible";
+
+//! Avoids segfaults on unknown xml structure
+std::string
+xml_safe_get(const TiXmlElement* elem, const std::string& attr) {
+    if (elem == nullptr) return "Closed";
+    if (elem->Attribute(attr.c_str()) != nullptr) return std::string(elem->Attribute(attr.c_str()));
+    return "Closed";
+}
+
+//! Parse a single host node (generally containing only one camera)
+//! Pushes it back to the data structure
+void
+parsing::parse_camera(TiXmlElement* xml_host, std::vector<stream_model>& data) const {
+    TiXmlElement* xml_streams = xml_host->FirstChild("ports")->ToElement();
+    stream_model stream;
+    for (TiXmlElement* xml_stream = xml_streams->FirstChild("port")->ToElement(); xml_stream;
+         xml_stream = xml_stream->NextSiblingElement("port")) {
+        stream.address = xml_safe_get(xml_host->FirstChild("address")->ToElement(), "addr");
+        stream.protocol = xml_safe_get(xml_stream, "protocol");
+        stream.port = static_cast<unsigned short>(std::stoi(xml_safe_get(xml_stream, "portid")));
+        TiXmlElement* state = xml_stream->FirstChild("state")->ToElement();
+        stream.state = xml_safe_get(state, "state");
+        TiXmlElement* service;
+        if (state->NextSibling("service") &&
+            (service = state->NextSibling("service")->ToElement())) {
+            stream.service_name = xml_safe_get(service, "name");
+            stream.product = xml_safe_get(service, "product");
+        } else {
+            stream.service_name = "Closed";
+            stream.product = "Closed";
+        }
+        data.push_back(stream);
+    }
+}
+
+//! Prints all detected cameras into the data structure and stops the program if
+//! no open RTSP streams were found
+bool
+parsing::print_detected_cameras(const std::vector<stream_model>& data) const {
+    int added = 0;
+    for (const auto& stream : data) {
+        if (!stream.service_name.compare("rtsp") && !stream.state.compare("open")) {
+            try {
+                LOG_INFO_("Generated JSON Result : " + deserialize(stream).toStyledString(),
+                          "print");
+                added++;
+            } catch (const std::runtime_error& e) {
+                LOG_WARN_("Port already scanned : " + std::string(e.what()), "parsing");
+                added++;
+            }
+        }
+    }
+    if (!added) {
+        LOG_WARN_(
+            "Mapping unsuccessful, no rtsp streams were discovered. You "
+            "should try other "
+            "subnetworks",
+            "parsing");
+        return false;
+    }
+    LOG_INFO_("Mapping successfuly ended, " + std::to_string(added) +
+                  " RTSP streams were discovered.",
+              "parsing");
+    (*cache)->set_streams(data);
+    return true;
+}
+
+//! Opens the nmap output file, parses the data of each discovered port
+//! Adds the RTSP ports only into the DB
+bool
+parsing::run() const {
+    std::vector<stream_model> data;
+    try {
+        TiXmlDocument doc(nmap_output.c_str());
+        doc.LoadFile();
+        TiXmlHandle docHandle(&doc);
+
+        TiXmlElement* nmaprun = docHandle.FirstChild("nmaprun").ToElement();
+        TiXmlNode* xml_node = nmaprun->FirstChild("host");
+        if (xml_node == NULL) return false;
+        TiXmlElement* xml_host;
+        if ((xml_host = xml_node->ToElement()) && xml_host->Attribute("endtime"))
+            for (xml_host = xml_node->ToElement(); xml_host;
+                 xml_host = xml_host->NextSiblingElement("host")) {
+                parse_camera(xml_host, data);
+            }
+        else
+            LOG_WARN_(no_hosts_found_, "parsing");
+        if (data.size() == 0) { LOG_WARN_("No cameras were discovered", "parsing"); }
+        return print_detected_cameras(data);
+    } catch (std::exception& e) {
+        LOG_ERR_("Error during parsing. brutepath aborted : " + std::string(e.what()), "parsing");
+        return false;
+    }
+}
+}
+}

cameradar_standalone/src/tasks/print.cpp

@@ -0,0 +1,46 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <tasks/print.h>
+
+namespace etix {
+namespace cameradar {
+
+//! Launches and checks the return of the nmap command
+//! Uses the subnets specified in the conf file to launch nmap
+bool
+print::run() const {
+    std::vector<stream_model> results = (*cache)->get_valid_streams();
+    std::ofstream file;
+    bool first = true;
+    file.open("result.json");
+    file << "[\n";
+    for (const auto& stream : results) {
+        LOG_INFO_("Found a valid RTSP Stream and generated a thumbnail at : " +
+                      stream.thumbnail_path,
+                  "print");
+
+        if (first)
+            first = false;
+        else
+            file << ",";
+        file << deserialize(stream).toStyledString();
+        LOG_INFO_("Generated JSON Result : " + deserialize(stream).toStyledString(), "print");
+    }
+    file << "\n]";
+    file.close();
+    return true;
+}
+}
+}

cameradar_standalone/src/tasks/stream_check.cpp

@@ -0,0 +1,66 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <tasks/stream_check.h>
+
+namespace etix {
+namespace cameradar {
+
+//! Gets all the discovered streams with good routes and logs
+//! And launches an ffmpeg command to generate a thumbnail
+//! In order to check for the stream validity
+bool
+stream_check::run() const {
+    GstElement* pipeline;
+    GstElement* elem;
+
+    gst_init(nullptr, nullptr);
+
+    std::vector<stream_model> streams = (*cache)->get_valid_streams();
+
+    for (const auto& stream : streams) {
+        GError* error = NULL;
+
+        pipeline =
+            gst_parse_launch("rtspsrc name=source ! rtph264depay ! h264parse ! fakesink", &error);
+
+        if (pipeline == NULL) {
+            LOG_ERR_("[" + stream.address + "] Can't configure pipeline", "stream_check");
+            return false;
+        } else {
+            elem = gst_bin_get_by_name(GST_BIN(pipeline), "source");
+            g_object_set(G_OBJECT(elem), "location", stream.address, "latency", 20, NULL);
+
+            if (gst_element_set_state(pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
+                LOG_ERR_(
+                    "This stream is unaccessible with GStreamer, there must be a "
+                    "configuration issue",
+                    "stream_check");
+                gst_object_unref(pipeline);
+                stream_model invalidstream{
+                    stream.address,   stream.port,         stream.username,  stream.password,
+                    stream.route,     stream.service_name, stream.product,   stream.protocol,
+                    "invalid stream", stream.path_found,   stream.ids_found, stream.thumbnail_path
+                };
+                (*cache)->update_stream(invalidstream);
+                return false;
+            }
+            LOG_INFO_("[" + stream.address + "] Set pipeline to playing", "stream_check");
+        }
+    }
+    LOG_INFO_("All streams could be accessed with GStreamer", "stream_check");
+    return true;
+}
+}
+}

cameradar_standalone/src/tasks/thumbnail.cpp

@@ -0,0 +1,97 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <tasks/thumbnail.h>
+
+namespace etix {
+namespace cameradar {
+
+std::string
+remove_trailing_backslash(std::string s) {
+    while (s.back() == '/') { s.pop_back(); }
+    return s;
+}
+
+// Tranforms the path into a path for the thumbnail
+// Example :
+// rtsp://username:password@172.16.100.13/live.sdp
+// will become /storage/path/172.16.100.13/1345425533.jpg
+std::string
+thumbnail::build_output_file_path(const std::string& path) const {
+    auto ss = std::stringstream{};
+
+    ss << remove_trailing_backslash(this->conf.thumbnail_storage_path);
+    ss << "/";
+    ss << path;
+    ss << "/";
+    ss << std::to_string(std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()));
+    ss << ".jpg";
+
+    return ss.str();
+}
+
+//! Gets all the discovered streams with good routes and logs
+//! And launches an ffmpeg command to generate a thumbnail
+//! In order to check for the stream validity
+bool
+thumbnail::run() const {
+    std::vector<stream_model> streams = (*cache)->get_valid_streams();
+    LOG_INFO_("Started thumbnail generation, it may take a while", "thumbnail");
+    for (const auto& stream : streams) {
+        if (signal_handler::instance().should_stop() != etix::cameradar::stop_priority::running)
+            break;
+        std::string ffmpeg_cmd =
+            "mkdir -p %s ; "
+            "ffmpeg "
+            "-y "
+            "-nostdin "
+            "-loglevel quiet "
+            "-i '%s' "
+            "-vcodec mjpeg "
+            "-vframes 1 "
+            "-an "
+            "-f image2 "
+            "-s 320x240 "
+            "'%s'";
+        std::string fullpath = make_path(stream);
+        std::string output = build_output_file_path(stream.address);
+        ffmpeg_cmd = tool::fmt(ffmpeg_cmd.c_str(),
+                               output.substr(0, output.find_last_of("/")).c_str(),
+                               fullpath.c_str(),
+                               output.c_str());
+        if (!launch_command(ffmpeg_cmd)) {
+            LOG_WARN_("The following command [" + ffmpeg_cmd +
+                          "] didn't work. That can either mean that the stream is "
+                          "not valid or "
+                          "that there is a problem with the camera.",
+                      "thumbnail_generation");
+        } else {
+            LOG_DEBUG_("Generated thumbnail : " + ffmpeg_cmd, "thumbnail_generation");
+            try {
+                stream_model result{ stream.address,    stream.port,      stream.username,
+                                     stream.password,   stream.route,     stream.service_name,
+                                     stream.product,    stream.protocol,  stream.state,
+                                     stream.path_found, stream.ids_found, output };
+                (*cache)->update_stream(result);
+
+            } catch (std::exception& e) { LOG_DEBUG_(e.what(), "thumbnail_generation"); }
+        }
+    }
+    LOG_INFO_("All thumbnails have been successfully generated in " +
+                  this->conf.thumbnail_storage_path,
+              "thumbnail_generation");
+    return true;
+}
+}
+}