Quick MySQL docker deployment & code cleaning

CMakeLists.txt

@@ -20,7 +20,7 @@ set (PROJECT_NAME cameradar)
 project (${PROJECT_NAME})
 
 set (${PROJECT_NAME}_VERSION_MAJOR 0)
-set (${PROJECT_NAME}_VERSION_MINOR 1)
+set (${PROJECT_NAME}_VERSION_MINOR 2)
 set (${PROJECT_NAME}_VERSION_PATCH 0)
 set (${PROJECT_NAME}_SUFFIX "-beta")
 set (${PROJECT_NAME}_VERSION "${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH}${${PROJECT_NAME}_SUFFIX}")
@@ -83,29 +83,36 @@ configure_file (
     "${PROJECT_BINARY_DIR}/version.h"
 )
 
-include_directories (
+# add all deps libraries to the link directories path
-        "cameradar_standalone/include"
+link_directories (
-        "deps/jsoncpp/src/deps.jsoncpp/include"
+    # third party libraries
-        "deps/boost/src/deps.boost/include"
+    "deps/jsoncpp/src/deps.jsoncpp/src/lib_json"
+    "deps/boost/src/deps.boost/libs"
+    "deps/mysql-connector/lib"
 )
 
+include_directories (
+    "cameradar_standalone/include"
+    "deps/jsoncpp/src/deps.jsoncpp/include"
+    "deps/boost/src/deps.boost/include"
+    "deps/mysql-connector/include"
+)
+
+set (${CAMERADAR_BINARIES} "")
+set (${CAMERADAR_LIBRARIES} "")
+
 #build cache managers
 add_subdirectory (deps)
 add_subdirectory (cameradar_standalone)
 add_subdirectory (cache_managers)
 
-set (${CAMERADAR_BINARIES} "")
-install (PROGRAMS ${CAMERADAR_BINARIES} DESTINATION bin)
-
-install (FILES ${CAMERADAR_CACHE_MANAGERS} DESTINATION cache_managers)
-
-set (${CAMERADAR_LIBRARIES} "")
 list (APPEND CAMERADAR_LIBRARIES ${CAMERADAR_INSTALL_DEPENDENCIES} ${CAMERADAR_LIBRARIES})
+
+install (PROGRAMS ${CAMERADAR_BINARIES} DESTINATION bin)
+install (FILES ${CAMERADAR_CACHE_MANAGERS} DESTINATION cache_managers)
 install (FILES ${CAMERADAR_LIBRARIES} DESTINATION libraries)
-
 install (DIRECTORY ${CMAKE_SOURCE_DIR}/deps/licenses DESTINATION libraries)
 
-
 # cpack configuration
 include (InstallRequiredSystemLibraries)
 set (CPACK_PACKAGE_DESCRIPTION_SUMMARY "cameradar")

README.md

@@ -38,7 +38,7 @@ Of course, you can also call for individual tasks if you plug in a Database to C
 
 ## Quick install
 
-The quick install uses docker to build Cameradar without polluting your machine with dependencies and makes it easy to deploy Cameradar in a few commands.
+The quick install uses docker to build Cameradar without polluting your machine with dependencies and makes it easy to deploy Cameradar in a few commands. **However, it may require networking knowledge, as your docker containers will need access to the cameras subnetwork.**
 
 ### Dependencies
 
@@ -46,7 +46,7 @@ The only dependencies are `docker` and `docker-compose`.
 
 ### Five steps guide
 
-1. `https://github.com/EtixLabs/cameradar.git`
+1. `git clone https://github.com/EtixLabs/cameradar.git`
 2. Go into the Cameradar repository, then to the `deployment` directory
 3. Tweak the `conf/cameradar.conf.json` as you need (see [the onfiguration guide here](#configuration) for more information)
 4. Run `docker-compose build cameradar` to build the cameradar container
@@ -63,15 +63,12 @@ The manual installation is recommended if you want to tweak Cameradar and quickl
 To install Cameradar you will need these packages
 
 * cmake (`cmake`)
-* gstreamer1.x (`libgstreamer1.0-dev`)
-* ffmpeg (`ffmpeg`)
-* libcurl (`libcurl4-openssl-dev`)
 
 ### Steps
 
 The simplest way would be to follow these steps :
 
-1. `https://github.com/EtixLabs/cameradar.git`
+1. `git clone https://github.com/EtixLabs/cameradar.git`
 2. Go into the Cameradar repository, create a directory named `build` and go in it
 3. In the build directory, run `cmake ..` This will generate the Makefiles you need to build Cameradar
 4. Run the command `make`
@@ -100,6 +97,13 @@ The only dependencies are `docker` and `docker-compose`.
 Here is the basic content of the configuration file with simple placeholders :
 ```json
 {
+  "mysql_db" : {
+     "host" : "MYSQL_SERVER_IP_ADDRESS",
+     "port" : MYSQL_SERVER_PORT,
+     "user": "root",
+     "password": "root",
+     "db_name": "cmrdr"
+  },
   "subnets" : "SUBNET1,SUBNET2,SUBNET3,[...]",
   "ports" : "PORT1,PORT2,[...]",
   "rtsp_url_file" : "conf/url.json",
@@ -206,7 +210,7 @@ If you have other cool ideas, feel free to share them with me at brendan.leglaun
 ## Next improvements
 
 - [x] Add a docker deployment to avoid the current deps hell
-- [ ] Development of a MySQL cache manager
+- [x] Development of a MySQL cache manager
 - [ ] Development of a JSON file cache manager
 - [ ] Development of an XML file cache manager
 

cache_managers/CMakeLists.txt

@@ -19,5 +19,6 @@ cmake_policy(SET CMP0042 NEW)
 set (LIBRARY_OUTPUT_PATH ${CAMERADAR_CACHE_MANAGER_OUTPUT_PATH})
 
 add_subdirectory(dumb_cache_manager)
+add_subdirectory(mysql_cache_manager)
 
 set (CAMERADAR_CACHE_MANAGERS ${CAMERADAR_CACHE_MANAGERS} PARENT_SCOPE)

cache_managers/dumb_cache_manager/CMakeLists.txt

@@ -20,7 +20,6 @@ project(dumb_cache_manager CXX)
 find_package(PkgConfig)
 
 include_directories (${PROJECT_SOURCE_DIR}/include ${CAMERADAR_INCLUDES})
-message("${CAMERADAR_INCLUDES}")
 
 include (find_sources)
 find_sources ("src" "include")

cache_managers/dumb_cache_manager/include/dumb_cache_manager.h

@@ -14,11 +14,11 @@
 
 #pragma once
 
-#include <vector>
 #include <cachemanager.h>
-#include <stream_model.h>
 #include <configuration.h>
 #include <logger.h>
+#include <stream_model.h>
+#include <vector>
 
 namespace etix {
 namespace cameradar {
@@ -42,9 +42,9 @@ public:
 
     void update_stream(const etix::cameradar::stream_model& newmodel);
 
-    std::vector<etix::cameradar::stream_model> get_streams() const;
+    std::vector<etix::cameradar::stream_model> get_streams();
 
-    std::vector<etix::cameradar::stream_model> get_valid_streams() const;
+    std::vector<etix::cameradar::stream_model> get_valid_streams();
 };
 }
 }

cache_managers/dumb_cache_manager/src/dumb_cache_manager.cpp

@@ -60,7 +60,7 @@ dumb_cache_manager::update_stream(const etix::cameradar::stream_model& newmodel)
 
 //! Gets all cached streams
 std::vector<etix::cameradar::stream_model>
-dumb_cache_manager::get_streams() const {
+dumb_cache_manager::get_streams() {
     std::vector<stream_model> ret;
     for (const auto& it : this->streams) {
         if (not it.service_name.compare("rtsp") && not it.state.compare("open")) ret.push_back(it);
@@ -70,7 +70,7 @@ dumb_cache_manager::get_streams() const {
 
 //! Gets all valid streams
 std::vector<etix::cameradar::stream_model>
-dumb_cache_manager::get_valid_streams() const {
+dumb_cache_manager::get_valid_streams() {
     std::vector<stream_model> ret;
     for (const auto& it : this->streams) {
         if ((not it.service_name.compare("rtsp") && not it.state.compare("open")) && it.ids_found &&

cache_managers/mysql_cache_manager/CMakeLists.txt

@@ -0,0 +1,33 @@
+## Copyright 2016 Etix Labs
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+cmake_minimum_required (VERSION 2.8.1)
+cmake_policy(SET CMP0042 NEW)
+
+project(mysql_cache_manager CXX)
+
+find_package(PkgConfig)
+
+include_directories (${PROJECT_SOURCE_DIR}/include ${CAMERADAR_INCLUDES})
+
+include (find_sources)
+find_sources ("src" "include")
+
+add_library (mysql_cache_manager SHARED ${SOURCES})
+set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--no-undefined")
+target_link_libraries (mysql_cache_manager jsoncpp mysqlcppconn pthread)
+
+set (CACHE_MANAGER_NAME ${CAMERADAR_CACHE_MANAGER_OUTPUT_PATH}/${CMAKE_SHARED_LIBRARY_PREFIX}mysql_cache_manager${CMAKE_SHARED_LIBRARY_SUFFIX})
+list (APPEND CAMERADAR_CACHE_MANAGERS ${CACHE_MANAGER_NAME})
+set (CAMERADAR_CACHE_MANAGERS ${CAMERADAR_CACHE_MANAGERS} PARENT_SCOPE)

cache_managers/mysql_cache_manager/include/db_conn.h

@@ -0,0 +1,85 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <cppconn/resultset.h> // for ResultSet
+#include <mutex>               // for mutex
+#include <stdbool.h>           // for bool, false
+#include <string>              // for string
+#include <utility>             // for pair, make_pair
+
+#include "query_result.h"
+
+namespace sql {
+class Connection;
+class Driver;
+class ResultSet;
+}
+
+namespace etix {
+
+namespace cameradar {
+
+namespace mysql {
+
+//! MySQL Database connection handling
+//! Abstracts all connection to the database
+class db_connection {
+private:
+    static const std::string create_database_query;
+
+    //! SQL driver
+    sql::Driver* driver = nullptr;
+    //! SQL connection
+    sql::Connection* connection = nullptr;
+    std::mutex access_mtx;
+    bool connected = false;
+
+    std::string db_name;
+
+    //! Create the database if it doesn't exist at connector launch
+    empty_result create_database(void);
+
+public:
+    db_connection(void);
+    ~db_connection(void);
+
+    //! Try to connect to the database
+    std::pair<bool, std::string> connect(const std::string& host,
+                                         const std::string& user,
+                                         const std::string& pass,
+                                         const std::string& db_name,
+                                         bool create_db_if_not_exist = true);
+
+    //! Execute a MySQL command
+    empty_result execute(const std::string& request);
+
+    //! Execute a query
+    query_result<sql::ResultSet*> query(const std::string& query);
+
+    bool is_connected();
+
+    //! Return db_name
+    const std::string&
+    get_db_name(void) const {
+        return this->db_name;
+    }
+};
+
+} // mysql
+
+} // cameradar
+
+} // etix

cache_managers/mysql_cache_manager/include/mysql_cache_manager.h

@@ -0,0 +1,81 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <cachemanager.h>
+#include <configuration.h>
+#include <db_conn.h>
+#include <fmt.h>
+#include <logger.h>
+#include <stream_model.h>
+#include <vector>
+
+namespace etix {
+
+namespace cameradar {
+
+struct mysql_configuration {
+    unsigned int port;
+    std::string host;
+    std::string db_name;
+    std::string user;
+    std::string password;
+
+    mysql_configuration() = default;
+
+    mysql_configuration(unsigned int port,
+                        const std::string& host,
+                        const std::string& db_name,
+                        const std::string& user = "",
+                        const std::string& password = "")
+    : port(port), host(host), db_name(db_name), user(user), password(password) {}
+};
+
+class mysql_cache_manager : public cache_manager_base {
+private:
+    static const std::string name;
+    std::vector<etix::cameradar::stream_model> streams;
+    std::shared_ptr<etix::cameradar::configuration> configuration;
+    etix::cameradar::mysql_configuration db_conf;
+    etix::cameradar::mysql::db_connection connection;
+
+    static const std::string create_table_query;
+    static const std::string insert_with_id_query;
+    static const std::string exist_query;
+    static const std::string get_results_query;
+    static const std::string update_result_query;
+
+public:
+    using cache_manager_base::cache_manager_base;
+    ~mysql_cache_manager();
+
+    // Specific to MySQL
+    bool execute_query(const std::string& query);
+
+    const std::string& get_name() const override;
+    static const std::string& static_get_name();
+    bool load_mysql_conf(std::shared_ptr<etix::cameradar::configuration> configuration);
+    bool configure(std::shared_ptr<etix::cameradar::configuration> configuration) override;
+
+    void set_streams(std::vector<etix::cameradar::stream_model> model);
+
+    void update_stream(const etix::cameradar::stream_model& newmodel);
+
+    std::vector<etix::cameradar::stream_model> get_streams();
+
+    std::vector<etix::cameradar::stream_model> get_valid_streams();
+};
+}
+}

cache_managers/mysql_cache_manager/include/query_result.h

@@ -0,0 +1,65 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+namespace etix {
+
+namespace cameradar {
+
+namespace mysql {
+
+enum class execute_result { success, not_found, no_row_updated, sql_error, error };
+
+//! Wrapper of a DB query result
+//! Templated on the data type we want to return (list<model>, bool, whatever)
+template <typename DataType>
+struct query_result {
+    DataType data;
+    execute_result state;
+    std::string error_msg;
+
+    inline bool
+    success(void) const {
+        return state == execute_result::success;
+    }
+    inline bool
+    error(void) const {
+        return not success();
+    }
+};
+
+//! Empty query result for when we just want to return the status
+//! of the request with no associated data
+template <>
+struct query_result<void> {
+    execute_result state;
+    std::string error_msg;
+
+    inline bool
+    success(void) const {
+        return state == execute_result::success;
+    }
+    inline bool
+    error(void) const {
+        return not success();
+    }
+};
+typedef query_result<void> empty_result;
+
+} //! mysql
+
+} //! cameradar
+
+} //! etix

cache_managers/mysql_cache_manager/src/db_conn.cpp

@@ -0,0 +1,139 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "db_conn.h"            // for db_connection
+#include "cppconn/connection.h" // for Connection
+#include "query_result.h"       // for queries
+#include <cppconn/driver.h>     // for get_driver_instance, etc
+#include <cppconn/exception.h>  // for SQLException
+#include <cppconn/statement.h>  // for Statement
+#include <fmt.h>                // for fmt
+#include <logger.h>             // for LOG_
+
+namespace etix {
+
+namespace cameradar {
+
+namespace mysql {
+
+const std::string db_connection::create_database_query = "CREATE DATABASE IF NOT EXISTS %s";
+
+db_connection::db_connection() : connected(false) {}
+
+db_connection::~db_connection() { delete this->connection; }
+
+std::pair<bool, std::string>
+db_connection::connect(const std::string& host,
+                       const std::string& user,
+                       const std::string& pass,
+                       const std::string& db_name,
+                       bool create_db_if_not_exist) {
+    this->db_name = db_name;
+
+    try {
+        this->driver = get_driver_instance();
+        if (this->driver == nullptr) {
+            return std::make_pair(false, "Cannot instantiate sql_driver");
+        }
+        this->connection = driver->connect(host, user, pass);
+        if (this->connection == nullptr) return std::make_pair(false, "Cannot connect to mysql");
+
+        this->connected = true;
+        if (create_db_if_not_exist) {
+            auto cdb = this->create_database();
+            if (cdb.state == mysql::execute_result::sql_error) { return { false, cdb.error_msg }; }
+            this->connection->setSchema(db_name);
+        }
+    } catch (sql::SQLException& e) {
+        this->connected = false;
+        return { false, e.what() };
+    }
+
+    return std::make_pair(true, "");
+}
+
+empty_result
+db_connection::execute(const std::string& request) {
+    std::lock_guard<std::mutex> lock(this->access_mtx);
+
+    sql::Statement* stmt = nullptr;
+    empty_result return_value = { execute_result::success, "" };
+
+    if (!this->is_connected()) {
+        return { execute_result::sql_error, "Error, not connected to MySQL database" };
+    }
+
+    try {
+        stmt = this->connection->createStatement();
+        stmt->execute(request);
+        if (stmt->getUpdateCount() == 0) {
+            return_value = { execute_result::no_row_updated, "No row updated" };
+        }
+    } catch (sql::SQLException& e) { return_value = { execute_result::sql_error, e.what() }; }
+    if (stmt) { delete stmt; }
+
+    return return_value;
+}
+
+query_result<sql::ResultSet*>
+db_connection::query(const std::string& query) {
+    std::lock_guard<std::mutex> lock(this->access_mtx);
+
+    sql::Statement* stmt = nullptr;
+    query_result<sql::ResultSet*> return_value = { nullptr, execute_result::success, "" };
+
+    if (!this->is_connected()) {
+        return { nullptr, execute_result::sql_error, "Error, not connected to MySQL database" };
+    }
+
+    try {
+        stmt = this->connection->createStatement();
+        return_value = { stmt->executeQuery(query), execute_result::success, "" };
+    } catch (sql::SQLException& e) {
+        return_value = { nullptr, execute_result::sql_error, e.what() };
+    }
+
+    if (stmt) { delete stmt; }
+
+    return return_value;
+}
+
+bool
+db_connection::is_connected() {
+    if (this->connection == nullptr) return false;
+
+    // check if our connection is always valid
+    if (this->connection->isClosed() || not this->connection->isValid()) {
+        LOG_INFO_("MySQL database connection is either closed or invalid, try to reconnect.",
+                  "db_connection");
+        this->connection->reconnect();
+        if (this->connection->isClosed() || not this->connection->isValid()) {
+            this->connected = false;
+            LOG_ERR_("Unable to reconnect to MySQL.", "db_connection");
+        }
+    }
+    return this->connected;
+}
+
+empty_result
+db_connection::create_database() {
+    auto query = tool::fmt(this->create_database_query, this->db_name.c_str());
+    return this->execute(query);
+}
+
+} // mysql
+
+} // cameradar
+
+} // etix

cache_managers/mysql_cache_manager/src/mysql_cache_manager.cpp

@@ -0,0 +1,269 @@
+// Copyright 2016 Etix Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <mysql_cache_manager.h>
+
+/* DATA FORMAT
+**
+**    Example :
+**
+**   "address" : "173.16.100.45",
+**   "ids_found" : true,
+**   "password" : "123456",
+**   "path_found" : true,
+**   "port" : 554,
+**   "product" : "Vivotek FD9381-HTV",
+**   "protocol" : "tcp",
+**   "route" : "/live.sdp",
+**   "service_name" : "rtsp",
+**   "state" : "open",
+**   "thumbnail_path" : "/tmp/127.0.0.1/1463735257.jpg",
+**   "username" : "admin"
+**
+*/
+
+namespace etix {
+
+namespace cameradar {
+
+const std::string mysql_cache_manager::create_table_query =
+    "CREATE TABLE IF NOT EXISTS `results` ("
+    "`id` int(11) UNSIGNED NOT NULL AUTO_INCREMENT, "
+    "`address` tinytext NOT NULL, "
+    "`password` tinytext NOT NULL, "
+    "`product` tinytext NOT NULL, "
+    "`protocol` tinytext NOT NULL, "
+    "`route` tinytext NOT NULL, "
+    "`service_name` tinytext NOT NULL, "
+    "`state` tinytext NOT NULL, "
+    "`thumbnail_path` tinytext NOT NULL, "
+    "`username` tinytext NOT NULL, "
+    "`port` int(11) UNSIGNED NOT NULL, "
+    "`ids_found` tinytext NOT NULL, "
+    "`path_found` tinytext NOT NULL, "
+    "PRIMARY KEY (`id`));";
+
+const std::string mysql_cache_manager::insert_with_id_query =
+    "INSERT INTO `%s`.`results`"
+    " (`address`, `password`, `product`, `protocol`, `route`, `service_name`, `state`, "
+    "`thumbnail_path`, `username`, `port`, `ids_found`, `path_found`)"
+    " VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')";
+
+const std::string mysql_cache_manager::update_result_query =
+    "UPDATE `%s`.`results` SET"
+    " `results`.`address` = '%s',"
+    " `results`.`password` = '%s',"
+    " `results`.`product` = '%s',"
+    " `results`.`protocol` = '%s',"
+    " `results`.`route` = '%s',"
+    " `results`.`service_name` = '%s',"
+    " `results`.`state` = '%s',"
+    " `results`.`thumbnail_path` = '%s',"
+    " `results`.`username` = '%s',"
+    " `results`.`port` = '%s',"
+    " `results`.`ids_found` = '%s',"
+    " `results`.`path_found` = '%s'"
+    " WHERE `results`.`address` LIKE '%s'";
+
+const std::string mysql_cache_manager::exist_query =
+    "SELECT * FROM `%s`.`results` WHERE `results`.`address` = '%s'";
+
+const std::string mysql_cache_manager::get_results_query = "SELECT * FROM `%s`.`results`";
+
+const std::string mysql_cache_manager::name = "mysql-cache-manager";
+
+mysql_cache_manager::~mysql_cache_manager() {}
+
+const std::string&
+mysql_cache_manager::get_name() const {
+    return mysql_cache_manager::static_get_name();
+}
+
+const std::string&
+mysql_cache_manager::static_get_name() {
+    return mysql_cache_manager::name;
+}
+
+bool
+mysql_cache_manager::configure(std::shared_ptr<etix::cameradar::configuration> configuration) {
+    return this->load_mysql_conf(configuration);
+}
+
+bool
+mysql_cache_manager::execute_query(const std::string& query) {
+    auto check_err = [](const auto& res) {
+        if (res.state == mysql::execute_result::sql_error) {
+            LOG_WARN_(res.error_msg, "mysql_cache_manager");
+            return false;
+        }
+        return true;
+    };
+    return check_err(this->connection.execute(query));
+}
+
+bool
+mysql_cache_manager::load_mysql_conf(
+    std::shared_ptr<etix::cameradar::configuration> configuration) {
+    this->configuration = configuration;
+
+    try {
+        this->db_conf.host = configuration->raw_conf["mysql_db"]["host"].asString();
+        this->db_conf.port = configuration->raw_conf["mysql_db"]["port"].asUInt();
+        this->db_conf.user = configuration->raw_conf["mysql_db"]["user"].asString();
+        this->db_conf.password = configuration->raw_conf["mysql_db"]["password"].asString();
+        this->db_conf.db_name = configuration->raw_conf["mysql_db"]["db_name"].asString();
+    } catch (std::exception& e) {
+        LOG_ERR_("Configuration of the MySQL db failed : " + std::string(e.what()),
+                 "mysql_cache_manager");
+        return false;
+    }
+
+    if (not this->connection
+                .connect(db_conf.host + ":" + std::to_string(db_conf.port),
+                         db_conf.user,
+                         db_conf.password,
+                         db_conf.db_name)
+                .first) {
+        LOG_ERR_("Configuration of the MySQL DB failed", "mysql_cache_manager");
+        return false;
+    }
+
+    // Tries to create the Result table in the DB and returns the success state
+    return (execute_query(create_table_query));
+}
+
+//! Replaces all cached streams by the content of the vector given as
+//! parameter
+void
+mysql_cache_manager::set_streams(std::vector<etix::cameradar::stream_model> models) {
+    LOG_DEBUG_("Beginning stream list DB insertion", "mysql_cache_manager");
+    for (const auto& model : models) {
+        auto query = tool::fmt(
+            this->exist_query, this->connection.get_db_name().c_str(), model.address.c_str());
+        auto result = this->connection.query(query);
+        // If an entry already exists for this address in the database,
+        // no need to insert it.
+        if (result.data->next()) return;
+
+        query = tool::fmt(this->insert_with_id_query,
+                          this->connection.get_db_name().c_str(),
+                          model.address.c_str(),
+                          model.password.c_str(),
+                          model.product.c_str(),
+                          model.protocol.c_str(),
+                          model.route.c_str(),
+                          model.service_name.c_str(),
+                          model.state.c_str(),
+                          model.thumbnail_path.c_str(),
+                          model.username.c_str(),
+                          std::to_string(model.port).c_str(),
+                          std::to_string(model.ids_found).c_str(),
+                          std::to_string(model.path_found).c_str());
+        execute_query(query);
+    }
+}
+
+//! Inserts a single stream to the cache
+void
+mysql_cache_manager::update_stream(const etix::cameradar::stream_model& model) {
+    auto query = tool::fmt(this->update_result_query,
+                           this->connection.get_db_name().c_str(),
+                           model.address.c_str(),
+                           model.password.c_str(),
+                           model.product.c_str(),
+                           model.protocol.c_str(),
+                           model.route.c_str(),
+                           model.service_name.c_str(),
+                           model.state.c_str(),
+                           model.thumbnail_path.c_str(),
+                           model.username.c_str(),
+                           std::to_string(model.port).c_str(),
+                           std::to_string(model.ids_found).c_str(),
+                           std::to_string(model.path_found).c_str(),
+                           model.address.c_str());
+    execute_query(query);
+}
+
+//! Gets all cached streams
+std::vector<etix::cameradar::stream_model>
+mysql_cache_manager::get_streams() {
+    auto query = tool::fmt(this->get_results_query, this->connection.get_db_name().c_str());
+    auto result = this->connection.query(query);
+
+    if (not result.data) {
+        delete result.data;
+        return {};
+    }
+
+    std::vector<stream_model> lst;
+    while (result.data->next()) {
+        // If it's an open RTSP stream
+        if (not result.data->getString("state").compare("open") &&
+            not result.data->getString("service_name").compare("rtsp")) {
+            stream_model s{
+                result.data->getString("address"),     result.data->getUInt("port"),
+                result.data->getString("username"),    result.data->getString("password"),
+                result.data->getString("route"),       result.data->getString("service_name"),
+                result.data->getString("product"),     result.data->getString("protocol"),
+                result.data->getString("state"),       result.data->getBoolean("ids_found"),
+                result.data->getBoolean("path_found"), result.data->getString("thumbnail_path")
+            };
+            lst.push_back(s);
+        }
+    }
+
+    delete result.data;
+    return lst;
+}
+
+//! Gets all valid streams
+std::vector<etix::cameradar::stream_model>
+mysql_cache_manager::get_valid_streams() {
+    auto query = tool::fmt(this->get_results_query, this->connection.get_db_name().c_str());
+    auto result = this->connection.query(query);
+
+    if (not result.data) {
+        delete result.data;
+        return {};
+    }
+
+    std::vector<stream_model> lst;
+    while (result.data->next()) {
+        // If the ID and the Path were found add this stream
+        if (not result.data->getString("ids_found").compare("1") &&
+            not result.data->getString("path_found").compare("1")) {
+            stream_model s{
+                result.data->getString("address"),     result.data->getUInt("port"),
+                result.data->getString("username"),    result.data->getString("password"),
+                result.data->getString("route"),       result.data->getString("service_name"),
+                result.data->getString("product"),     result.data->getString("protocol"),
+                result.data->getString("state"),       result.data->getBoolean("ids_found"),
+                result.data->getBoolean("path_found"), result.data->getString("thumbnail_path")
+            };
+            lst.push_back(s);
+        }
+    }
+
+    delete result.data;
+    return lst;
+}
+
+extern "C" {
+cache_manager_iface*
+cache_manager_instance_new() {
+    return new mysql_cache_manager();
+}
+}
+}
+}

cameradar_standalone/conf/cameradar.conf.json

@@ -1,5 +1,13 @@
 {
-  "subnets" : "172.16.100.13,localhost",
+  "mysql_db" : {
+    "host" : "0.0.0.0",
+    "port" : 3306,
+    "user": "root",
+    "password": "root",
+    "db_name": "cctv_dev"
+  },
+
+  "subnets" : "172.16.100.11",
 
   // If not specified, will scan all ports (1-65535)
   "ports" : "554,8554",
@@ -9,6 +17,9 @@
   // You must give an accessible path to an already existing directory
   "thumbnail_storage_path" : "/tmp",
 
-  "cache_manager_path" : "../cache_managers/dumb_cache_manager",
+  // This is the path that will be used in the Docker container
-  "cache_manager_name" : "dumb"
+  // if you're not familiar with Docker, only change the
+  // cache_manager_name value
+  "cache_manager_path" : "../cache_managers",
+  "cache_manager_name" : "mysql"
 }

cameradar_standalone/include/cachemanager.h

@@ -14,10 +14,10 @@
 
 #pragma once
 
-#include <vector>
+#include <configuration.h>
 #include <memory>
 #include <stream_model.h>
-#include <configuration.h>
+#include <vector>
 
 namespace etix {
 namespace cameradar {
@@ -42,10 +42,10 @@ public:
     virtual void update_stream(const etix::cameradar::stream_model& newmodel) = 0;
 
     //! Gets all cached streams
-    virtual std::vector<etix::cameradar::stream_model> get_streams() const = 0;
+    virtual std::vector<etix::cameradar::stream_model> get_streams() = 0;
 
     //! Gets all valid streams which have been accessed
-    virtual std::vector<etix::cameradar::stream_model> get_valid_streams() const = 0;
+    virtual std::vector<etix::cameradar::stream_model> get_valid_streams() = 0;
 };
 
 class cache_manager_base : public cache_manager_iface {
@@ -68,10 +68,10 @@ public:
     virtual void update_stream(const etix::cameradar::stream_model& newmodel) = 0;
 
     //! Gets all cached streams
-    virtual std::vector<etix::cameradar::stream_model> get_streams() const = 0;
+    virtual std::vector<etix::cameradar::stream_model> get_streams() = 0;
 
     //! Gets all valid streams which have been accessed
-    virtual std::vector<etix::cameradar::stream_model> get_valid_streams() const = 0;
+    virtual std::vector<etix::cameradar::stream_model> get_valid_streams() = 0;
 
     //! Get the manager's instance
     cache_manager_base& get_instance();

cameradar_standalone/include/signal_handler.h

@@ -14,11 +14,15 @@
 
 #pragma once
 
+#include <assert.h> // assert
 #include <csignal>  // sigint
 #include <iostream> // stc::cout
-#include <assert.h> // assert
+
+// To avoid an unused warning for the asserted in handle_signal
+#define _unused(x) ((void)(x))
 
 namespace etix {
+
 namespace cameradar {
 
 enum class stop_priority { running, stop, force_stop };
@@ -30,6 +34,7 @@ public:
     virtual int
     handle_signal(int signum) {
         assert(signum == SIGINT);
+        _unused(signum);
         std::cout << "\b\b\b\033[K";
         if (this->ss == stop_priority::running)
             this->ss = stop_priority::stop;

cameradar_standalone/include/stream_model.h

@@ -14,8 +14,8 @@
 
 #pragma once
 
-#include <string>
 #include <json/value.h>
+#include <string>
 
 namespace etix {
 namespace cameradar {
@@ -24,7 +24,7 @@ struct stream_model {
     // Ex : "172.16.100.113"
     std::string address;
     // Ex : 8554
-    unsigned short port;
+    unsigned int port;
     // Ex : "admin"
     std::string username = "";
     // Ex : "123456"

cameradar_standalone/include/tasks/print.h

@@ -14,12 +14,11 @@
 
 #pragma once
 
-#include <cameradar_task.h>                // task interface
+#include <cachemanager.h>   // cacheManager
-#include <boost/algorithm/string/find.hpp> // boost::find
+#include <cameradar_task.h> // task interface
-#include <iostream>                        // std::ofstream
+#include <fstream>          // std::ofstream
-#include <fstream>                         // std::ofstream
+#include <iostream>         // std::ofstream
-#include <stream_model.h>                  // data model
+#include <stream_model.h>   // data model
-#include <cachemanager.h>                  // cacheManager
 
 namespace etix {
 namespace cameradar {

cameradar_standalone/src/configuration.cpp

@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <configuration.h> // configuration
 #include <fstream>         // std::ifstream
 #include <unistd.h>        // access, F_OK
-#include <configuration.h> // configuration
 
 namespace etix {
 
@@ -22,9 +22,9 @@ namespace cameradar {
 
 const std::string configuration::name_ = "configuration";
 
-// read a file at the path "path"
+// Read a file at the path "path"
-// if the file is available we return the whole content as an std::string inside
+// If the file is available we return the whole content as
-// a pair
+// an std::string inside a pair
 // otherwise return false and an empty string inside a pair
 std::pair<bool, std::string>
 read_file(const std::string& path) {
@@ -107,14 +107,6 @@ configuration::load_url() {
         auto root = Json::Value();
         auto reader = Json::Reader();
         reader.parse(content, root);
-        //    auto result = tool::json::check_fields(
-        //        {{"urls", Json::arrayValue, root["urls"]}}, "general
-        //        configuration");
-
-        //    if (not result.first) {
-        //      LOG_ERR_(result.second, "general configuration");
-        //      return false;
-        //    }
 
         for (unsigned int i = 0; i < root["urls"].size(); i++) {
             if (not root["urls"][i].isString()) {
@@ -194,7 +186,6 @@ load(const std::string& path) {
     }
     // Deserialize the json to a configuration struct
     // and return
-    // REPLACE THIS WITH JSONCPP
     std::pair<bool, configuration> conf = serialize(root);
     conf.second.raw_conf = root;
     conf.first &= conf.second.load_url();

cameradar_standalone/src/dispatcher.cpp

@@ -20,12 +20,20 @@ namespace cameradar {
 // The main loop of the binary
 void
 dispatcher::run() {
+    if (not(*cache)->configure(std::make_shared<configuration>(conf))) {
+        LOG_ERR_(
+            "There was a problem with the cache manager, Cameradar can't work properly without "
+            "cache management",
+            "dispatcher");
+        return;
+    }
     std::thread worker(&dispatcher::do_stuff, this);
     using namespace std::chrono_literals;
-    // catch CTRL+C signal
+
+    // Catch CTRL+C signal
     signal_handler::instance();
 
-    // wait for event or end
+    // Wait for event or end
     while (signal_handler::instance().should_stop() not_eq stop_priority::stop &&
            current != task::finished) {
         std::this_thread::sleep_for(30ms);
@@ -36,7 +44,7 @@ dispatcher::run() {
         LOG_INFO_("Press CTRL+C again to force stop", "dispatcher");
     }
 
-    // waiting for task to cleanup / force stop command
+    // Waiting for task to cleanup / force stop command
     while ((signal_handler::instance().should_stop() not_eq stop_priority::force_stop) and
            doing_stuff()) {
         std::this_thread::sleep_for(std::chrono::milliseconds(30));
@@ -76,7 +84,8 @@ dispatcher::do_stuff() {
         if (queue.front()->run())
             queue.pop_front();
         else {
-            LOG_ERR_("An error occured in one of the tasks, Cameradar will now stop.", "dispatcher");
+            LOG_ERR_("An error occured in one of the tasks, Cameradar will now stop.",
+                     "dispatcher");
             break;
         }
     }

cameradar_standalone/src/main.cpp

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <fs.h>         // fs::home
-#include <opt_parse.h>  // parsing opt
-#include <dispatcher.h> // program loop
-#include <iostream>     // iostream
 #include "version.h"    // versionning
+#include <dispatcher.h> // program loop
+#include <fs.h>         // fs::home
+#include <iostream>     // iostream
+#include <opt_parse.h>  // parsing opt
 
 namespace cmrdr = etix::cameradar;
 

cameradar_standalone/src/tasks/brutelogs.cpp

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <tasks/brutelogs.h>
 #include <cachemanager.h>
+#include <tasks/brutelogs.h>
 
 namespace etix {
 namespace cameradar {
@@ -36,10 +36,10 @@ brutelogs::test_ids(const etix::cameradar::stream_model& stream,
     std::string path = stream.service_name + "://";
     if (username != "" || password != "") { path += username + ":" + password + "@"; }
     path += stream.address + ":" + std::to_string(stream.port);
-    LOG_DEBUG_("Testing ids : " + path, "bruteforce");
+    LOG_DEBUG_("Testing ids : " + path, "brutelogs");
     try {
         if (curl_describe(path, true)) {
-            LOG_DEBUG_("[FOUND IDS] : " + path, "bruteforce");
+            LOG_DEBUG_("[FOUND IDS] : " + path, "brutelogs");
             found = true;
             stream_model newstream{
                 stream.address, stream.port,         username,       password,
@@ -55,7 +55,7 @@ brutelogs::test_ids(const etix::cameradar::stream_model& stream,
             (*cache)->update_stream(newstream);
         }
     } catch (const std::runtime_error& e) {
-        LOG_DEBUG_("Ids already tested : " + std::string(e.what()), "bruteforce");
+        LOG_DEBUG_("Ids already tested : " + std::string(e.what()), "brutelogs");
     }
     return found;
 }
@@ -75,7 +75,7 @@ brutelogs::run() const {
     LOG_INFO_(
         "Beginning bruteforce of the usernames and passwords task, it may "
         "take a while.",
-        "bruteforce");
+        "brutelogs");
     std::vector<etix::cameradar::stream_model> streams = (*cache)->get_streams();
     bool doubleskip;
     size_t found = 0;
@@ -88,7 +88,7 @@ brutelogs::run() const {
                           " : This camera's ids were already discovered in "
                           "the database. Skipping to "
                           "the next camera.",
-                      "bruteforce");
+                      "brutelogs");
             ++found;
         } else {
             for (const auto& username : conf.usernames) {
@@ -110,12 +110,12 @@ brutelogs::run() const {
         }
     }
     if (!found) {
-        LOG_WARN_(no_ids_warning_, "bruteforce");
+        LOG_WARN_(no_ids_warning_, "brutelogs");
         return false;
     } else
         LOG_INFO_("Found " + std::to_string(found) + " ids for " + std::to_string(streams.size()) +
                       " cameras",
-                  "bruteforce");
+                  "brutelogs");
     return true;
 }
 }

cameradar_standalone/src/tasks/mapping.cpp

@@ -28,9 +28,12 @@ namespace cameradar {
 //! problem.
 bool
 nmap_is_ok() {
-    return (launch_command("test `dpkg -l | cut -c 5-9 | grep nmap` = nmap")
+    return (
-    // && launch_command("test `nmap --version | cut -c 14-18  | head -n2 | tail -n1` = 6.47")
+        launch_command("test `dpkg -l | cut -c 5-9 | grep nmap` = nmap")
-    && launch_command("mkdir -p scans")); // Creates the directory in which the scans will be stored
+        // && launch_command("test `nmap --version | cut -c 14-18  | head -n2 | tail -n1` = 6.47")
+        &&
+        launch_command(
+            "mkdir -p scans")); // Creates the directory in which the scans will be stored
 }
 
 //! Launches and checks the return of the nmap command
@@ -44,6 +47,7 @@ mapping::run() const {
         LOG_INFO_("Beginning mapping task. This may take a while.", "mapping");
         std::string cmd =
             "nmap -T4 -A " + subnets + " -p " + this->conf.ports + " -oX " + nmap_output;
+        LOG_DEBUG_("Launching nmap : " + cmd, "mapping");
         bool ret = launch_command(cmd);
         if (ret)
             LOG_INFO_("Nmap XML output successfully generated in file: " + nmap_output, "mapping");

cmake/mysql_connector.cmake

@@ -0,0 +1,43 @@
+# Copyright (C) 2015 Etix Labs - All Rights Reserved.
+# All information contained herein is, and remains the property of Etix Labs and its suppliers,
+# if any. The intellectual and technical concepts contained herein are proprietary to Etix Labs
+# Dissemination of this information or reproduction of this material is strictly forbidden unless
+# prior written permission is obtained from Etix Labs.
+
+# MySQL Connector dependency
+message(STATUS "Configuring deps.mysqlconnector")
+
+set (MYSQL_CONNECTOR_VERSION 1.1.6)
+set (MD5 9e49dcfc1408b18b3d3ca02781ff7efb)
+set (MYSQL_CONNECTOR_DIR mysql-connector)
+set (MYSQL_CONNECTOR_PATH ${DEPS_DIR}/${MYSQL_CONNECTOR_DIR})
+
+set (BOOST_ROOT_DIR ${DEPS_DIR}/boost/src/deps.boost)
+
+# include(ExternalProject)
+ExternalProject_Add(
+    deps.mysql_connector
+    PREFIX ${MYSQL_CONNECTOR_PATH}
+    URL http://dev.mysql.com/get/Downloads/Connector-C++/mysql-connector-c++-${MYSQL_CONNECTOR_VERSION}.tar.gz
+    URL_HASH MD5=${MD5}
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -DBOOST_ROOT=${BOOST_ROOT_DIR} "-DCMAKE_INSTALL_PREFIX=${MYSQL_CONNECTOR_PATH}" -DBUILD_TYPE=Release -DMYSQL_CXXFLAGS=-fexceptions <SOURCE_DIR>
+    BUILD_IN_SOURCE ON
+    UPDATE_COMMAND ""
+    BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
+    INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} install
+    LOG_DOWNLOAD ON
+    LOG_UPDATE ON
+    LOG_CONFIGURE ON
+    LOG_BUILD ON
+)
+
+set (MYSQL_CONNECTOR_INCLUDE_DIR "${MYSQL_CONNECTOR_PATH}/include" PARENT_SCOPE)
+set (MYSQL_CONNECTOR_LIBRARY_DIR "${MYSQL_CONNECTOR_PATH}/lib")
+
+set (MYSQL_CONNECTOR_LIBRARY_DIR ${MYSQL_CONNECTOR_LIBRARY_DIR} PARENT_SCOPE)
+
+# list all the hiredis libraries
+file(GLOB MYSQL_CONNECTOR_INSTALL_DEPENDENCIES "${MYSQL_CONNECTOR_LIBRARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlcppconn*${CMAKE_SHARED_LIBRARY_SUFFIX}*")
+list (APPEND CAMERADAR_INSTALL_DEPENDENCIES ${MYSQL_CONNECTOR_INSTALL_DEPENDENCIES})
+
+set(CAMERADAR_INSTALL_DEPENDENCIES ${CAMERADAR_INSTALL_DEPENDENCIES} PARENT_SCOPE)

deployment/Dockerfile

@@ -10,7 +10,9 @@ RUN apt-get update && apt-get install -y \
     libgstreamer1.0-dev \
     gstreamer1.0-plugins-base \
     gstreamer1.0-plugins-good \
-    libcurl4-openssl-dev
+    libcurl4-openssl-dev \
+    libmysqlclient18 \
+    mysql-client
 
 ADD cameradar_*_Release_Linux.tar.gz /
 RUN mv cameradar_*_Release_Linux cameradar

deployment/conf/cameradar.conf.json

@@ -1,5 +1,13 @@
 {
-  "subnets" : "172.16.100.13,localhost",
+  "mysql_db" : {
+    "host" : "__MYSQL_ADDR__",
+    "port" : __MYSQL_PORT__,
+    "user": "root",
+    "password": "root",
+    "db_name": "cmrdr"
+  },
+
+  "subnets" : "localhost",
 
   // If not specified, will scan all ports (1-65535)
   "ports" : "554,8554",
@@ -9,6 +17,9 @@
   // You must give an accessible path to an already existing directory
   "thumbnail_storage_path" : "/tmp",
 
+  // This is the path that will be used in the Docker container
+  // if you're not familiar with Docker, only change the
+  // cache_manager_name value
   "cache_manager_path" : "/cameradar/cache_managers",
-  "cache_manager_name" : "dumb"
+  "cache_manager_name" : "mysql"
 }

deployment/docker-compose.yml

@@ -5,3 +5,12 @@ cameradar:
   volumes:
     - "./conf:/tmp/conf:ro"
     - "./cameradar_thumbnails:/tmp/cameradar_thumbnails"
+  links:
+   - ext_cctv_mysql
+ext_cctv_mysql:
+  image: mysql:5.7
+  environment:
+   MYSQL_ROOT_PASSWORD: root
+   MYSQL_DATABASE: cmrdr
+  ports:
+   - "3306:3306"

deployment/run.sh

@@ -23,6 +23,16 @@ echo -n "replacing cameras ports in configuration "
 sed -i s#__PORTS_TO_CHECK__#$CAMERAS_PORTS#g $CONF
 echo -e $COL_GREEN"ok"$COL_RESET
 
+# Replace ext_cctv_mysql with the IP address of your DB or the name of its Docker
+# container. The container has to be linked in docker-compose.yml for cameradar
+# to be able to interact with it.
+echo -n "replacing mysql host and port in configuration "
+sed -i s#__MYSQL_ADDR__#ext_cctv_mysql#g $CONF
+
+# Reaplce 3306 with the port of your DB
+sed -i s#__MYSQL_PORT__#3306#g $CONF
+echo -e $COL_GREEN"ok"$COL_RESET
+
 /cameradar/bin/cameradar -l 1 -c /conf/cameradar.conf.json &
 cameradar_pid=$!
 

deps/CMakeLists.txt

@@ -17,7 +17,7 @@ cmake_minimum_required (VERSION 2.8.1)
 # Lib subdirectory
 
 include (jsoncpp)
-include (boost)
+include (mysql_connector)
 
 set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM ON)
 set (CAMERADAR_INSTALL_DEPENDENCIES ${CAMERADAR_INSTALL_DEPENDENCIES} PARENT_SCOPE)