adding support for a collecto config file.

/scrutiny/config/collector.yaml

Adding ability to specify host identifier (label), that is updated on every collector run.
Can be specified by `host-id` CLI or `COLLECTOR_HOST_ID` env var.

Created a config class, interface and associated tests.

Created a "TransformDetectedDrives" function, that will allow users to insert drives not detected by Smarctl --scan, ignore drives that they dont want, and override smartctl device type.

Added Upsert functionality when registering devices.

Replaced "github.com/jinzhu/gorm" with "gorm.io/gorm" (ORM location moved, was using incorrect lib url)
Removed machineid library.

README.md

@@ -128,7 +128,7 @@ docker exec scrutiny /scrutiny/bin/scrutiny-collector-metrics run
 ```
 
 # Configuration
-We support a global YAML configuration file that must be located at /scrutiny/config/scrutiny.yaml
+We support a global YAML configuration file that must be located at `/scrutiny/config/scrutiny.yaml`
 
 Check the [example.scrutiny.yml](example.scrutiny.yaml) file for a fully commented version.
 

collector/cmd/collector-metrics/collector-metrics.go

@@ -3,6 +3,8 @@ package main
 import (
 	"fmt"
 	"github.com/analogj/scrutiny/collector/pkg/collector"
+	"github.com/analogj/scrutiny/collector/pkg/config"
+	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/version"
 	"github.com/sirupsen/logrus"
 	"io"
@@ -20,6 +22,20 @@ var goarch string
 
 func main() {
 
+	config, err := config.Create()
+	if err != nil {
+		fmt.Printf("FATAL: %+v\n", err)
+		os.Exit(1)
+	}
+
+	//we're going to load the config file manually, since we need to validate it.
+	err = config.ReadConfig("/scrutiny/config/collector.yaml") // Find and read the config file
+	if _, ok := err.(errors.ConfigFileMissingError); ok {      // Handle errors reading the config file
+		//ignore "could not find config file"
+	} else if err != nil {
+		os.Exit(1)
+	}
+
 	cli.CommandHelpTemplate = `NAME:
    {{.HelpName}} - {{.Usage}}
 USAGE:
@@ -75,6 +91,17 @@ OPTIONS:
 				Name:  "run",
 				Usage: "Run the scrutiny smartctl metrics collector",
 				Action: func(c *cli.Context) error {
+					if c.IsSet("config") {
+						err = config.ReadConfig(c.String("config")) // Find and read the config file
+						if err != nil {                             // Handle errors reading the config file
+							//ignore "could not find config file"
+							fmt.Printf("Could not find config file at specified path: %s", c.String("config"))
+							return err
+						}
+					}
+					if c.IsSet("host-id") {
+						config.Set("host.id", c.String("host-id")) // set/override the host-id using CLI.
+					}
 
 					collectorLogger := logrus.WithFields(logrus.Fields{
 						"type": "metrics",
@@ -97,6 +124,7 @@ OPTIONS:
 					}
 
 					metricCollector, err := collector.CreateMetricsCollector(
+						config,
 						collectorLogger,
 						c.String("api-endpoint"),
 					)
@@ -109,6 +137,10 @@ OPTIONS:
 				},
 
 				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:  "config",
+						Usage: "Specify the path to the devices file",
+					},
 					&cli.StringFlag{
 						Name:    "api-endpoint",
 						Usage:   "The api server endpoint",
@@ -128,12 +160,18 @@ OPTIONS:
 						Usage:   "Enable debug logging",
 						EnvVars: []string{"COLLECTOR_DEBUG", "DEBUG"},
 					},
+
+					&cli.BoolFlag{
+						Name:    "host-id",
+						Usage:   "Host identifier/label, used for grouping devices",
+						EnvVars: []string{"COLLECTOR_HOST_ID"},
+					},
 				},
 			},
 		},
 	}
 
-	err := app.Run(os.Args)
+	err = app.Run(os.Args)
 	if err != nil {
 		log.Fatal(color.HiRedString("ERROR: %v", err))
 	}

collector/pkg/collector/metrics.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/collector/pkg/models"
@@ -16,17 +17,19 @@ import (
 )
 
 type MetricsCollector struct {
+	config config.Interface
 	BaseCollector
 	apiEndpoint *url.URL
 }
 
-func CreateMetricsCollector(logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
+func CreateMetricsCollector(appConfig config.Interface, logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
 	apiEndpointUrl, err := url.Parse(apiEndpoint)
 	if err != nil {
 		return MetricsCollector{}, err
 	}
 
 	sc := MetricsCollector{
+		config:      appConfig,
 		apiEndpoint: apiEndpointUrl,
 		BaseCollector: BaseCollector{
 			logger: logger,
@@ -49,6 +52,7 @@ func (mc *MetricsCollector) Run() error {
 
 	deviceDetector := detect.Detect{
 		Logger: mc.logger,
+		Config: mc.config,
 	}
 	detectedStorageDevices, err := deviceDetector.Start()
 	if err != nil {

collector/pkg/config/config.go

@@ -0,0 +1,100 @@
+package config
+
+import (
+	"github.com/analogj/go-util/utils"
+	"github.com/analogj/scrutiny/collector/pkg/errors"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+	"log"
+	"os"
+)
+
+// When initializing this class the following methods must be called:
+// Config.New
+// Config.Init
+// This is done automatically when created via the Factory.
+type configuration struct {
+	*viper.Viper
+}
+
+//Viper uses the following precedence order. Each item takes precedence over the item below it:
+// explicit call to Set
+// flag
+// env
+// config
+// key/value store
+// default
+
+func (c *configuration) Init() error {
+	c.Viper = viper.New()
+	//set defaults
+	c.SetDefault("host.id", "")
+
+	c.SetDefault("devices", []string{})
+
+	//c.SetDefault("collect.short.command", "-a -o on -S on")
+
+	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
+	c.SetConfigType("yaml")
+	//c.SetConfigName("drawbridge")
+	//c.AddConfigPath("$HOME/")
+
+	//CLI options will be added via the `Set()` function
+	return nil
+}
+
+func (c *configuration) ReadConfig(configFilePath string) error {
+	configFilePath, err := utils.ExpandPath(configFilePath)
+	if err != nil {
+		return err
+	}
+
+	if !utils.FileExists(configFilePath) {
+		log.Printf("No configuration file found at %v. Using Defaults.", configFilePath)
+		return errors.ConfigFileMissingError("The configuration file could not be found.")
+	}
+
+	//validate config file contents
+	//err = c.ValidateConfigFile(configFilePath)
+	//if err != nil {
+	//	log.Printf("Config file at `%v` is invalid: %s", configFilePath, err)
+	//	return err
+	//}
+
+	log.Printf("Loading configuration file: %s", configFilePath)
+
+	config_data, err := os.Open(configFilePath)
+	if err != nil {
+		log.Printf("Error reading configuration file: %s", err)
+		return err
+	}
+
+	err = c.MergeConfig(config_data)
+	if err != nil {
+		return err
+	}
+
+	return c.ValidateConfig()
+}
+
+// This function ensures that the merged config works correctly.
+func (c *configuration) ValidateConfig() error {
+
+	//TODO:
+	// check that device prefix matches OS
+	// check that schema of config file is valid
+
+	return nil
+}
+
+func (c *configuration) GetScanOverrides() []models.ScanOverride {
+	// we have to support 2 types of device types.
+	// - simple device type (device_type: 'sat')
+	// and list of device types (type: \n- 3ware,0 \n- 3ware,1 \n- 3ware,2)
+	// GetString will return "" if this is a list of device types.
+
+	overrides := []models.ScanOverride{}
+	c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
+	return overrides
+}

collector/pkg/config/config_test.go

@@ -0,0 +1,64 @@
+package config_test
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/config"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/stretchr/testify/require"
+	"path"
+	"testing"
+)
+
+func TestConfiguration_GetScanOverrides_Simple(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "simple_device.yaml"))
+	require.NoError(t, err, "should correctly load simple device config")
+	scanOverrides := testConfig.GetScanOverrides()
+
+	//assert
+	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat"}, Ignore: false}}, scanOverrides)
+}
+
+func TestConfiguration_GetScanOverrides_Ignore(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "ignore_device.yaml"))
+	require.NoError(t, err, "should correctly load ignore device config")
+	scanOverrides := testConfig.GetScanOverrides()
+
+	//assert
+	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}, scanOverrides)
+}
+
+func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "raid_device.yaml"))
+	require.NoError(t, err, "should correctly load ignore device config")
+	scanOverrides := testConfig.GetScanOverrides()
+
+	//assert
+	require.Equal(t, []models.ScanOverride{
+		{
+			Device:     "/dev/bus/0",
+			DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
+			Ignore:     false,
+		},
+		{
+			Device:     "/dev/twa0",
+			DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
+			Ignore:     false,
+		}}, scanOverrides)
+}

collector/pkg/config/factory.go

@@ -0,0 +1,9 @@
+package config
+
+func Create() (Interface, error) {
+	config := new(configuration)
+	if err := config.Init(); err != nil {
+		return nil, err
+	}
+	return config, nil
+}

collector/pkg/config/interface.go

@@ -0,0 +1,26 @@
+package config
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/spf13/viper"
+)
+
+// Create mock using:
+// mockgen -source=collector/pkg/config/interface.go -destination=collector/pkg/config/mock/mock_config.go
+type Interface interface {
+	Init() error
+	ReadConfig(configFilePath string) error
+	Set(key string, value interface{})
+	SetDefault(key string, value interface{})
+
+	AllSettings() map[string]interface{}
+	IsSet(key string) bool
+	Get(key string) interface{}
+	GetBool(key string) bool
+	GetInt(key string) int
+	GetString(key string) string
+	GetStringSlice(key string) []string
+	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
+
+	GetScanOverrides() []models.ScanOverride
+}

collector/pkg/config/mock/mock_config.go

@@ -0,0 +1,218 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: collector/pkg/config/interface.go
+
+// Package mock_config is a generated GoMock package.
+package mock_config
+
+import (
+	models "github.com/analogj/scrutiny/collector/pkg/models"
+	gomock "github.com/golang/mock/gomock"
+	viper "github.com/spf13/viper"
+	reflect "reflect"
+)
+
+// MockInterface is a mock of Interface interface
+type MockInterface struct {
+	ctrl     *gomock.Controller
+	recorder *MockInterfaceMockRecorder
+}
+
+// MockInterfaceMockRecorder is the mock recorder for MockInterface
+type MockInterfaceMockRecorder struct {
+	mock *MockInterface
+}
+
+// NewMockInterface creates a new mock instance
+func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
+	mock := &MockInterface{ctrl: ctrl}
+	mock.recorder = &MockInterfaceMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
+	return m.recorder
+}
+
+// Init mocks base method
+func (m *MockInterface) Init() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Init")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Init indicates an expected call of Init
+func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
+}
+
+// ReadConfig mocks base method
+func (m *MockInterface) ReadConfig(configFilePath string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReadConfig indicates an expected call of ReadConfig
+func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
+}
+
+// Set mocks base method
+func (m *MockInterface) Set(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Set", key, value)
+}
+
+// Set indicates an expected call of Set
+func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
+}
+
+// SetDefault mocks base method
+func (m *MockInterface) SetDefault(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetDefault", key, value)
+}
+
+// SetDefault indicates an expected call of SetDefault
+func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
+}
+
+// AllSettings mocks base method
+func (m *MockInterface) AllSettings() map[string]interface{} {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AllSettings")
+	ret0, _ := ret[0].(map[string]interface{})
+	return ret0
+}
+
+// AllSettings indicates an expected call of AllSettings
+func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings))
+}
+
+// IsSet mocks base method
+func (m *MockInterface) IsSet(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsSet", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsSet indicates an expected call of IsSet
+func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
+}
+
+// Get mocks base method
+func (m *MockInterface) Get(key string) interface{} {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", key)
+	ret0, _ := ret[0].(interface{})
+	return ret0
+}
+
+// Get indicates an expected call of Get
+func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key)
+}
+
+// GetBool mocks base method
+func (m *MockInterface) GetBool(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetBool", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// GetBool indicates an expected call of GetBool
+func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key)
+}
+
+// GetInt mocks base method
+func (m *MockInterface) GetInt(key string) int {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetInt", key)
+	ret0, _ := ret[0].(int)
+	return ret0
+}
+
+// GetInt indicates an expected call of GetInt
+func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
+}
+
+// GetString mocks base method
+func (m *MockInterface) GetString(key string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetString", key)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetString indicates an expected call of GetString
+func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key)
+}
+
+// GetStringSlice mocks base method
+func (m *MockInterface) GetStringSlice(key string) []string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetStringSlice", key)
+	ret0, _ := ret[0].([]string)
+	return ret0
+}
+
+// GetStringSlice indicates an expected call of GetStringSlice
+func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key)
+}
+
+// UnmarshalKey mocks base method
+func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{key, rawVal}
+	for _, a := range decoderOpts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "UnmarshalKey", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UnmarshalKey indicates an expected call of UnmarshalKey
+func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{key, rawVal}, decoderOpts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...)
+}
+
+// GetScanOverrides mocks base method
+func (m *MockInterface) GetScanOverrides() []models.ScanOverride {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetScanOverrides")
+	ret0, _ := ret[0].([]models.ScanOverride)
+	return ret0
+}
+
+// GetScanOverrides indicates an expected call of GetScanOverrides
+func (mr *MockInterfaceMockRecorder) GetScanOverrides() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScanOverrides", reflect.TypeOf((*MockInterface)(nil).GetScanOverrides))
+}

collector/pkg/config/testdata/ignore_device.yaml

@@ -0,0 +1,4 @@
+version: 1
+devices:
+  - device: /dev/sda
+    ignore: true

collector/pkg/config/testdata/raid_device.yaml

@@ -0,0 +1,19 @@
+version: 1
+devices:
+  - device: /dev/bus/0
+    type:
+      - megaraid,14
+      - megaraid,15
+      - megaraid,18
+      - megaraid,19
+      - megaraid,20
+      - megaraid,21
+
+  - device: /dev/twa0
+    type:
+      - 3ware,0
+      - 3ware,1
+      - 3ware,2
+      - 3ware,3
+      - 3ware,4
+      - 3ware,5

collector/pkg/config/testdata/simple_device.yaml

@@ -0,0 +1,27 @@
+version: 1
+devices:
+  - device: /dev/sda
+    type: 'sat'
+#
+#  # example to show how to ignore a specific disk/device.
+#  - device: /dev/sda
+#    ignore: true
+#
+#  # examples showing how to force smartctl to detect disks inside a raid array/virtual disk
+#  - device: /dev/bus/0
+#    type:
+#      - megaraid,14
+#      - megaraid,15
+#      - megaraid,18
+#      - megaraid,19
+#      - megaraid,20
+#      - megaraid,21
+#
+#  - device: /dev/twa0
+#    type:
+#      - 3ware,0
+#      - 3ware,1
+#      - 3ware,2
+#      - 3ware,3
+#      - 3ware,4
+#      - 3ware,5

collector/pkg/detect/detect.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"github.com/denisbrodbeck/machineid"
 	"github.com/sirupsen/logrus"
 	"os"
 	"strings"
@@ -14,6 +14,7 @@ import (
 
 type Detect struct {
 	Logger *logrus.Entry
+	Config config.Interface
 }
 
 //private/common functions
@@ -24,7 +25,7 @@ type Detect struct {
 //
 // To handle these issues, we have OS specific wrapper functions that update/modify these detected devices.
 // models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
-func (d *Detect) smartctlScan() ([]models.Device, error) {
+func (d *Detect) SmartctlScan() ([]models.Device, error) {
 	//we use smartctl to detect all the drives available.
 	detectedDeviceConnJson, err := common.ExecCmd(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
 	if err != nil {
@@ -39,14 +40,7 @@ func (d *Detect) smartctlScan() ([]models.Device, error) {
 		return nil, err
 	}
 
-	detectedDevices := []models.Device{}
+	detectedDevices := d.TransformDetectedDevices(detectedDeviceConns)
-
-	for _, detectedDevice := range detectedDeviceConns.Devices {
-		detectedDevices = append(detectedDevices, models.Device{
-			DeviceType: detectedDevice.Type,
-			DeviceName: strings.TrimPrefix(detectedDevice.Name, DevicePrefix()),
-		})
-	}
 
 	return detectedDevices, nil
 }
@@ -55,7 +49,7 @@ func (d *Detect) smartctlScan() ([]models.Device, error) {
 // It has a couple of issues however:
 // - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
 // - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
-func (d *Detect) smartCtlInfo(device *models.Device) error {
+func (d *Detect) SmartCtlInfo(device *models.Device) error {
 
 	args := []string{"--info", "-j"}
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
@@ -77,8 +71,8 @@ func (d *Detect) smartCtlInfo(device *models.Device) error {
 		return err
 	}
 
-	//DeviceType and DeviceName are already populated.
+	//WWN: this is a serial number/world-wide number that will not change.
-	//WWN
+	//DeviceType and DeviceName are already populated, however may change between collector runs (eg. config/host restart)
 	//InterfaceType:
 	device.ModelName = availableDeviceInfo.ModelName
 	device.InterfaceSpeed = availableDeviceInfo.InterfaceSpeed.Current.String
@@ -110,7 +104,60 @@ func (d *Detect) smartCtlInfo(device *models.Device) error {
 	return nil
 }
 
-//uses https://github.com/denisbrodbeck/machineid to get a OS specific unique machine ID.
+// function will remove devices that are marked for "ignore" in config file
-func (d *Detect) getMachineId() (string, error) {
+// will also add devices that are specified in config file, but "missing" from smartctl --scan
-	return machineid.ProtectedID("scrutiny")
+// this function will also update the deviceType to the option specified in config.
+func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []models.Device {
+	groupedDevices := map[string][]models.Device{}
+
+	for _, scannedDevice := range detectedDeviceConns.Devices {
+
+		deviceFile := strings.ToLower(scannedDevice.Name)
+
+		detectedDevice := models.Device{
+			HostId:     d.Config.GetString("host.id"),
+			DeviceType: scannedDevice.Type,
+			DeviceName: strings.TrimPrefix(deviceFile, DevicePrefix()),
+		}
+
+		//find (or create) a slice to contain the devices in this group
+		if groupedDevices[deviceFile] == nil {
+			groupedDevices[deviceFile] = []models.Device{}
+		}
+
+		// add this scanned device to the group
+		groupedDevices[deviceFile] = append(groupedDevices[deviceFile], detectedDevice)
+	}
+
+	//now tha we've "grouped" all the devices, lets override any groups specified in the config file.
+
+	for _, overrideDevice := range d.Config.GetScanOverrides() {
+		overrideDeviceFile := strings.ToLower(overrideDevice.Device)
+
+		if overrideDevice.Ignore {
+			// this device file should be deleted if it exists
+			delete(groupedDevices, overrideDeviceFile)
+		} else {
+			//create a new device group, and replace the one generated by smartctl --scan
+			overrideDeviceGroup := []models.Device{}
+
+			for _, overrideDeviceType := range overrideDevice.DeviceType {
+				overrideDeviceGroup = append(overrideDeviceGroup, models.Device{
+					HostId:     d.Config.GetString("host.id"),
+					DeviceType: overrideDeviceType,
+					DeviceName: strings.TrimPrefix(overrideDeviceFile, DevicePrefix()),
+				})
+			}
+
+			groupedDevices[overrideDeviceFile] = overrideDeviceGroup
+		}
+	}
+
+	//flatten map
+	detectedDevices := []models.Device{}
+	for _, group := range groupedDevices {
+		detectedDevices = append(detectedDevices, group...)
+	}
+
+	return detectedDevices
 }

collector/pkg/detect/detect_test.go

@@ -0,0 +1,138 @@
+package detect_test
+
+import (
+	mock_config "github.com/analogj/scrutiny/collector/pkg/config/mock"
+	"github.com/analogj/scrutiny/collector/pkg/detect"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	detectedDevices := models.Scan{
+		Devices: []models.ScanDevice{
+			{
+				Name:     "/dev/sda",
+				InfoName: "/dev/sda",
+				Protocol: "scsi",
+				Type:     "scsi",
+			},
+		},
+	}
+
+	d := detect.Detect{
+		Config: fakeConfig,
+	}
+
+	//test
+	transformedDevices := d.TransformDetectedDevices(detectedDevices)
+
+	//assert
+	require.Equal(t, "sda", transformedDevices[0].DeviceName)
+	require.Equal(t, "scsi", transformedDevices[0].DeviceType)
+}
+
+func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
+	detectedDevices := models.Scan{
+		Devices: []models.ScanDevice{
+			{
+				Name:     "/dev/sda",
+				InfoName: "/dev/sda",
+				Protocol: "scsi",
+				Type:     "scsi",
+			},
+		},
+	}
+
+	d := detect.Detect{
+		Config: fakeConfig,
+	}
+
+	//test
+	transformedDevices := d.TransformDetectedDevices(detectedDevices)
+
+	//assert
+	require.Equal(t, []models.Device{}, transformedDevices)
+}
+
+func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{
+		{
+			Device:     "/dev/bus/0",
+			DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
+			Ignore:     false,
+		},
+		{
+			Device:     "/dev/twa0",
+			DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
+			Ignore:     false,
+		}})
+	detectedDevices := models.Scan{
+		Devices: []models.ScanDevice{
+			{
+				Name:     "/dev/bus/0",
+				InfoName: "/dev/bus/0",
+				Protocol: "scsi",
+				Type:     "scsi",
+			},
+		},
+	}
+
+	d := detect.Detect{
+		Config: fakeConfig,
+	}
+
+	//test
+	transformedDevices := d.TransformDetectedDevices(detectedDevices)
+
+	//assert
+	require.Equal(t, 12, len(transformedDevices))
+}
+
+func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
+	detectedDevices := models.Scan{
+		Devices: []models.ScanDevice{
+			{
+				Name:     "/dev/sda",
+				InfoName: "/dev/sda",
+				Protocol: "ata",
+				Type:     "ata",
+			},
+		},
+	}
+
+	d := detect.Detect{
+		Config: fakeConfig,
+	}
+
+	//test
+	transformedDevices := d.TransformDetectedDevices(detectedDevices)
+
+	//assert
+	require.Equal(t, 1, len(transformedDevices))
+	require.Equal(t, "sat+megaraid", transformedDevices[0].DeviceType)
+}

collector/pkg/detect/devices_darwin.go

@@ -12,7 +12,7 @@ func DevicePrefix() string {
 
 func (d *Detect) Start() ([]models.Device, error) {
 	// call the base/common functionality to get a list of devicess
-	detectedDevices, err := d.smartctlScan()
+	detectedDevices, err := d.SmartctlScan()
 	if err != nil {
 		return nil, err
 	}
@@ -25,7 +25,7 @@ func (d *Detect) Start() ([]models.Device, error) {
 
 	//inflate device info for detected devices.
 	for ndx, _ := range detectedDevices {
-		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+		d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
 	}
 
 	return detectedDevices, nil

collector/pkg/detect/devices_freebsd.go

@@ -12,14 +12,14 @@ func DevicePrefix() string {
 
 func (d *Detect) Start() ([]models.Device, error) {
 	// call the base/common functionality to get a list of devices
-	detectedDevices, err := d.smartctlScan()
+	detectedDevices, err := d.SmartctlScan()
 	if err != nil {
 		return nil, err
 	}
 
 	//inflate device info for detected devices.
 	for ndx, _ := range detectedDevices {
-		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+		d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
 	}
 
 	return detectedDevices, nil

collector/pkg/detect/devices_linux.go

@@ -12,14 +12,14 @@ func DevicePrefix() string {
 
 func (d *Detect) Start() ([]models.Device, error) {
 	// call the base/common functionality to get a list of devices
-	detectedDevices, err := d.smartctlScan()
+	detectedDevices, err := d.SmartctlScan()
 	if err != nil {
 		return nil, err
 	}
 
 	//inflate device info for detected devices.
 	for ndx, _ := range detectedDevices {
-		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+		d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
 	}
 
 	return detectedDevices, nil

collector/pkg/detect/devices_windows.go

@@ -11,14 +11,14 @@ func DevicePrefix() string {
 
 func (d *Detect) Start() ([]models.Device, error) {
 	// call the base/common functionality to get a list of devices
-	detectedDevices, err := d.smartctlScan()
+	detectedDevices, err := d.SmartctlScan()
 	if err != nil {
 		return nil, err
 	}
 
 	//inflate device info for detected devices.
 	for ndx, _ := range detectedDevices {
-		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+		d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
 	}
 
 	return detectedDevices, nil

collector/pkg/models/device.go

@@ -1,7 +1,8 @@
 package models
 
 type Device struct {
-	WWN string `json:"wwn"`
+	WWN    string `json:"wwn"`
+	HostId string `json:"host_id"`
 
 	DeviceName     string `json:"device_name"`
 	Manufacturer   string `json:"manufacturer"`

collector/pkg/models/scan.go

@@ -10,10 +10,11 @@ type Scan struct {
 		Argv         []string `json:"argv"`
 		ExitStatus   int      `json:"exit_status"`
 	} `json:"smartctl"`
-	Devices []struct {
+	Devices []ScanDevice `json:"devices"`
-		Name     string `json:"name"`
+}
-		InfoName string `json:"info_name"`
+type ScanDevice struct {
-		Type     string `json:"type"`
+	Name     string `json:"name"`
-		Protocol string `json:"protocol"`
+	InfoName string `json:"info_name"`
-	} `json:"devices"`
+	Type     string `json:"type"`
+	Protocol string `json:"protocol"`
 }

collector/pkg/models/scan_override.go

@@ -0,0 +1,7 @@
+package models
+
+type ScanOverride struct {
+	Device     string   `mapstructure:"device"`
+	DeviceType []string `mapstructure:"type"`
+	Ignore     bool     `mapstructure:"ignore"`
+}

example.collector.yaml

@@ -0,0 +1,65 @@
+# Commented Scrutiny Configuration File
+#
+# The default location for this file is /scrutiny/config/collector.yaml.
+# In some cases to improve clarity default values are specified,
+# uncommented. Other example values are commented out.
+#
+# When this file is parsed by Scrutiny, all configuration file keys are
+# lowercased automatically. As such, Configuration keys are case-insensitive,
+# and should be lowercase in this file to be consistent with usage.
+
+
+######################################################################
+# Version
+#
+# version specifies the version of this configuration file schema, not
+# the scrutiny binary. There is only 1 version available at the moment
+version: 1
+
+# The host id is a label used for identifying groups of disks running on the same host
+# Primiarly used for hub/spoke deployments (can be left empty if using all-in-one image).
+host:
+  id: ""
+
+
+# This block allows you to override/customize the settings for devices detected by
+# Scrutiny via `smartctl --scan`
+# See the "--device=TYPE" section of https://linux.die.net/man/8/smartctl
+# type can be a 'string' or a 'list'
+devices:
+#  # example for forcing device type detection for a single disk
+#  - device: /dev/sda
+#    type: 'sat'
+#
+#  # example to show how to ignore a specific disk/device.
+#  - device: /dev/sda
+#    ignore: true
+#
+#  # examples showing how to force smartctl to detect disks inside a raid array/virtual disk
+#  - device: /dev/bus/0
+#    type:
+#      - megaraid,14
+#      - megaraid,15
+#      - megaraid,18
+#      - megaraid,19
+#      - megaraid,20
+#      - megaraid,21
+#
+#  - device: /dev/twa0
+#    type:
+#      - 3ware,0
+#      - 3ware,1
+#      - 3ware,2
+#      - 3ware,3
+#      - 3ware,4
+#      - 3ware,5
+
+
+
+########################################################################################################################
+# FEATURES COMING SOON
+#
+# The following commented out sections are a preview of additional configuration options that will be available soon.
+#
+########################################################################################################################
+

example.scrutiny.yaml

@@ -1,6 +1,6 @@
 # Commented Scrutiny Configuration File
 #
-# The default location for this file is ~/scrutiny.yaml.
+# The default location for this file is /scrutiny/config/scrutiny.yaml.
 # In some cases to improve clarity default values are specified,
 # uncommented. Other example values are commented out.
 #

go.mod

@@ -6,13 +6,15 @@ require (
 	github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae // indirect
 	github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
 	github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a
-	github.com/denisbrodbeck/machineid v1.0.1
+	github.com/denisbrodbeck/machineid v1.0.1 // indirect
 	github.com/fatih/color v1.9.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/golang/mock v1.4.3
 	github.com/jaypipes/ghw v0.6.1
 	github.com/jinzhu/gorm v1.9.14
 	github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect
+	github.com/mattn/go-sqlite3 v1.14.4
+	github.com/mitchellh/mapstructure v1.2.2
 	github.com/sirupsen/logrus v1.2.0
 	github.com/spf13/viper v1.7.0
 	github.com/stretchr/testify v1.5.1
@@ -20,4 +22,6 @@ require (
 	golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gorm.io/driver/sqlite v1.1.3
+	gorm.io/gorm v1.20.2
 )

go.sum

@@ -153,6 +153,8 @@ github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBef
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
+github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
@@ -186,6 +188,10 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
+github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA=
+github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
+github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI=
+github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -441,6 +447,11 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc=
+gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c=
+gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+gorm.io/gorm v1.20.2 h1:bZzSEnq7NDGsrd+n3evOOedDrY5oLM5QPlCjZJUK2ro=
+gorm.io/gorm v1.20.2/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
 gosrc.io/xmpp v0.1.1 h1:iMtE9W3fx254+4E6rI34AOPJDqWvpfQR6EYaVMzhJ4s=
 gosrc.io/xmpp v0.1.1/go.mod h1:4JgaXzw4MnEv2sGltONtK3GMhj+h9gpQ7cO8nwbFJLU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

webapp/backend/cmd/scrutiny/scrutiny.go

@@ -60,7 +60,7 @@ OPTIONS:
 		},
 		Before: func(c *cli.Context) error {
 
-			drawbridge := "github.com/AnalogJ/scrutiny"
+			scrutiny := "github.com/AnalogJ/scrutiny"
 
 			var versionInfo string
 			if len(goos) > 0 && len(goarch) > 0 {
@@ -69,7 +69,7 @@ OPTIONS:
 				versionInfo = fmt.Sprintf("dev-%s", version.VERSION)
 			}
 
-			subtitle := drawbridge + utils.LeftPad2Len(versionInfo, " ", 65-len(drawbridge))
+			subtitle := scrutiny + utils.LeftPad2Len(versionInfo, " ", 65-len(scrutiny))
 
 			color.New(color.FgGreen).Fprintf(c.App.Writer, fmt.Sprintf(utils.StripIndent(
 				`

webapp/backend/pkg/models/db/device.go

@@ -21,7 +21,8 @@ type Device struct {
 	UpdatedAt time.Time
 	DeletedAt *time.Time
 
-	WWN string `json:"wwn" gorm:"primary_key"`
+	WWN    string `json:"wwn" gorm:"primary_key"`
+	HostId string `json:"host_id"`
 
 	DeviceName     string  `json:"device_name"`
 	Manufacturer   string  `json:"manufacturer"`
@@ -151,6 +152,8 @@ func (dv *Device) ApplyMetadataRules() error {
 	return nil
 }
 
+// This function is called every time the collector sends SMART data to the API.
+// It can be used to update device data that can change over time.
 func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
 	dv.Firmware = info.FirmwareVersion
 	return nil

webapp/backend/pkg/models/db/smart.go

@@ -3,7 +3,7 @@ package db
 import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"github.com/jinzhu/gorm"
+	"gorm.io/gorm"
 	"time"
 )
 

webapp/backend/pkg/models/db/smart_ata_attribute.go

@@ -2,7 +2,7 @@ package db
 
 import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"github.com/jinzhu/gorm"
+	"gorm.io/gorm"
 	"strings"
 )
 

webapp/backend/pkg/models/db/smart_scsci_attribute.go

@@ -2,7 +2,7 @@ package db
 
 import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"github.com/jinzhu/gorm"
+	"gorm.io/gorm"
 )
 
 type SmartScsiAttribute struct {

webapp/backend/pkg/web/handler/get_device_details.go

@@ -4,8 +4,8 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
-	"github.com/jinzhu/gorm"
 	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
 	"net/http"
 )
 

webapp/backend/pkg/web/handler/get_devices_summary.go

@@ -3,8 +3,8 @@ package handler
 import (
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
-	"github.com/jinzhu/gorm"
 	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
 	"net/http"
 )
 

webapp/backend/pkg/web/handler/register_devices.go

@@ -3,12 +3,15 @@ package handler
 import (
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
-	"github.com/jinzhu/gorm"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+
 	"github.com/sirupsen/logrus"
 	"net/http"
 )
 
-// filter devices that are detected by various collectors.
+// register devices that are detected by various collectors.
+// This function is run everytime a collector is about to start a run. It can be used to update device data.
 func RegisterDevices(c *gin.Context) {
 	db := c.MustGet("DB").(*gorm.DB)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
@@ -21,11 +24,15 @@ func RegisterDevices(c *gin.Context) {
 		return
 	}
 
-	//TODO: filter devices here (remove excludes, force includes)
 	errs := []error{}
 	for _, dev := range collectorDeviceWrapper.Data {
-		//insert devices into DB if not already there.
+		//insert devices into DB (and update specified columns if device is already registered)
-		if err := db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev).Error; err != nil {
+		// update device fields that may change: (DeviceType, HostID)
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "wwn"}},
+			DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name"}),
+		}).Create(&dev).Error; err != nil {
+
 			errs = append(errs, err)
 		}
 	}

webapp/backend/pkg/web/middleware/sqlite3.go

@@ -5,20 +5,24 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
-	"github.com/jinzhu/gorm"
-	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/sirupsen/logrus"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
 )
 
-func DatabaseMiddleware(appConfig config.Interface, logger logrus.FieldLogger) gin.HandlerFunc {
+func DatabaseMiddleware(appConfig config.Interface, globalLogger logrus.FieldLogger) gin.HandlerFunc {
+
 	//var database *gorm.DB
 	fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location"))
-	database, err := gorm.Open("sqlite3", appConfig.GetString("web.database.location"))
+	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
+		//TODO: figure out how to log database queries again.
+		//Logger: logger
+	})
 	if err != nil {
 		panic("Failed to connect to database!")
 	}
 
-	database.SetLogger(&GormLogger{Logger: logger})
+	//database.SetLogger()
 	database.AutoMigrate(&db.Device{})
 	database.AutoMigrate(&db.SelfTest{})
 	database.AutoMigrate(&db.Smart{})