gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding support for a collecto config file.

Browse Source 
/scrutiny/config/collector.yaml

Adding ability to specify host identifier (label), that is updated on every collector run.
Can be specified by `host-id` CLI or `COLLECTOR_HOST_ID` env var.

Created a config class, interface and associated tests.

Created a "TransformDetectedDrives" function, that will allow users to insert drives not detected by Smarctl --scan, ignore drives that they dont want, and override smartctl device type.

Added Upsert functionality when registering devices.

Replaced "github.com/jinzhu/gorm" with "gorm.io/gorm" (ORM location moved, was using incorrect lib url)
Removed machineid library.
This commit is contained in:
Jason Kulatunga
2020-10-07 21:54:29 -06:00
parent 32e7044c67
commit b44ef5cb9c
33 changed files with 851 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -128,7 +128,7 @@ docker exec scrutiny /scrutiny/bin/scrutiny-collector-metrics run
```
# Configuration
We support a global YAML configuration file that must be located at /scrutiny/config/scrutiny.yaml
We support a global YAML configuration file that must be located at `/scrutiny/config/scrutiny.yaml`
Check the [example.scrutiny.yml](example.scrutiny.yaml) file for a fully commented version.
collector/cmd/collector-metrics/collector-metrics.go
+39 -1
View File
@@ -3,6 +3,8 @@ package main
import (
"fmt"
"github.com/analogj/scrutiny/collector/pkg/collector"
"github.com/analogj/scrutiny/collector/pkg/config"
"github.com/analogj/scrutiny/collector/pkg/errors"
"github.com/analogj/scrutiny/webapp/backend/pkg/version"
"github.com/sirupsen/logrus"
"io"
@@ -20,6 +22,20 @@ var goarch string
func main() {
config, err := config.Create()
if err != nil {
fmt.Printf("FATAL: %+v\n", err)
os.Exit(1)
}
//we're going to load the config file manually, since we need to validate it.
err = config.ReadConfig("/scrutiny/config/collector.yaml") // Find and read the config file
if _, ok := err.(errors.ConfigFileMissingError); ok { // Handle errors reading the config file
//ignore "could not find config file"
} else if err != nil {
os.Exit(1)
}
cli.CommandHelpTemplate = `NAME:
{{.HelpName}} - {{.Usage}}
USAGE:
@@ -75,6 +91,17 @@ OPTIONS:
Name: "run",
Usage: "Run the scrutiny smartctl metrics collector",
Action: func(c *cli.Context) error {
if c.IsSet("config") {
err = config.ReadConfig(c.String("config")) // Find and read the config file
if err != nil { // Handle errors reading the config file
//ignore "could not find config file"
fmt.Printf("Could not find config file at specified path: %s", c.String("config"))
return err
}
}
if c.IsSet("host-id") {
config.Set("host.id", c.String("host-id")) // set/override the host-id using CLI.
}
collectorLogger := logrus.WithFields(logrus.Fields{
"type": "metrics",
@@ -97,6 +124,7 @@ OPTIONS:
}
metricCollector, err := collector.CreateMetricsCollector(
config,
collectorLogger,
c.String("api-endpoint"),
)
@@ -109,6 +137,10 @@ OPTIONS:
},
Flags: []cli.Flag{
&cli.StringFlag{
Name: "config",
Usage: "Specify the path to the devices file",
},
&cli.StringFlag{
Name: "api-endpoint",
Usage: "The api server endpoint",
@@ -128,12 +160,18 @@ OPTIONS:
Usage: "Enable debug logging",
EnvVars: []string{"COLLECTOR_DEBUG", "DEBUG"},
},
&cli.BoolFlag{
Name: "host-id",
Usage: "Host identifier/label, used for grouping devices",
EnvVars: []string{"COLLECTOR_HOST_ID"},
},
},
},
},
}
err := app.Run(os.Args)
err = app.Run(os.Args)
if err != nil {
log.Fatal(color.HiRedString("ERROR: %v", err))
}
collector/pkg/collector/metrics.go
+5 -1
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"github.com/analogj/scrutiny/collector/pkg/common"
"github.com/analogj/scrutiny/collector/pkg/config"
"github.com/analogj/scrutiny/collector/pkg/detect"
"github.com/analogj/scrutiny/collector/pkg/errors"
"github.com/analogj/scrutiny/collector/pkg/models"
@@ -16,17 +17,19 @@ import (
)
type MetricsCollector struct {
config config.Interface
BaseCollector
apiEndpoint *url.URL
}
func CreateMetricsCollector(logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
func CreateMetricsCollector(appConfig config.Interface, logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
apiEndpointUrl, err := url.Parse(apiEndpoint)
if err != nil {
return MetricsCollector{}, err
}
sc := MetricsCollector{
config: appConfig,
apiEndpoint: apiEndpointUrl,
BaseCollector: BaseCollector{
logger: logger,
@@ -49,6 +52,7 @@ func (mc *MetricsCollector) Run() error {
deviceDetector := detect.Detect{
Logger: mc.logger,
Config: mc.config,
}
detectedStorageDevices, err := deviceDetector.Start()
if err != nil {
collector/pkg/config/config.go
+100
View File
@@ -0,0 +1,100 @@
package config
import (
"github.com/analogj/go-util/utils"
"github.com/analogj/scrutiny/collector/pkg/errors"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/mitchellh/mapstructure"
"github.com/spf13/viper"
"log"
"os"
)
// When initializing this class the following methods must be called:
// Config.New
// Config.Init
// This is done automatically when created via the Factory.
type configuration struct {
*viper.Viper
}
//Viper uses the following precedence order. Each item takes precedence over the item below it:
// explicit call to Set
// flag
// env
// config
// key/value store
// default
func (c *configuration) Init() error {
c.Viper = viper.New()
//set defaults
c.SetDefault("host.id", "")
c.SetDefault("devices", []string{})
//c.SetDefault("collect.short.command", "-a -o on -S on")
//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
c.SetConfigType("yaml")
//c.SetConfigName("drawbridge")
//c.AddConfigPath("$HOME/")
//CLI options will be added via the `Set()` function
return nil
}
func (c *configuration) ReadConfig(configFilePath string) error {
configFilePath, err := utils.ExpandPath(configFilePath)
if err != nil {
return err
}
if !utils.FileExists(configFilePath) {
log.Printf("No configuration file found at %v. Using Defaults.", configFilePath)
return errors.ConfigFileMissingError("The configuration file could not be found.")
}
//validate config file contents
//err = c.ValidateConfigFile(configFilePath)
//if err != nil {
// log.Printf("Config file at `%v` is invalid: %s", configFilePath, err)
// return err
//}
log.Printf("Loading configuration file: %s", configFilePath)
config_data, err := os.Open(configFilePath)
if err != nil {
log.Printf("Error reading configuration file: %s", err)
return err
}
err = c.MergeConfig(config_data)
if err != nil {
return err
}
return c.ValidateConfig()
}
// This function ensures that the merged config works correctly.
func (c *configuration) ValidateConfig() error {
//TODO:
// check that device prefix matches OS
// check that schema of config file is valid
return nil
}
func (c *configuration) GetScanOverrides() []models.ScanOverride {
// we have to support 2 types of device types.
// - simple device type (device_type: 'sat')
// and list of device types (type: \n- 3ware,0 \n- 3ware,1 \n- 3ware,2)
// GetString will return "" if this is a list of device types.
overrides := []models.ScanOverride{}
c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
return overrides
}
collector/pkg/config/config_test.go
+64
View File
@@ -0,0 +1,64 @@
package config_test
import (
"github.com/analogj/scrutiny/collector/pkg/config"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/stretchr/testify/require"
"path"
"testing"
)
func TestConfiguration_GetScanOverrides_Simple(t *testing.T) {
t.Parallel()
//setup
testConfig, _ := config.Create()
//test
err := testConfig.ReadConfig(path.Join("testdata", "simple_device.yaml"))
require.NoError(t, err, "should correctly load simple device config")
scanOverrides := testConfig.GetScanOverrides()
//assert
require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat"}, Ignore: false}}, scanOverrides)
}
func TestConfiguration_GetScanOverrides_Ignore(t *testing.T) {
t.Parallel()
//setup
testConfig, _ := config.Create()
//test
err := testConfig.ReadConfig(path.Join("testdata", "ignore_device.yaml"))
require.NoError(t, err, "should correctly load ignore device config")
scanOverrides := testConfig.GetScanOverrides()
//assert
require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}, scanOverrides)
}
func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
t.Parallel()
//setup
testConfig, _ := config.Create()
//test
err := testConfig.ReadConfig(path.Join("testdata", "raid_device.yaml"))
require.NoError(t, err, "should correctly load ignore device config")
scanOverrides := testConfig.GetScanOverrides()
//assert
require.Equal(t, []models.ScanOverride{
{
Device: "/dev/bus/0",
DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
Ignore: false,
},
{
Device: "/dev/twa0",
DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
Ignore: false,
}}, scanOverrides)
}
collector/pkg/config/factory.go
+9
View File
@@ -0,0 +1,9 @@
package config
func Create() (Interface, error) {
config := new(configuration)
if err := config.Init(); err != nil {
return nil, err
}
return config, nil
}
collector/pkg/config/interface.go
+26
View File
@@ -0,0 +1,26 @@
package config
import (
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/spf13/viper"
)
// Create mock using:
// mockgen -source=collector/pkg/config/interface.go -destination=collector/pkg/config/mock/mock_config.go
type Interface interface {
Init() error
ReadConfig(configFilePath string) error
Set(key string, value interface{})
SetDefault(key string, value interface{})
AllSettings() map[string]interface{}
IsSet(key string) bool
Get(key string) interface{}
GetBool(key string) bool
GetInt(key string) int
GetString(key string) string
GetStringSlice(key string) []string
UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
GetScanOverrides() []models.ScanOverride
}
collector/pkg/config/mock/mock_config.go
+218
View File
@@ -0,0 +1,218 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: collector/pkg/config/interface.go
// Package mock_config is a generated GoMock package.
package mock_config
import (
models "github.com/analogj/scrutiny/collector/pkg/models"
gomock "github.com/golang/mock/gomock"
viper "github.com/spf13/viper"
reflect "reflect"
)
// MockInterface is a mock of Interface interface
type MockInterface struct {
ctrl *gomock.Controller
recorder *MockInterfaceMockRecorder
}
// MockInterfaceMockRecorder is the mock recorder for MockInterface
type MockInterfaceMockRecorder struct {
mock *MockInterface
}
// NewMockInterface creates a new mock instance
func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
mock := &MockInterface{ctrl: ctrl}
mock.recorder = &MockInterfaceMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
return m.recorder
}
// Init mocks base method
func (m *MockInterface) Init() error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Init")
ret0, _ := ret[0].(error)
return ret0
}
// Init indicates an expected call of Init
func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
}
// ReadConfig mocks base method
func (m *MockInterface) ReadConfig(configFilePath string) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
ret0, _ := ret[0].(error)
return ret0
}
// ReadConfig indicates an expected call of ReadConfig
func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
}
// Set mocks base method
func (m *MockInterface) Set(key string, value interface{}) {
m.ctrl.T.Helper()
m.ctrl.Call(m, "Set", key, value)
}
// Set indicates an expected call of Set
func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
}
// SetDefault mocks base method
func (m *MockInterface) SetDefault(key string, value interface{}) {
m.ctrl.T.Helper()
m.ctrl.Call(m, "SetDefault", key, value)
}
// SetDefault indicates an expected call of SetDefault
func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
}
// AllSettings mocks base method
func (m *MockInterface) AllSettings() map[string]interface{} {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AllSettings")
ret0, _ := ret[0].(map[string]interface{})
return ret0
}
// AllSettings indicates an expected call of AllSettings
func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings))
}
// IsSet mocks base method
func (m *MockInterface) IsSet(key string) bool {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "IsSet", key)
ret0, _ := ret[0].(bool)
return ret0
}
// IsSet indicates an expected call of IsSet
func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
}
// Get mocks base method
func (m *MockInterface) Get(key string) interface{} {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Get", key)
ret0, _ := ret[0].(interface{})
return ret0
}
// Get indicates an expected call of Get
func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key)
}
// GetBool mocks base method
func (m *MockInterface) GetBool(key string) bool {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetBool", key)
ret0, _ := ret[0].(bool)
return ret0
}
// GetBool indicates an expected call of GetBool
func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key)
}
// GetInt mocks base method
func (m *MockInterface) GetInt(key string) int {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetInt", key)
ret0, _ := ret[0].(int)
return ret0
}
// GetInt indicates an expected call of GetInt
func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
}
// GetString mocks base method
func (m *MockInterface) GetString(key string) string {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetString", key)
ret0, _ := ret[0].(string)
return ret0
}
// GetString indicates an expected call of GetString
func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key)
}
// GetStringSlice mocks base method
func (m *MockInterface) GetStringSlice(key string) []string {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetStringSlice", key)
ret0, _ := ret[0].([]string)
return ret0
}
// GetStringSlice indicates an expected call of GetStringSlice
func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key)
}
// UnmarshalKey mocks base method
func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
m.ctrl.T.Helper()
varargs := []interface{}{key, rawVal}
for _, a := range decoderOpts {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "UnmarshalKey", varargs...)
ret0, _ := ret[0].(error)
return ret0
}
// UnmarshalKey indicates an expected call of UnmarshalKey
func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]interface{}{key, rawVal}, decoderOpts...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...)
}
// GetScanOverrides mocks base method
func (m *MockInterface) GetScanOverrides() []models.ScanOverride {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetScanOverrides")
ret0, _ := ret[0].([]models.ScanOverride)
return ret0
}
// GetScanOverrides indicates an expected call of GetScanOverrides
func (mr *MockInterfaceMockRecorder) GetScanOverrides() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScanOverrides", reflect.TypeOf((*MockInterface)(nil).GetScanOverrides))
}
collector/pkg/config/testdata/ignore_device.yaml
Vendored
+4
View File
@@ -0,0 +1,4 @@
version: 1
devices:
- device: /dev/sda
ignore: true
collector/pkg/config/testdata/raid_device.yaml
Vendored
+19
View File
@@ -0,0 +1,19 @@
version: 1
devices:
- device: /dev/bus/0
type:
- megaraid,14
- megaraid,15
- megaraid,18
- megaraid,19
- megaraid,20
- megaraid,21
- device: /dev/twa0
type:
- 3ware,0
- 3ware,1
- 3ware,2
- 3ware,3
- 3ware,4
- 3ware,5
collector/pkg/config/testdata/simple_device.yaml
Vendored
+27
View File
@@ -0,0 +1,27 @@
version: 1
devices:
- device: /dev/sda
type: 'sat'
#
# # example to show how to ignore a specific disk/device.
# - device: /dev/sda
# ignore: true
#
# # examples showing how to force smartctl to detect disks inside a raid array/virtual disk
# - device: /dev/bus/0
# type:
# - megaraid,14
# - megaraid,15
# - megaraid,18
# - megaraid,19
# - megaraid,20
# - megaraid,21
#
# - device: /dev/twa0
# type:
# - 3ware,0
# - 3ware,1
# - 3ware,2
# - 3ware,3
# - 3ware,4
# - 3ware,5
collector/pkg/detect/detect.go
+63 -16
View File
@@ -4,9 +4,9 @@ import (
"encoding/json"
"fmt"
"github.com/analogj/scrutiny/collector/pkg/common"
"github.com/analogj/scrutiny/collector/pkg/config"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
"github.com/denisbrodbeck/machineid"
"github.com/sirupsen/logrus"
"os"
"strings"
@@ -14,6 +14,7 @@ import (
type Detect struct {
Logger *logrus.Entry
Config config.Interface
}
//private/common functions
@@ -24,7 +25,7 @@ type Detect struct {
//
// To handle these issues, we have OS specific wrapper functions that update/modify these detected devices.
// models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
func (d *Detect) smartctlScan() ([]models.Device, error) {
func (d *Detect) SmartctlScan() ([]models.Device, error) {
//we use smartctl to detect all the drives available.
detectedDeviceConnJson, err := common.ExecCmd(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
if err != nil {
@@ -39,14 +40,7 @@ func (d *Detect) smartctlScan() ([]models.Device, error) {
return nil, err
}
detectedDevices := []models.Device{}
for _, detectedDevice := range detectedDeviceConns.Devices {
detectedDevices = append(detectedDevices, models.Device{
DeviceType: detectedDevice.Type,
DeviceName: strings.TrimPrefix(detectedDevice.Name, DevicePrefix()),
})
}
detectedDevices := d.TransformDetectedDevices(detectedDeviceConns)
return detectedDevices, nil
}
@@ -55,7 +49,7 @@ func (d *Detect) smartctlScan() ([]models.Device, error) {
// It has a couple of issues however:
// - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
// - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
func (d *Detect) smartCtlInfo(device *models.Device) error {
func (d *Detect) SmartCtlInfo(device *models.Device) error {
args := []string{"--info", "-j"}
//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
@@ -77,8 +71,8 @@ func (d *Detect) smartCtlInfo(device *models.Device) error {
return err
}
//DeviceType and DeviceName are already populated.
//WWN
//WWN: this is a serial number/world-wide number that will not change.
//DeviceType and DeviceName are already populated, however may change between collector runs (eg. config/host restart)
//InterfaceType:
device.ModelName = availableDeviceInfo.ModelName
device.InterfaceSpeed = availableDeviceInfo.InterfaceSpeed.Current.String
@@ -110,7 +104,60 @@ func (d *Detect) smartCtlInfo(device *models.Device) error {
return nil
}
//uses https://github.com/denisbrodbeck/machineid to get a OS specific unique machine ID.
func (d *Detect) getMachineId() (string, error) {
return machineid.ProtectedID("scrutiny")
// function will remove devices that are marked for "ignore" in config file
// will also add devices that are specified in config file, but "missing" from smartctl --scan
// this function will also update the deviceType to the option specified in config.
func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []models.Device {
groupedDevices := map[string][]models.Device{}
for _, scannedDevice := range detectedDeviceConns.Devices {
deviceFile := strings.ToLower(scannedDevice.Name)
detectedDevice := models.Device{
HostId: d.Config.GetString("host.id"),
DeviceType: scannedDevice.Type,
DeviceName: strings.TrimPrefix(deviceFile, DevicePrefix()),
}
//find (or create) a slice to contain the devices in this group
if groupedDevices[deviceFile] == nil {
groupedDevices[deviceFile] = []models.Device{}
}
// add this scanned device to the group
groupedDevices[deviceFile] = append(groupedDevices[deviceFile], detectedDevice)
}
//now tha we've "grouped" all the devices, lets override any groups specified in the config file.
for _, overrideDevice := range d.Config.GetScanOverrides() {
overrideDeviceFile := strings.ToLower(overrideDevice.Device)
if overrideDevice.Ignore {
// this device file should be deleted if it exists
delete(groupedDevices, overrideDeviceFile)
} else {
//create a new device group, and replace the one generated by smartctl --scan
overrideDeviceGroup := []models.Device{}
for _, overrideDeviceType := range overrideDevice.DeviceType {
overrideDeviceGroup = append(overrideDeviceGroup, models.Device{
HostId: d.Config.GetString("host.id"),
DeviceType: overrideDeviceType,
DeviceName: strings.TrimPrefix(overrideDeviceFile, DevicePrefix()),
})
}
groupedDevices[overrideDeviceFile] = overrideDeviceGroup
}
}
//flatten map
detectedDevices := []models.Device{}
for _, group := range groupedDevices {
detectedDevices = append(detectedDevices, group...)
}
return detectedDevices
}
collector/pkg/detect/detect_test.go
+138
View File
@@ -0,0 +1,138 @@
package detect_test
import (
mock_config "github.com/analogj/scrutiny/collector/pkg/config/mock"
"github.com/analogj/scrutiny/collector/pkg/detect"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/require"
"testing"
)
func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
//setup
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
detectedDevices := models.Scan{
Devices: []models.ScanDevice{
{
Name: "/dev/sda",
InfoName: "/dev/sda",
Protocol: "scsi",
Type: "scsi",
},
},
}
d := detect.Detect{
Config: fakeConfig,
}
//test
transformedDevices := d.TransformDetectedDevices(detectedDevices)
//assert
require.Equal(t, "sda", transformedDevices[0].DeviceName)
require.Equal(t, "scsi", transformedDevices[0].DeviceType)
}
func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
//setup
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
detectedDevices := models.Scan{
Devices: []models.ScanDevice{
{
Name: "/dev/sda",
InfoName: "/dev/sda",
Protocol: "scsi",
Type: "scsi",
},
},
}
d := detect.Detect{
Config: fakeConfig,
}
//test
transformedDevices := d.TransformDetectedDevices(detectedDevices)
//assert
require.Equal(t, []models.Device{}, transformedDevices)
}
func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
//setup
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{
{
Device: "/dev/bus/0",
DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
Ignore: false,
},
{
Device: "/dev/twa0",
DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
Ignore: false,
}})
detectedDevices := models.Scan{
Devices: []models.ScanDevice{
{
Name: "/dev/bus/0",
InfoName: "/dev/bus/0",
Protocol: "scsi",
Type: "scsi",
},
},
}
d := detect.Detect{
Config: fakeConfig,
}
//test
transformedDevices := d.TransformDetectedDevices(detectedDevices)
//assert
require.Equal(t, 12, len(transformedDevices))
}
func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
//setup
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
detectedDevices := models.Scan{
Devices: []models.ScanDevice{
{
Name: "/dev/sda",
InfoName: "/dev/sda",
Protocol: "ata",
Type: "ata",
},
},
}
d := detect.Detect{
Config: fakeConfig,
}
//test
transformedDevices := d.TransformDetectedDevices(detectedDevices)
//assert
require.Equal(t, 1, len(transformedDevices))
require.Equal(t, "sat+megaraid", transformedDevices[0].DeviceType)
}
collector/pkg/detect/devices_darwin.go
+2 -2
View File
@@ -12,7 +12,7 @@ func DevicePrefix() string {
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devicess
detectedDevices, err := d.smartctlScan()
detectedDevices, err := d.SmartctlScan()
if err != nil {
return nil, err
}
@@ -25,7 +25,7 @@ func (d *Detect) Start() ([]models.Device, error) {
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
collector/pkg/detect/devices_freebsd.go
+2 -2
View File
@@ -12,14 +12,14 @@ func DevicePrefix() string {
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devices
detectedDevices, err := d.smartctlScan()
detectedDevices, err := d.SmartctlScan()
if err != nil {
return nil, err
}
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
collector/pkg/detect/devices_linux.go
+2 -2
View File
@@ -12,14 +12,14 @@ func DevicePrefix() string {
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devices
detectedDevices, err := d.smartctlScan()
detectedDevices, err := d.SmartctlScan()
if err != nil {
return nil, err
}
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
collector/pkg/detect/devices_windows.go
+2 -2
View File
@@ -11,14 +11,14 @@ func DevicePrefix() string {
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devices
detectedDevices, err := d.smartctlScan()
detectedDevices, err := d.SmartctlScan()
if err != nil {
return nil, err
}
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
collector/pkg/models/device.go
+1
View File
@@ -2,6 +2,7 @@ package models
type Device struct {
WWN string `json:"wwn"`
HostId string `json:"host_id"`
DeviceName string `json:"device_name"`
Manufacturer string `json:"manufacturer"`
collector/pkg/models/scan.go
+3 -2
View File
@@ -10,10 +10,11 @@ type Scan struct {
Argv []string `json:"argv"`
ExitStatus int `json:"exit_status"`
} `json:"smartctl"`
Devices []struct {
Devices []ScanDevice `json:"devices"`
}
type ScanDevice struct {
Name string `json:"name"`
InfoName string `json:"info_name"`
Type string `json:"type"`
Protocol string `json:"protocol"`
} `json:"devices"`
}
collector/pkg/models/scan_override.go
+7
View File
@@ -0,0 +1,7 @@
package models
type ScanOverride struct {
Device string `mapstructure:"device"`
DeviceType []string `mapstructure:"type"`
Ignore bool `mapstructure:"ignore"`
}
example.collector.yaml
+65
View File
@@ -0,0 +1,65 @@
# Commented Scrutiny Configuration File
#
# The default location for this file is /scrutiny/config/collector.yaml.
# In some cases to improve clarity default values are specified,
# uncommented. Other example values are commented out.
#
# When this file is parsed by Scrutiny, all configuration file keys are
# lowercased automatically. As such, Configuration keys are case-insensitive,
# and should be lowercase in this file to be consistent with usage.
######################################################################
# Version
#
# version specifies the version of this configuration file schema, not
# the scrutiny binary. There is only 1 version available at the moment
version: 1
# The host id is a label used for identifying groups of disks running on the same host
# Primiarly used for hub/spoke deployments (can be left empty if using all-in-one image).
host:
id: ""
# This block allows you to override/customize the settings for devices detected by
# Scrutiny via `smartctl --scan`
# See the "--device=TYPE" section of https://linux.die.net/man/8/smartctl
# type can be a 'string' or a 'list'
devices:
# # example for forcing device type detection for a single disk
# - device: /dev/sda
# type: 'sat'
#
# # example to show how to ignore a specific disk/device.
# - device: /dev/sda
# ignore: true
#
# # examples showing how to force smartctl to detect disks inside a raid array/virtual disk
# - device: /dev/bus/0
# type:
# - megaraid,14
# - megaraid,15
# - megaraid,18
# - megaraid,19
# - megaraid,20
# - megaraid,21
#
# - device: /dev/twa0
# type:
# - 3ware,0
# - 3ware,1
# - 3ware,2
# - 3ware,3
# - 3ware,4
# - 3ware,5
########################################################################################################################
# FEATURES COMING SOON
#
# The following commented out sections are a preview of additional configuration options that will be available soon.
#
########################################################################################################################
example.scrutiny.yaml
+1 -1
View File
@@ -1,6 +1,6 @@
# Commented Scrutiny Configuration File
#
# The default location for this file is ~/scrutiny.yaml.
# The default location for this file is /scrutiny/config/scrutiny.yaml.
# In some cases to improve clarity default values are specified,
# uncommented. Other example values are commented out.
#
go.mod
+5 -1
View File
@@ -6,13 +6,15 @@ require (
github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae // indirect
github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a
github.com/denisbrodbeck/machineid v1.0.1
github.com/denisbrodbeck/machineid v1.0.1 // indirect
github.com/fatih/color v1.9.0
github.com/gin-gonic/gin v1.6.3
github.com/golang/mock v1.4.3
github.com/jaypipes/ghw v0.6.1
github.com/jinzhu/gorm v1.9.14
github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect
github.com/mattn/go-sqlite3 v1.14.4
github.com/mitchellh/mapstructure v1.2.2
github.com/sirupsen/logrus v1.2.0
github.com/spf13/viper v1.7.0
github.com/stretchr/testify v1.5.1
@@ -20,4 +22,6 @@ require (
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
golang.org/x/sync v0.0.0-20190423024810-112230192c58
gopkg.in/yaml.v2 v2.3.0 // indirect
gorm.io/driver/sqlite v1.1.3
gorm.io/gorm v1.20.2
)
go.sum
+11
View File
@@ -153,6 +153,8 @@ github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBef
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
@@ -186,6 +188,10 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA=
github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI=
github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -441,6 +447,11 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc=
gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c=
gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
gorm.io/gorm v1.20.2 h1:bZzSEnq7NDGsrd+n3evOOedDrY5oLM5QPlCjZJUK2ro=
gorm.io/gorm v1.20.2/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
gosrc.io/xmpp v0.1.1 h1:iMtE9W3fx254+4E6rI34AOPJDqWvpfQR6EYaVMzhJ4s=
gosrc.io/xmpp v0.1.1/go.mod h1:4JgaXzw4MnEv2sGltONtK3GMhj+h9gpQ7cO8nwbFJLU=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
webapp/backend/cmd/scrutiny/scrutiny.go
+2 -2
View File
@@ -60,7 +60,7 @@ OPTIONS:
},
Before: func(c *cli.Context) error {
drawbridge := "github.com/AnalogJ/scrutiny"
scrutiny := "github.com/AnalogJ/scrutiny"
var versionInfo string
if len(goos) > 0 && len(goarch) > 0 {
@@ -69,7 +69,7 @@ OPTIONS:
versionInfo = fmt.Sprintf("dev-%s", version.VERSION)
}
subtitle := drawbridge + utils.LeftPad2Len(versionInfo, " ", 65-len(drawbridge))
subtitle := scrutiny + utils.LeftPad2Len(versionInfo, " ", 65-len(scrutiny))
color.New(color.FgGreen).Fprintf(c.App.Writer, fmt.Sprintf(utils.StripIndent(
`
webapp/backend/pkg/models/db/device.go
+3
View File
@@ -22,6 +22,7 @@ type Device struct {
DeletedAt *time.Time
WWN string `json:"wwn" gorm:"primary_key"`
HostId string `json:"host_id"`
DeviceName string `json:"device_name"`
Manufacturer string `json:"manufacturer"`
@@ -151,6 +152,8 @@ func (dv *Device) ApplyMetadataRules() error {
return nil
}
// This function is called every time the collector sends SMART data to the API.
// It can be used to update device data that can change over time.
func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
dv.Firmware = info.FirmwareVersion
return nil
webapp/backend/pkg/models/db/smart.go
+1 -1
View File
@@ -3,7 +3,7 @@ package db
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
"github.com/jinzhu/gorm"
"gorm.io/gorm"
"time"
)
webapp/backend/pkg/models/db/smart_ata_attribute.go
+1 -1
View File
@@ -2,7 +2,7 @@ package db
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
"github.com/jinzhu/gorm"
"gorm.io/gorm"
"strings"
)
webapp/backend/pkg/models/db/smart_scsci_attribute.go
+1 -1
View File
@@ -2,7 +2,7 @@ package db
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
"github.com/jinzhu/gorm"
"gorm.io/gorm"
)
type SmartScsiAttribute struct {
webapp/backend/pkg/web/handler/get_device_details.go
+1 -1
View File
@@ -4,8 +4,8 @@ import (
"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"net/http"
)
webapp/backend/pkg/web/handler/get_devices_summary.go
+1 -1
View File
@@ -3,8 +3,8 @@ package handler
import (
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"net/http"
)
webapp/backend/pkg/web/handler/register_devices.go
+12 -5
View File
@@ -3,12 +3,15 @@ package handler
import (
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/sirupsen/logrus"
"net/http"
)
// filter devices that are detected by various collectors.
// register devices that are detected by various collectors.
// This function is run everytime a collector is about to start a run. It can be used to update device data.
func RegisterDevices(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
@@ -21,11 +24,15 @@ func RegisterDevices(c *gin.Context) {
return
}
//TODO: filter devices here (remove excludes, force includes)
errs := []error{}
for _, dev := range collectorDeviceWrapper.Data {
//insert devices into DB if not already there.
if err := db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev).Error; err != nil {
//insert devices into DB (and update specified columns if device is already registered)
// update device fields that may change: (DeviceType, HostID)
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "wwn"}},
DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name"}),
}).Create(&dev).Error; err != nil {
errs = append(errs, err)
}
}
webapp/backend/pkg/web/middleware/sqlite3.go
+9 -5
View File
@@ -5,20 +5,24 @@ import (
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/sirupsen/logrus"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func DatabaseMiddleware(appConfig config.Interface, logger logrus.FieldLogger) gin.HandlerFunc {
func DatabaseMiddleware(appConfig config.Interface, globalLogger logrus.FieldLogger) gin.HandlerFunc {
//var database *gorm.DB
fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location"))
database, err := gorm.Open("sqlite3", appConfig.GetString("web.database.location"))
database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
//TODO: figure out how to log database queries again.
//Logger: logger
})
if err != nil {
panic("Failed to connect to database!")
}
database.SetLogger(&GormLogger{Logger: logger})
//database.SetLogger()
database.AutoMigrate(&db.Device{})
database.AutoMigrate(&db.SelfTest{})
database.AutoMigrate(&db.Smart{})