(v0.4.9) Automated packaging of release by Packagr

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

.dockerignore

@@ -1,4 +1,3 @@
-/dist
 /vendor
 /.idea
 /.github

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,20 +22,19 @@ See [/docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](docs/TROUBLESHOOTING_DEVICE_COLL
 
 ```
 docker run -it --rm -p 8080:8080 \
+-v `pwd`/config:/opt/scrutiny/config \
 -v /run/udev:/run/udev:ro \
 --cap-add SYS_RAWIO \
 --device=/dev/sda \
 --device=/dev/sdb \
 -e DEBUG=true \
--e COLLECTOR_LOG_FILE=/tmp/collector.log \
+-e COLLECTOR_LOG_FILE=/opt/scrutiny/config/collector.log \
--e SCRUTINY_LOG_FILE=/tmp/web.log \
+-e SCRUTINY_LOG_FILE=/opt/scrutiny/config/web.log \
 --name scrutiny \
 ghcr.io/analogj/scrutiny:master-omnibus
 
 # in another terminal trigger the collector
 docker exec scrutiny scrutiny-collector-metrics run
-
-# then use docker cp to copy the log files out of the container.
-docker cp scrutiny:/tmp/collector.log collector.log
-docker cp scrutiny:/tmp/web.log web.log
 ```
+
+The log files will be available on your host in the `config` directory. Please attach them to this issue. 

.github/workflows/docker-build.yaml

@@ -25,6 +25,8 @@ jobs:
           fetch-depth: 0
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       # Login against a Docker registry except on PR
@@ -60,8 +62,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
+#          cache-from: type=gha
-          cache-to: type=gha,mode=max
+#          cache-to: type=gha,mode=max
 
   web:
     runs-on: ubuntu-latest
@@ -72,8 +74,22 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: "Populate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
+      - name: "Generate frontend"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: node:lts
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            cd /work
+            make frontend
+            ls -alt /work
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       # Login against a Docker registry except on PR
@@ -97,8 +113,6 @@ jobs:
             type=ref,enable=true,event=branch,suffix=-web
             type=ref,enable=true,event=tag,suffix=-web
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-      - name: "Generate frontend version information"
-        run: "cd webapp/frontend && ./git.version.sh"
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
@@ -110,8 +124,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
+#          cache-from: type=gha
-          cache-to: type=gha,mode=max
+#          cache-to: type=gha,mode=max
   omnibus:
     runs-on: ubuntu-latest
     permissions:
@@ -121,8 +135,22 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: "Populate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
+      - name: "Generate frontend & version information"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: node:lts
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            cd /work
+            make frontend
+            ls -alt /work
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       # Login against a Docker registry except on PR
@@ -144,8 +172,6 @@ jobs:
             type=ref,enable=true,event=branch,suffix=-omnibus
             type=ref,enable=true,event=tag,suffix=-omnibus
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-      - name: "Generate frontend version information"
-        run: "cd webapp/frontend && ./git.version.sh"
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
@@ -157,5 +183,5 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
+#          cache-from: type=gha
-          cache-to: type=gha,mode=max
+#          cache-to: type=gha,mode=max

CONTRIBUTING.md

@@ -169,3 +169,19 @@ docker run -it --rm -p 8080:8080 \
 ghcr.io/analogj/scrutiny:master-omnibus
 /opt/scrutiny/bin/scrutiny-collector-metrics run
 ```
+
+
+# Running Tests
+
+```bash
+docker run -p 8086:8086 -d --rm \
+-e DOCKER_INFLUXDB_INIT_MODE=setup \
+-e DOCKER_INFLUXDB_INIT_USERNAME=admin \
+-e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
+-e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
+-e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
+-e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \
+influxdb:2.2
+go test ./...
+
+```

Makefile

@@ -9,6 +9,7 @@ BINARY=\
   linux/arm-7   \
   linux/arm64   \
 
+.ONESHELL: # Applies to every targets in the file! .ONESHELL instructs make to invoke a single instance of the shell and provide it with the entire recipe, regardless of how many lines it contains.
 .PHONY: all $(BINARY)
 all: $(BINARY) windows/amd64
 
@@ -38,5 +39,28 @@ windows/amd64:
 	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
 	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
 
+
+docker-collector:
+	@echo "building collector docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.collector -t analogj/scrutiny-dev:collector .
+
+docker-web:
+	@echo "building web docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.web -t analogj/scrutiny-dev:web .
+
+docker-omnibus:
+	@echo "building omnibus docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile -t analogj/scrutiny-dev:omnibus .
+
+# reduce logging, disable angular-cli analytics for ci environment
+frontend: export NPM_CONFIG_LOGLEVEL = warn
+frontend: export NG_CLI_ANALYTICS = false
+frontend:
+	cd webapp/frontend
+	npm install -g @angular/cli@9.1.4
+	mkdir -p $(CURDIR)/dist
+	npm install
+	npm run build:prod -- --output-path=$(CURDIR)/dist
+
 # clean:
 # 	rm scrutiny-collector-metrics-* scrutiny-web-*

README.md

@@ -17,8 +17,6 @@
 WebUI for smartd S.M.A.R.T monitoring
 
 > NOTE: Scrutiny is a Work-in-Progress and still has some rough edges.
->
-> WARNING: Once the [InfluxDB](https://github.com/AnalogJ/scrutiny/tree/influxdb) branch is merged, Scrutiny will use both sqlite and InfluxDB for data storage. Unfortunately, this may not be backwards compatible with the database structures in the master (sqlite only) branch. 
 
 [![](docs/dashboard.png)](https://imgur.com/a/5k8qMzS)
 
@@ -60,11 +58,12 @@ Scrutiny uses `smartctl --scan` to detect devices/drives.
 - All RAID controllers supported by `smartctl` are automatically supported by Scrutiny.
     - While some RAID controllers support passing through the underlying SMART data to `smartctl` others do not.
     - In some cases `--scan` does not correctly detect the device type, returning [incomplete SMART data](https://github.com/AnalogJ/scrutiny/issues/45).
-    Scrutiny will eventually support overriding detected device type via the config file.
+    Scrutiny supports overriding detected device type via the config file: see [example.collector.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.collector.yaml)
 - If you use docker, you **must** pass though the RAID virtual disk to the container using `--device` (see below)
     - This device may be in `/dev/*` or `/dev/bus/*`.
     - If you're unsure, run `smartctl --scan` on your host, and pass all listed devices to the container.
 
+See [docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](./docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md) for help
 
 ## Docker
 

collector/pkg/collector/metrics.go

@@ -116,12 +116,13 @@ func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceT
 	}
 	mc.logger.Infof("Collecting smartctl results for %s\n", deviceName)
 
-	args := []string{"-x", "-j"}
+	fullDeviceName := fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName)
+	args := strings.Split(mc.config.GetCommandMetricsSmartArgs(fullDeviceName), " ")
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
 	if len(deviceType) > 0 && deviceType != "scsi" && deviceType != "ata" {
-		args = append(args, "-d", deviceType)
+		args = append(args, "--device", deviceType)
 	}
-	args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName))
+	args = append(args, fullDeviceName)
 
 	result, err := mc.shell.Command(mc.logger, "smartctl", args, "", os.Environ())
 	resultBytes := []byte(result)

collector/pkg/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"fmt"
 	"github.com/analogj/go-util/utils"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/collector/pkg/models"
@@ -8,6 +9,8 @@ import (
 	"github.com/spf13/viper"
 	"log"
 	"os"
+	"sort"
+	"strings"
 )
 
 // When initializing this class the following methods must be called:
@@ -16,6 +19,8 @@ import (
 // This is done automatically when created via the Factory.
 type configuration struct {
 	*viper.Viper
+
+	deviceOverrides []models.ScanOverride
 }
 
 //Viper uses the following precedence order. Each item takes precedence over the item below it:
@@ -38,6 +43,10 @@ func (c *configuration) Init() error {
 
 	c.SetDefault("api.endpoint", "http://localhost:8080")
 
+	c.SetDefault("commands.metrics_scan_args", "--scan --json")
+	c.SetDefault("commands.metrics_info_args", "--info --json")
+	c.SetDefault("commands.metrics_smart_args", "--xall --json")
+
 	//c.SetDefault("collect.short.command", "-a -o on -S on")
 
 	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
@@ -90,16 +99,89 @@ func (c *configuration) ValidateConfig() error {
 	// check that device prefix matches OS
 	// check that schema of config file is valid
 
-	return nil
+	// check that the collector commands are valid
+	commandArgStrings := map[string]string{
+		"commands.metrics_scan_args":  c.GetString("commands.metrics_scan_args"),
+		"commands.metrics_info_args":  c.GetString("commands.metrics_info_args"),
+		"commands.metrics_smart_args": c.GetString("commands.metrics_smart_args"),
+	}
+
+	errorStrings := []string{}
+	for configKey, commandArgString := range commandArgStrings {
+		args := strings.Split(commandArgString, " ")
+		//ensure that the args string contains `--json` or `-j` flag
+		containsJsonFlag := false
+		containsDeviceFlag := false
+		for _, flag := range args {
+			if strings.HasPrefix(flag, "--json") || strings.HasPrefix(flag, "-j") {
+				containsJsonFlag = true
+			}
+			if strings.HasPrefix(flag, "--device") || strings.HasPrefix(flag, "-d") {
+				containsDeviceFlag = true
+			}
+		}
+
+		if !containsJsonFlag {
+			errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' is missing '--json' flag", configKey))
+		}
+
+		if containsDeviceFlag {
+			errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' must not contain '--device' or '-d' flag", configKey))
+		}
+	}
+	//sort(errorStrings)
+	sort.Strings(errorStrings)
+
+	if len(errorStrings) == 0 {
+		return nil
+	} else {
+		return errors.ConfigValidationError(strings.Join(errorStrings, ", "))
+	}
 }
 
-func (c *configuration) GetScanOverrides() []models.ScanOverride {
+func (c *configuration) GetDeviceOverrides() []models.ScanOverride {
 	// we have to support 2 types of device types.
 	// - simple device type (device_type: 'sat')
 	// and list of device types (type: \n- 3ware,0 \n- 3ware,1 \n- 3ware,2)
 	// GetString will return "" if this is a list of device types.
 
-	overrides := []models.ScanOverride{}
+	if c.deviceOverrides == nil {
-	c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
+		overrides := []models.ScanOverride{}
-	return overrides
+		c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
+		c.deviceOverrides = overrides
+	}
+
+	return c.deviceOverrides
+}
+
+func (c *configuration) GetCommandMetricsInfoArgs(deviceName string) string {
+	overrides := c.GetDeviceOverrides()
+
+	for _, deviceOverrides := range overrides {
+		if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) {
+			//found matching device
+			if len(deviceOverrides.Commands.MetricsInfoArgs) > 0 {
+				return deviceOverrides.Commands.MetricsInfoArgs
+			} else {
+				return c.GetString("commands.metrics_info_args")
+			}
+		}
+	}
+	return c.GetString("commands.metrics_info_args")
+}
+
+func (c *configuration) GetCommandMetricsSmartArgs(deviceName string) string {
+	overrides := c.GetDeviceOverrides()
+
+	for _, deviceOverrides := range overrides {
+		if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) {
+			//found matching device
+			if len(deviceOverrides.Commands.MetricsSmartArgs) > 0 {
+				return deviceOverrides.Commands.MetricsSmartArgs
+			} else {
+				return c.GetString("commands.metrics_smart_args")
+			}
+		}
+	}
+	return c.GetString("commands.metrics_smart_args")
 }

collector/pkg/config/config_test.go

@@ -30,7 +30,7 @@ func TestConfiguration_GetScanOverrides_Simple(t *testing.T) {
 	//test
 	err := testConfig.ReadConfig(path.Join("testdata", "simple_device.yaml"))
 	require.NoError(t, err, "should correctly load simple device config")
-	scanOverrides := testConfig.GetScanOverrides()
+	scanOverrides := testConfig.GetDeviceOverrides()
 
 	//assert
 	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat"}, Ignore: false}}, scanOverrides)
@@ -45,7 +45,7 @@ func TestConfiguration_GetScanOverrides_Ignore(t *testing.T) {
 	//test
 	err := testConfig.ReadConfig(path.Join("testdata", "ignore_device.yaml"))
 	require.NoError(t, err, "should correctly load ignore device config")
-	scanOverrides := testConfig.GetScanOverrides()
+	scanOverrides := testConfig.GetDeviceOverrides()
 
 	//assert
 	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}, scanOverrides)
@@ -60,7 +60,7 @@ func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
 	//test
 	err := testConfig.ReadConfig(path.Join("testdata", "raid_device.yaml"))
 	require.NoError(t, err, "should correctly load ignore device config")
-	scanOverrides := testConfig.GetScanOverrides()
+	scanOverrides := testConfig.GetDeviceOverrides()
 
 	//assert
 	require.Equal(t, []models.ScanOverride{
@@ -75,3 +75,53 @@ func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
 			Ignore:     false,
 		}}, scanOverrides)
 }
+
+func TestConfiguration_InvalidCommands_MissingJson(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_missing_json.yaml"))
+	require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_scan_args' is missing '--json' flag"`, "should throw an error because json flag is missing")
+}
+
+func TestConfiguration_InvalidCommands_IncludesDevice(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_includes_device.yaml"))
+	require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_info_args' must not contain '--device' or '-d' flag, configuration key 'commands.metrics_smart_args' must not contain '--device' or '-d' flag"`, "should throw an error because device flags detected")
+}
+
+func TestConfiguration_OverrideCommands(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "override_commands.yaml"))
+	require.NoError(t, err, "should not throw an error")
+	require.Equal(t, "--xall --json -T permissive", testConfig.GetString("commands.metrics_smart_args"))
+}
+
+func TestConfiguration_OverrideDeviceCommands_MetricsInfoArgs(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "override_device_commands.yaml"))
+	require.NoError(t, err, "should correctly override device command")
+
+	//assert
+	require.Equal(t, "--info --json -T permissive", testConfig.GetCommandMetricsInfoArgs("/dev/sda"))
+	require.Equal(t, "--info --json", testConfig.GetCommandMetricsInfoArgs("/dev/sdb"))
+	//require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Commands: {MetricsInfoArgs: "--info --json -T "}}}, scanOverrides)
+}

collector/pkg/config/interface.go

@@ -22,5 +22,7 @@ type Interface interface {
 	GetStringSlice(key string) []string
 	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
 
-	GetScanOverrides() []models.ScanOverride
+	GetDeviceOverrides() []models.ScanOverride
+	GetCommandMetricsInfoArgs(deviceName string) string
+	GetCommandMetricsSmartArgs(deviceName string) string
 }

collector/pkg/config/mock/mock_config.go

@@ -5,88 +5,37 @@
 package mock_config
 
 import (
+	reflect "reflect"
+
 	models "github.com/analogj/scrutiny/collector/pkg/models"
 	gomock "github.com/golang/mock/gomock"
 	viper "github.com/spf13/viper"
-	reflect "reflect"
 )
 
-// MockInterface is a mock of Interface interface
+// MockInterface is a mock of Interface interface.
 type MockInterface struct {
 	ctrl     *gomock.Controller
 	recorder *MockInterfaceMockRecorder
 }
 
-// MockInterfaceMockRecorder is the mock recorder for MockInterface
+// MockInterfaceMockRecorder is the mock recorder for MockInterface.
 type MockInterfaceMockRecorder struct {
 	mock *MockInterface
 }
 
-// NewMockInterface creates a new mock instance
+// NewMockInterface creates a new mock instance.
 func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
 	mock := &MockInterface{ctrl: ctrl}
 	mock.recorder = &MockInterfaceMockRecorder{mock}
 	return mock
 }
 
-// EXPECT returns an object that allows the caller to indicate expected use
+// EXPECT returns an object that allows the caller to indicate expected use.
 func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
-// Init mocks base method
+// AllSettings mocks base method.
-func (m *MockInterface) Init() error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Init")
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Init indicates an expected call of Init
-func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
-}
-
-// ReadConfig mocks base method
-func (m *MockInterface) ReadConfig(configFilePath string) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// ReadConfig indicates an expected call of ReadConfig
-func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
-}
-
-// Set mocks base method
-func (m *MockInterface) Set(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Set", key, value)
-}
-
-// Set indicates an expected call of Set
-func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
-}
-
-// SetDefault mocks base method
-func (m *MockInterface) SetDefault(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "SetDefault", key, value)
-}
-
-// SetDefault indicates an expected call of SetDefault
-func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
-}
-
-// AllSettings mocks base method
 func (m *MockInterface) AllSettings() map[string]interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "AllSettings")
@@ -94,27 +43,13 @@ func (m *MockInterface) AllSettings() map[string]interface{} {
 	return ret0
 }
 
-// AllSettings indicates an expected call of AllSettings
+// AllSettings indicates an expected call of AllSettings.
 func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings))
 }
 
-// IsSet mocks base method
+// Get mocks base method.
-func (m *MockInterface) IsSet(key string) bool {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "IsSet", key)
-	ret0, _ := ret[0].(bool)
-	return ret0
-}
-
-// IsSet indicates an expected call of IsSet
-func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
-}
-
-// Get mocks base method
 func (m *MockInterface) Get(key string) interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Get", key)
@@ -122,13 +57,13 @@ func (m *MockInterface) Get(key string) interface{} {
 	return ret0
 }
 
-// Get indicates an expected call of Get
+// Get indicates an expected call of Get.
 func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key)
 }
 
-// GetBool mocks base method
+// GetBool mocks base method.
 func (m *MockInterface) GetBool(key string) bool {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetBool", key)
@@ -136,13 +71,55 @@ func (m *MockInterface) GetBool(key string) bool {
 	return ret0
 }
 
-// GetBool indicates an expected call of GetBool
+// GetBool indicates an expected call of GetBool.
 func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key)
 }
 
-// GetInt mocks base method
+// GetCommandMetricsInfoArgs mocks base method.
+func (m *MockInterface) GetCommandMetricsInfoArgs(deviceName string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCommandMetricsInfoArgs", deviceName)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetCommandMetricsInfoArgs indicates an expected call of GetCommandMetricsInfoArgs.
+func (mr *MockInterfaceMockRecorder) GetCommandMetricsInfoArgs(deviceName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsInfoArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsInfoArgs), deviceName)
+}
+
+// GetCommandMetricsSmartArgs mocks base method.
+func (m *MockInterface) GetCommandMetricsSmartArgs(deviceName string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCommandMetricsSmartArgs", deviceName)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetCommandMetricsSmartArgs indicates an expected call of GetCommandMetricsSmartArgs.
+func (mr *MockInterfaceMockRecorder) GetCommandMetricsSmartArgs(deviceName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsSmartArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsSmartArgs), deviceName)
+}
+
+// GetDeviceOverrides mocks base method.
+func (m *MockInterface) GetDeviceOverrides() []models.ScanOverride {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDeviceOverrides")
+	ret0, _ := ret[0].([]models.ScanOverride)
+	return ret0
+}
+
+// GetDeviceOverrides indicates an expected call of GetDeviceOverrides.
+func (mr *MockInterfaceMockRecorder) GetDeviceOverrides() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceOverrides", reflect.TypeOf((*MockInterface)(nil).GetDeviceOverrides))
+}
+
+// GetInt mocks base method.
 func (m *MockInterface) GetInt(key string) int {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetInt", key)
@@ -150,13 +127,13 @@ func (m *MockInterface) GetInt(key string) int {
 	return ret0
 }
 
-// GetInt indicates an expected call of GetInt
+// GetInt indicates an expected call of GetInt.
 func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
 }
 
-// GetString mocks base method
+// GetString mocks base method.
 func (m *MockInterface) GetString(key string) string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetString", key)
@@ -164,13 +141,13 @@ func (m *MockInterface) GetString(key string) string {
 	return ret0
 }
 
-// GetString indicates an expected call of GetString
+// GetString indicates an expected call of GetString.
 func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key)
 }
 
-// GetStringSlice mocks base method
+// GetStringSlice mocks base method.
 func (m *MockInterface) GetStringSlice(key string) []string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetStringSlice", key)
@@ -178,13 +155,79 @@ func (m *MockInterface) GetStringSlice(key string) []string {
 	return ret0
 }
 
-// GetStringSlice indicates an expected call of GetStringSlice
+// GetStringSlice indicates an expected call of GetStringSlice.
 func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key)
 }
 
-// UnmarshalKey mocks base method
+// Init mocks base method.
+func (m *MockInterface) Init() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Init")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Init indicates an expected call of Init.
+func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
+}
+
+// IsSet mocks base method.
+func (m *MockInterface) IsSet(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsSet", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsSet indicates an expected call of IsSet.
+func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
+}
+
+// ReadConfig mocks base method.
+func (m *MockInterface) ReadConfig(configFilePath string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReadConfig indicates an expected call of ReadConfig.
+func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
+}
+
+// Set mocks base method.
+func (m *MockInterface) Set(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Set", key, value)
+}
+
+// Set indicates an expected call of Set.
+func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
+}
+
+// SetDefault mocks base method.
+func (m *MockInterface) SetDefault(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetDefault", key, value)
+}
+
+// SetDefault indicates an expected call of SetDefault.
+func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
+}
+
+// UnmarshalKey mocks base method.
 func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{key, rawVal}
@@ -196,23 +239,9 @@ func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts
 	return ret0
 }
 
-// UnmarshalKey indicates an expected call of UnmarshalKey
+// UnmarshalKey indicates an expected call of UnmarshalKey.
 func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{key, rawVal}, decoderOpts...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...)
 }
-
-// GetScanOverrides mocks base method
-func (m *MockInterface) GetScanOverrides() []models.ScanOverride {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetScanOverrides")
-	ret0, _ := ret[0].([]models.ScanOverride)
-	return ret0
-}
-
-// GetScanOverrides indicates an expected call of GetScanOverrides
-func (mr *MockInterfaceMockRecorder) GetScanOverrides() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScanOverrides", reflect.TypeOf((*MockInterface)(nil).GetScanOverrides))
-}

collector/pkg/config/testdata/invalid_commands_includes_device.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan --json' # used to detect devices
+  metrics_info_args: '--info --json --device=sat' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json -d sat' # used to retrieve smart data for each device.

collector/pkg/config/testdata/invalid_commands_missing_json.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan' # used to detect devices
+  metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json' # used to retrieve smart data for each device.

collector/pkg/config/testdata/override_commands.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan --json' # used to detect devices
+  metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device.

collector/pkg/config/testdata/override_device_commands.yaml

@@ -0,0 +1,5 @@
+version: 1
+devices:
+  - device: /dev/sda
+    commands:
+      metrics_info_args: "--info --json -T permissive"

collector/pkg/detect/detect.go

@@ -28,7 +28,8 @@ type Detect struct {
 // models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
 func (d *Detect) SmartctlScan() ([]models.Device, error) {
 	//we use smartctl to detect all the drives available.
-	detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
+	args := strings.Split(d.Config.GetString("commands.metrics_scan_args"), " ")
+	detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ())
 	if err != nil {
 		d.Logger.Errorf("Error scanning for devices: %v", err)
 		return nil, err
@@ -51,13 +52,13 @@ func (d *Detect) SmartctlScan() ([]models.Device, error) {
 // - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
 // - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
 func (d *Detect) SmartCtlInfo(device *models.Device) error {
-
+	fullDeviceName := fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName)
-	args := []string{"--info", "-j"}
+	args := strings.Split(d.Config.GetCommandMetricsInfoArgs(fullDeviceName), " ")
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
 	if len(device.DeviceType) > 0 && device.DeviceType != "scsi" && device.DeviceType != "ata" {
-		args = append(args, "-d", device.DeviceType)
+		args = append(args, "--device", device.DeviceType)
 	}
-	args = append(args, fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName))
+	args = append(args, fullDeviceName)
 
 	availableDeviceInfoJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ())
 	if err != nil {
@@ -138,7 +139,7 @@ func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []mod
 
 	//now tha we've "grouped" all the devices, lets override any groups specified in the config file.
 
-	for _, overrideDevice := range d.Config.GetScanOverrides() {
+	for _, overrideDevice := range d.Config.GetDeviceOverrides() {
 		overrideDeviceFile := strings.ToLower(overrideDevice.Device)
 
 		if overrideDevice.Ignore {

collector/pkg/detect/detect_test.go

@@ -18,7 +18,8 @@ func TestDetect_SmartctlScan(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
 	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_simple.json")
@@ -45,7 +46,8 @@ func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
 	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_megaraid.json")
@@ -75,7 +77,8 @@ func TestDetect_SmartctlScan_Nvme(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
 	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_nvme.json")
@@ -104,7 +107,9 @@ func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -134,7 +139,9 @@ func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -163,7 +170,8 @@ func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{
 		{
 			Device:     "/dev/bus/0",
 			DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
@@ -202,7 +210,8 @@ func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{

collector/pkg/models/scan_override.go

@@ -4,4 +4,8 @@ type ScanOverride struct {
 	Device     string   `mapstructure:"device"`
 	DeviceType []string `mapstructure:"type"`
 	Ignore     bool     `mapstructure:"ignore"`
+	Commands   struct {
+		MetricsInfoArgs  string `mapstructure:"metrics_info_args"`
+		MetricsSmartArgs string `mapstructure:"metrics_smart_args"`
+	} `mapstructure:"commands"`
 }

docker/Dockerfile

@@ -11,39 +11,25 @@ RUN go mod vendor && \
     go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
 
 ########
-FROM node:lts-slim as frontendbuild
+FROM ubuntu:latest as runtime
-
-#reduce logging, disable angular-cli analytics for ci environment
-ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
-
-WORKDIR /opt/scrutiny/src
-COPY webapp/frontend /opt/scrutiny/src
-
-RUN npm install -g @angular/cli@9.1.4 && \
-    mkdir -p /scrutiny/dist && \
-    npm install && \
-    npm run build:prod -- --output-path=/opt/scrutiny/dist
-
-
-########
-FROM ubuntu:bionic as runtime
 ARG TARGETARCH
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 ENV INFLUXD_CONFIG_PATH=/opt/scrutiny/influxdb
 
-RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates curl tzdata \
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates curl tzdata \
     && update-ca-certificates \
     &&  case ${TARGETARCH} in \
             "amd64")  S6_ARCH=amd64  ;; \
             "arm64")  S6_ARCH=aarch64  ;; \
         esac \
     && curl https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-${S6_ARCH}.tar.gz -L -s --output /tmp/s6-overlay-${S6_ARCH}.tar.gz \
-    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C /
+    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C / --exclude="./bin" \
-
+    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C /usr ./bin \
-ADD https://dl.influxdata.com/influxdb/releases/influxdb2-2.2.0-${TARGETARCH}.deb /tmp/
+    && rm -rf /tmp/s6-overlay-${S6_ARCH}.tar.gz \
-RUN dpkg -i /tmp/influxdb2-2.2.0-${TARGETARCH}.deb && rm -rf /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
+    && curl -L https://dl.influxdata.com/influxdb/releases/influxdb2-2.2.0-${TARGETARCH}.deb --output /tmp/influxdb2-2.2.0-${TARGETARCH}.deb \
+    && dpkg -i --force-all /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
 
 COPY /rootfs /
 
@@ -51,7 +37,7 @@ COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /opt/scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
-COPY --from=frontendbuild /opt/scrutiny/dist /opt/scrutiny/web
+COPY dist /opt/scrutiny/web
 RUN chmod +x /opt/scrutiny/bin/scrutiny && \
     chmod +x /opt/scrutiny/bin/scrutiny-collector-selftest && \
     chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \

docker/Dockerfile.collector

@@ -10,11 +10,11 @@ RUN go mod vendor && \
     go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
 
 ########
-FROM ubuntu:bionic as runtime
+FROM ubuntu:latest as runtime
 WORKDIR /scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates tzdata && update-ca-certificates
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && update-ca-certificates
 
 COPY /docker/entrypoint-collector.sh /entrypoint-collector.sh
 COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny

docker/Dockerfile.web

@@ -9,22 +9,7 @@ RUN go mod vendor && \
     go build -ldflags '-w -extldflags "-static"' -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go
 
 ########
-FROM node:lts-slim as frontendbuild
+FROM ubuntu:latest as runtime
-
-#reduce logging, disable angular-cli analytics for ci environment
-ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
-
-WORKDIR /opt/scrutiny/src
-COPY webapp/frontend /opt/scrutiny/src
-
-RUN npm install -g @angular/cli@9.1.4 && \
-    mkdir -p /opt/scrutiny/dist && \
-    npm install && \
-    npm run build:prod -- --output-path=/opt/scrutiny/dist
-
-
-########
-FROM ubuntu:bionic as runtime
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
@@ -32,7 +17,7 @@ ENV PATH="/opt/scrutiny/bin:${PATH}"
 RUN apt-get update && apt-get install -y ca-certificates curl tzdata && update-ca-certificates
 
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
-COPY --from=frontendbuild /opt/scrutiny/dist /opt/scrutiny/web
+COPY dist /opt/scrutiny/web
 RUN chmod +x /opt/scrutiny/bin/scrutiny && \
     mkdir -p /opt/scrutiny/web && \
     mkdir -p /opt/scrutiny/config && \

docs/INSTALL_HUB_SPOKE.md

@@ -1 +1 @@
-> See [docker/example.hubspoke.docker-compose.yml](./docker/example.hubspoke.docker-compose.yml) for a docker-compose file.
+> See [docker/example.hubspoke.docker-compose.yml](https://github.com/AnalogJ/scrutiny/blob/master/docker/example.hubspoke.docker-compose.yml) for a docker-compose file.

docs/INSTALL_MANUAL.md

@@ -57,7 +57,7 @@ web:
   # and store the information in the config file. If you 're re-using an existing influxdb installation, you'll need to provide
   # the `token`
   influxdb:
-    host: 0.0.0.0
+    host: localhost
     port: 8086
 #    token: 'my-token'
 #    org: 'my-org'
@@ -83,9 +83,11 @@ Now that we have downloaded the required files, let's prepare the filesystem.
 chmod +x /opt/scrutiny/bin/scrutiny-web-linux-amd64
 
 # Next, lets extract the frontend files.
+# NOTE: after extraction, there **should not** be a `dist` subdirectory in `/opt/scrutiny/web` directory.
 cd /opt/scrutiny/web
 tar xvzf scrutiny-web-frontend.tar.gz --strip-components 1 -C .
 
+
 # Cleanup
 rm -rf scrutiny-web-frontend.tar.gz
 ```
@@ -113,7 +115,8 @@ Unlike the webapp, the collector does have some dependencies:
 Unfortunately the version of `smartmontools` (which contains `smartctl`) available in some of the base OS repositories is ancient.
 So you'll need to install the v7+ version using one of the following commands:
 
-- **Ubuntu:** `apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1`
+- **Ubuntu (22.04/Jammy/LTS):** `apt-get install -y smartmontools`
+- **Ubuntu (18.04/Bionic):** `apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1`
 - **Centos8:**
     - `dnf install https://extras.getpagespeed.com/release-el8-latest.rpm`
     - `dnf install smartmontools`

docs/INSTALL_PFSENSE.md

@@ -0,0 +1,72 @@
+# pfsense Install
+
+This bascially follows the [Manual collector instructions](https://github.com/AnalogJ/scrutiny/blob/master/docs/INSTALL_MANUAL.md#collector) and assumes you are running a hub and spoke deployment and already have the web app setup.
+
+
+### Dependencies
+
+SSH into pfsense, hit `8` for the shell and install the required dependencies.
+
+```
+pkg install smartmontools
+```
+
+Ensure smartmontools is v7+. This won't be a problem in pfsense 2.6.0+
+
+
+### Directory Structure
+
+Now let's create a directory structure to contain the Scrutiny collector binary.
+
+```
+mkdir -p /opt/scrutiny/bin
+```
+
+
+### Download Files
+
+Next, we'll download the Scrutiny collector binary from the [latest Github release](https://github.com/analogj/scrutiny/releases).
+
+> NOTE: Ensure you have the latest version in the below command
+
+```
+fetch -o /opt/scrutiny/bin https://github.com/AnalogJ/scrutiny/releases/download/vX.X.X/scrutiny-collector-metrics-freebsd-amd64
+```
+
+
+### Prepare Scrutiny
+
+Now that we have downloaded the required files, let's prepare the filesystem.
+
+```
+chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64
+```
+
+
+### Start Scrutiny Collector, Populate Webapp
+
+Next, we will manually trigger the collector, to populate the Scrutiny dashboard:
+
+> NOTE: if you need to pass a config file to the scrutiny collector, you can provide it using the `--config` flag.
+
+```
+/opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64 run --api-endpoint "http://localhost:8080"
+```
+> NOTE: change the IP address to that of your web app
+
+### Schedule Collector with Cron
+
+Finally you need to schedule the collector to run periodically.
+
+Login to the pfsense webGUI and head to `Services/Cron` add an entry with the following details:
+
+```
+Minute: */15
+Hour: *
+Day of the Month: *
+Month of the Year: *
+Day of the Week: *
+User: root
+Command: /opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64 run --api-endpoint "http://localhost:8080" >/dev/null 2>&1
+```
+> NOTE: `>/dev/null 2>&1` is used to stop cron confirmation emails being sent.

docs/SUPPORTED_NAS_OS.md

@@ -4,11 +4,13 @@ These are the officially supported NAS OS's (with documentation and setup guides
 Once a guide is created (in `docs/guides/`) it will be linked here.
 
 - [ ] freenas/truenas
-- [x] [unraid](https://github.com/AnalogJ/scrutiny/blob/master/docs/INSTALL_UNRAID.md)
+- [x] [unraid](./INSTALL_UNRAID.md)
 - [ ] ESXI
 - [ ] Proxmox
 - [ ] Synology
 - [ ] OMV
 - [ ] Amahi
 - [ ] Running in a LXC container
+- [x] [PFSense](./INSTALL_UNRAID.md)
+- [ ] QNAP
 

docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md

@@ -52,6 +52,8 @@ If the output is the same, your devices will be processed by Scrutiny.
 In some cases `--scan` does not correctly detect the device type, returning [incomplete SMART data](https://github.com/AnalogJ/scrutiny/issues/45).
 Scrutiny will supports overriding the detected device type via the config file.
 
+[example.collector.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.collector.yaml)
+
 ### RAID Controllers (Megaraid/3ware/HBA/Adaptec/HPE/etc)
 Smartctl has support for a large number of [RAID controllers](https://www.smartmontools.org/wiki/Supported_RAID-Controllers), however this 
 support is not automatic, and may require some additional device type hinting. You can provide this information to the Scrutiny collector
@@ -111,12 +113,60 @@ instead of the block device (`/dev/nvme0n1`). See [#209](https://github.com/Anal
 
 ### ATA
 
-### Standby/Sleeping Disks
+### Exit Codes
+
+If you see an error message similar to `smartctl returned an error code (2) while processing /dev/sda`, this means that
+`smartctl` (not Scrutiny) exited with an error code. Scrutiny will attempt to print a helpful error message to help you debug,
+but you can look at the table (and associated links) below to debug `smartctl`. 
+
+> smartctl Return Values
+> The return values of smartctl are defined by a bitmask. If all is well with the disk, the return value (exit status) of 
+> smartctl is 0 (all bits turned off). If a problem occurs, or an error, potential error, or fault is detected, then 
+> a non-zero status is returned. In this case, the eight different bits in the return value have the following meanings 
+> for ATA disks; some of these values may also be returned for SCSI disks.
+> 
+> source: http://www.linuxguide.it/command_line/linux-manpage/do.php?file=smartctl#sect7
+
+
+| Exit Code (Isolated) | Binary | Problem Message |
+| --- | --- | --- |
+| 1 | Bit 0 | Command line did not parse. |
+| 2 | Bit 1 | Device open failed, or device did not return an IDENTIFY DEVICE structure. |
+| 4 | Bit 2 | Some SMART command to the disk failed, or there was a checksum error in a SMART data structure (see В´-bВ´ option above). |
+| 8 | Bit 3 | SMART status check returned “DISK FAILING". |
+| 16 | Bit 4 | We found prefail Attributes <= threshold. |
+| 32 | Bit 5 | SMART status check returned “DISK OK” but we found that some (usage or prefail) Attributes have been <= threshold at some time in the past. |
+| 64 | Bit 6 | The device error log contains records of errors. |
+| 128 | Bit 7 | The device self-test log contains records of errors. |
+
+#### Standby/Sleeping Disks
+
+Disks in Standby/Sleep can also cause `smartctl` to exit abnormally, usually with `exit code: 2`. 
+
 - https://github.com/AnalogJ/scrutiny/issues/221
 - https://github.com/AnalogJ/scrutiny/issues/157
 
 ### Volume Mount All Devices (`/dev`) - Privileged
 
+> WARNING: This is an insecure/dangerous workaround. Running Scrutiny (or any Docker image) with `--privileged` is equivalent to running it with root access. 
+
+If you have exhausted all other mechanisms to get your disks working with `smartctl` running within a container, you can try running the docker image with the following additional flags:
+
+- `--privileged` (instead of `--cap-add`) - this gives the docker container full access to your system. Scrutiny does not require this permission, however it can be helpful for `smartctl`
+- `-v /dev:/dev:ro` (instead of `--device`) - this mounts the `/dev` folder (containing all your device files) into the container, allowing `smartctl` to see your disks, exactly as if it were running on your host directly. 
+
+With this workaround your `docker run` command would look similar to the following:
+
+```bash
+docker run -it --rm -p 8080:8080 -p 8086:8086 \
+  -v `pwd`/scrutiny:/opt/scrutiny/config \
+  -v `pwd`/influxdb2:/opt/scrutiny/influxdb \
+  -v /run/udev:/run/udev:ro \
+  --privileged \
+  -v /dev:/dev \
+  --name scrutiny \
+  ghcr.io/analogj/scrutiny:master-omnibus
+```
 
 ## Scrutiny detects Failure but SMART Passed?
 
@@ -138,3 +188,17 @@ Thankfully the collector has a special `--host-id` flag (or `COLLECTOR_HOST_ID`
 
 See the [docs/INSTALL_HUB_SPOKE.md](/docs/INSTALL_HUB_SPOKE.md) guide for more information. 
 
+## Collector DEBUG mode
+
+You can use environmental variables to enable debug logging and/or log files for the collector:
+
+```bash
+DEBUG=true
+COLLECTOR_LOG_FILE=/tmp/collector.log
+```
+
+Or if you're not using docker, you can pass CLI arguments to the collector during startup:
+
+```bash
+scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
+```

example.collector.yaml

@@ -53,6 +53,13 @@ devices:
 #      - 3ware,3
 #      - 3ware,4
 #      - 3ware,5
+#
+#  # example to show how to override the smartctl command args (per device), see below for how to override these globally.
+#  - device: /dev/sda
+#    commands:
+#      metrics_info_args: '--info --json -T permissive' # used to determine device unique ID & register device with Scrutiny
+#      metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device.
+
 
 #log:
 #  file: '' #absolute or relative paths allowed, eg. web.log
@@ -64,6 +71,12 @@ devices:
 # if you need to use a custom base path (for a reverse proxy), you can add a suffix to the endpoint.
 #  See docs/TROUBLESHOOTING_REVERSE_PROXY.md for more info,
 
+# example to show how to override the smartctl command args globally
+#commands:
+#  metrics_scan_args: '--scan --json' # used to detect devices
+#  metrics_info_args: '--info --json' # used to determine device unique ID & register device with Scrutiny
+#  metrics_smart_args: '--xall --json' # used to retrieve smart data for each device.
+
 
 ########################################################################################################################
 # FEATURES COMING SOON

webapp/backend/pkg/constants.go

@@ -4,25 +4,34 @@ const DeviceProtocolAta = "ATA"
 const DeviceProtocolScsi = "SCSI"
 const DeviceProtocolNvme = "NVMe"
 
-const SmartAttributeStatusPassed = 0
+type AttributeStatus uint8
-const SmartAttributeStatusFailed = 1
-const SmartAttributeStatusWarning = 2
-
-const SmartWhenFailedFailingNow = "FAILING_NOW"
-const SmartWhenFailedInThePast = "IN_THE_PAST"
-
-//const SmartStatusPassed = "passed"
-//const SmartStatusFailed = "failed"
-
-type DeviceStatus int
 
 const (
-	DeviceStatusPassed         DeviceStatus = 0
+	// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
-	DeviceStatusFailedSmart    DeviceStatus = iota
+	AttributeStatusPassed          AttributeStatus = 0
-	DeviceStatusFailedScrutiny DeviceStatus = iota
+	AttributeStatusFailedSmart     AttributeStatus = 1
+	AttributeStatusWarningScrutiny AttributeStatus = 2
+	AttributeStatusFailedScrutiny  AttributeStatus = 4
 )
 
-func Set(b, flag DeviceStatus) DeviceStatus    { return b | flag }
+const AttributeWhenFailedFailingNow = "FAILING_NOW"
-func Clear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
+const AttributeWhenFailedInThePast = "IN_THE_PAST"
-func Toggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
+
-func Has(b, flag DeviceStatus) bool            { return b&flag != 0 }
+func AttributeStatusSet(b, flag AttributeStatus) AttributeStatus    { return b | flag }
+func AttributeStatusClear(b, flag AttributeStatus) AttributeStatus  { return b &^ flag }
+func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ flag }
+func AttributeStatusHas(b, flag AttributeStatus) bool               { return b&flag != 0 }
+
+type DeviceStatus uint8
+
+const (
+	// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
+	DeviceStatusPassed         DeviceStatus = 0
+	DeviceStatusFailedSmart    DeviceStatus = 1
+	DeviceStatusFailedScrutiny DeviceStatus = 2
+)
+
+func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus    { return b | flag }
+func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
+func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
+func DeviceStatusHas(b, flag DeviceStatus) bool            { return b&flag != 0 }

webapp/backend/pkg/database/scrutiny_repository_device.go

@@ -58,7 +58,7 @@ func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string
 		return device, fmt.Errorf("Could not get device from DB: %v", err)
 	}
 
-	device.DeviceStatus = pkg.Set(device.DeviceStatus, status)
+	device.DeviceStatus = pkg.DeviceStatusSet(device.DeviceStatus, status)
 	return device, sr.gormClient.Model(&device).Updates(device).Error
 }
 

webapp/backend/pkg/models/device.go

@@ -21,9 +21,9 @@ type Device struct {
 	WWN string `json:"wwn" gorm:"primary_key"`
 
 	DeviceName     string `json:"device_name"`
-	DeviceUUID	   string `json:"device_uuid"`
+	DeviceUUID     string `json:"device_uuid"`
-	DeviceSerialID	   string `json:"device_serial_id"`
+	DeviceSerialID string `json:"device_serial_id"`
-	DeviceLabel	   string `json:"device_label"`
+	DeviceLabel    string `json:"device_label"`
 
 	Manufacturer   string `json:"manufacturer"`
 	ModelName      string `json:"model_name"`
@@ -166,7 +166,7 @@ func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
 	dv.DeviceProtocol = info.Device.Protocol
 
 	if !info.SmartStatus.Passed {
-		dv.DeviceStatus = pkg.Set(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
+		dv.DeviceStatus = pkg.DeviceStatusSet(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
 	}
 
 	return nil

webapp/backend/pkg/models/measurements/smart.go

@@ -110,7 +110,7 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er
 	sm.PowerCycleCount = info.PowerCycleCount
 	sm.PowerOnHours = info.PowerOnTime.Hours
 	if !info.SmartStatus.Passed {
-		sm.Status = pkg.DeviceStatusFailedSmart
+		sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedSmart)
 	}
 
 	sm.DeviceProtocol = info.Device.Protocol
@@ -148,8 +148,9 @@ func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTa
 		}
 		attrModel.PopulateAttributeStatus()
 		sm.Attributes[strconv.Itoa(collectorAttr.ID)] = &attrModel
-		if attrModel.Status == pkg.SmartAttributeStatusFailed {
+
-			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		if pkg.AttributeStatusHas(attrModel.Status, pkg.AttributeStatusFailedScrutiny) {
+			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
 		}
 	}
 }
@@ -178,8 +179,8 @@ func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.Nv
 
 	//find analyzed attribute status
 	for _, val := range sm.Attributes {
-		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
+		if pkg.AttributeStatusHas(val.GetStatus(), pkg.AttributeStatusFailedScrutiny) {
-			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
 		}
 	}
 }
@@ -204,8 +205,8 @@ func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog
 
 	//find analyzed attribute status
 	for _, val := range sm.Attributes {
-		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
+		if pkg.AttributeStatusHas(val.GetStatus(), pkg.AttributeStatusFailedScrutiny) {
-			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
 		}
 	}
 }

webapp/backend/pkg/models/measurements/smart_ata_attribute.go

@@ -18,13 +18,13 @@ type SmartAtaAttribute struct {
 	WhenFailed  string `json:"when_failed"`
 
 	//Generated data
-	TransformedValue int64   `json:"transformed_value"`
+	TransformedValue int64               `json:"transformed_value"`
-	Status           int64   `json:"status"`
+	Status           pkg.AttributeStatus `json:"status"`
-	StatusReason     string  `json:"status_reason,omitempty"`
+	StatusReason     string              `json:"status_reason,omitempty"`
-	FailureRate      float64 `json:"failure_rate,omitempty"`
+	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
-func (sa *SmartAtaAttribute) GetStatus() int64 {
+func (sa *SmartAtaAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }
 
@@ -43,7 +43,7 @@ func (sa *SmartAtaAttribute) Flatten() map[string]interface{} {
 
 		//Generated Data
 		fmt.Sprintf("attr.%s.transformed_value", idString): sa.TransformedValue,
-		fmt.Sprintf("attr.%s.status", idString):            sa.Status,
+		fmt.Sprintf("attr.%s.status", idString):            int64(sa.Status),
 		fmt.Sprintf("attr.%s.status_reason", idString):     sa.StatusReason,
 		fmt.Sprintf("attr.%s.failure_rate", idString):      sa.FailureRate,
 	}
@@ -77,7 +77,7 @@ func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
 	case "transformed_value":
 		sa.TransformedValue = val.(int64)
 	case "status":
-		sa.Status = val.(int64)
+		sa.Status = pkg.AttributeStatus(val.(int64))
 	case "status_reason":
 		sa.StatusReason = val.(string)
 	case "failure_rate":
@@ -89,16 +89,16 @@ func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
 //populate attribute status, using SMART Thresholds & Observed Metadata
 // Chainable
 func (sa *SmartAtaAttribute) PopulateAttributeStatus() *SmartAtaAttribute {
-	if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedFailingNow {
+	if strings.ToUpper(sa.WhenFailed) == pkg.AttributeWhenFailedFailingNow {
 		//this attribute has previously failed
-		sa.Status = pkg.SmartAttributeStatusFailed
+		sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedSmart)
-		sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
+		sa.StatusReason += "Attribute is failing manufacturer SMART threshold"
 		//if the Smart Status is failed, we should exit early, no need to look at thresholds.
 		return sa
 
-	} else if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedInThePast {
+	} else if strings.ToUpper(sa.WhenFailed) == pkg.AttributeWhenFailedInThePast {
-		sa.Status = pkg.SmartAttributeStatusWarning
+		sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
-		sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
+		sa.StatusReason += "Attribute has previously failed manufacturer SMART threshold"
 	}
 
 	if smartMetadata, ok := thresholds.AtaMetadata[sa.AttributeId]; ok {
@@ -138,16 +138,16 @@ func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttri
 
 			if smartMetadata.Critical {
 				if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = pkg.SmartAttributeStatusFailed
+					sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
-					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
+					sa.StatusReason += "Observed Failure Rate for Critical Attribute is greater than 10%"
 				}
 			} else {
 				if obsThresh.AnnualFailureRate >= 0.20 {
-					sa.Status = pkg.SmartAttributeStatusFailed
+					sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
+					sa.StatusReason += "Observed Failure Rate for Non-Critical Attribute is greater than 20%"
 				} else if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = pkg.SmartAttributeStatusWarning
+					sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
+					sa.StatusReason += "Observed Failure Rate for Non-Critical Attribute is greater than 10%"
 				}
 			}
 
@@ -157,7 +157,7 @@ func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttri
 	}
 	// no bucket found
 	if smartMetadata.Critical {
-		sa.Status = pkg.SmartAttributeStatusWarning
+		sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
 		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
 	}
 

webapp/backend/pkg/models/measurements/smart_attribute.go

@@ -1,7 +1,9 @@
 package measurements
 
+import "github.com/analogj/scrutiny/webapp/backend/pkg"
+
 type SmartAttribute interface {
 	Flatten() (fields map[string]interface{})
 	Inflate(key string, val interface{})
-	GetStatus() int64
+	GetStatus() pkg.AttributeStatus
 }

webapp/backend/pkg/models/measurements/smart_nvme_attribute.go

@@ -12,13 +12,13 @@ type SmartNvmeAttribute struct {
 	Value       int64  `json:"value"`
 	Threshold   int64  `json:"thresh"`
 
-	TransformedValue int64   `json:"transformed_value"`
+	TransformedValue int64               `json:"transformed_value"`
-	Status           int64   `json:"status"`
+	Status           pkg.AttributeStatus `json:"status"`
-	StatusReason     string  `json:"status_reason,omitempty"`
+	StatusReason     string              `json:"status_reason,omitempty"`
-	FailureRate      float64 `json:"failure_rate,omitempty"`
+	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
-func (sa *SmartNvmeAttribute) GetStatus() int64 {
+func (sa *SmartNvmeAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }
 
@@ -30,7 +30,7 @@ func (sa *SmartNvmeAttribute) Flatten() map[string]interface{} {
 
 		//Generated Data
 		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
-		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            int64(sa.Status),
 		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
 		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
 	}
@@ -54,7 +54,7 @@ func (sa *SmartNvmeAttribute) Inflate(key string, val interface{}) {
 	case "transformed_value":
 		sa.TransformedValue = val.(int64)
 	case "status":
-		sa.Status = val.(int64)
+		sa.Status = pkg.AttributeStatus(val.(int64))
 	case "status_reason":
 		sa.StatusReason = val.(string)
 	case "failure_rate":
@@ -72,8 +72,8 @@ func (sa *SmartNvmeAttribute) PopulateAttributeStatus() *SmartNvmeAttribute {
 			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
 			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
 				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = pkg.SmartAttributeStatusFailed
+				sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
-				sa.StatusReason = "Attribute is failing recommended SMART threshold"
+				sa.StatusReason += "Attribute is failing recommended SMART threshold"
 			}
 		}
 	}

webapp/backend/pkg/models/measurements/smart_scsci_attribute.go

@@ -12,13 +12,13 @@ type SmartScsiAttribute struct {
 	Value       int64  `json:"value"`
 	Threshold   int64  `json:"thresh"`
 
-	TransformedValue int64   `json:"transformed_value"`
+	TransformedValue int64               `json:"transformed_value"`
-	Status           int64   `json:"status"`
+	Status           pkg.AttributeStatus `json:"status"`
-	StatusReason     string  `json:"status_reason,omitempty"`
+	StatusReason     string              `json:"status_reason,omitempty"`
-	FailureRate      float64 `json:"failure_rate,omitempty"`
+	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
-func (sa *SmartScsiAttribute) GetStatus() int64 {
+func (sa *SmartScsiAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }
 
@@ -30,7 +30,7 @@ func (sa *SmartScsiAttribute) Flatten() map[string]interface{} {
 
 		//Generated Data
 		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
-		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            int64(sa.Status),
 		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
 		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
 	}
@@ -54,7 +54,7 @@ func (sa *SmartScsiAttribute) Inflate(key string, val interface{}) {
 	case "transformed_value":
 		sa.TransformedValue = val.(int64)
 	case "status":
-		sa.Status = val.(int64)
+		sa.Status = pkg.AttributeStatus(val.(int64))
 	case "status_reason":
 		sa.StatusReason = val.(string)
 	case "failure_rate":
@@ -73,7 +73,7 @@ func (sa *SmartScsiAttribute) PopulateAttributeStatus() *SmartScsiAttribute {
 			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
 			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
 				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = pkg.SmartAttributeStatusFailed
+				sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
 				sa.StatusReason = "Attribute is failing recommended SMART threshold"
 			}
 		}

webapp/backend/pkg/models/measurements/smart_test.go

@@ -328,9 +328,12 @@ func TestFromCollectorSmartInfo(t *testing.T) {
 	require.Equal(t, 18, len(smartMdl.Attributes))
 
 	//check that temperature was correctly parsed
-
 	require.Equal(t, int64(163210330144), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).RawValue)
 	require.Equal(t, int64(32), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).TransformedValue)
+
+	//ensure that Scrutiny warning for a non critical attribute does not set device status to failed.
+	require.Equal(t, pkg.AttributeStatusWarningScrutiny, smartMdl.Attributes["3"].GetStatus())
+
 }
 
 func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) {
@@ -402,7 +405,7 @@ func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
 	require.Equal(t, pkg.DeviceStatusFailedScrutiny, smartMdl.Status)
-	require.Equal(t, int64(pkg.SmartAttributeStatusFailed), smartMdl.Attributes["199"].GetStatus(),
+	require.Equal(t, pkg.AttributeStatusFailedScrutiny, smartMdl.Attributes["199"].GetStatus(),
 		"scrutiny should detect that %d failed (status: %d, %s)",
 		smartMdl.Attributes["199"].(*measurements.SmartAtaAttribute).AttributeId,
 		smartMdl.Attributes["199"].GetStatus(), smartMdl.Attributes["199"].(*measurements.SmartAtaAttribute).StatusReason,
@@ -435,7 +438,7 @@ func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
 	require.Equal(t, pkg.DeviceStatusFailedScrutiny, smartMdl.Status)
-	require.Equal(t, int64(pkg.SmartAttributeStatusFailed), smartMdl.Attributes["media_errors"].GetStatus(),
+	require.Equal(t, pkg.AttributeStatusFailedScrutiny, smartMdl.Attributes["media_errors"].GetStatus(),
 		"scrutiny should detect that %s failed (status: %d, %s)",
 		smartMdl.Attributes["media_errors"].(*measurements.SmartNvmeAttribute).AttributeId,
 		smartMdl.Attributes["media_errors"].GetStatus(),

webapp/backend/pkg/version/version.go

@@ -2,4 +2,4 @@ package version
 
 // VERSION is the app-global version string, which will be replaced with a
 // new value during packaging
-const VERSION = "0.4.8"
+const VERSION = "0.4.9"

webapp/backend/pkg/web/server.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 type AppEngine struct {
@@ -68,6 +69,11 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 }
 
 func (ae *AppEngine) Start() error {
+	//set the gin mode
+	gin.SetMode(gin.ReleaseMode)
+	if strings.ToLower(ae.Config.GetString("log.level")) == "debug" {
+		gin.SetMode(gin.DebugMode)
+	}
 
 	logger := logrus.New()
 	//set default log level

webapp/frontend/src/@treo/services/config/config.service.ts

@@ -2,6 +2,7 @@ import { Inject, Injectable } from '@angular/core';
 import { BehaviorSubject, Observable } from 'rxjs';
 import * as _ from 'lodash';
 import { TREO_APP_CONFIG } from '@treo/services/config/config.constants';
+import { AppConfig } from 'app/core/config/app.config';
 
 const SCRUTINY_CONFIG_LOCAL_STORAGE_KEY = 'scrutiny';
 
@@ -12,6 +13,7 @@ export class TreoConfigService
 {
     // Private
     private _config: BehaviorSubject<any>;
+
     /**
      * Constructor
      */
@@ -20,12 +22,11 @@ export class TreoConfigService
         let currentScrutinyConfig = defaultConfig
 
         let localConfigStr = localStorage.getItem(SCRUTINY_CONFIG_LOCAL_STORAGE_KEY)
-        if(localConfigStr){
+        if (localConfigStr){
             //check localstorage for a value
             let localConfig = JSON.parse(localConfigStr)
-            currentScrutinyConfig = localConfig
+            currentScrutinyConfig = Object.assign({}, localConfig, currentScrutinyConfig) // make sure defaults are available if missing from localStorage.
         }
-
         // Set the private defaults
         this._config = new BehaviorSubject(currentScrutinyConfig);
     }
@@ -41,7 +42,7 @@ export class TreoConfigService
     set config(value: any)
     {
         // Merge the new config over to the current config
-        const config = _.merge({}, this._config.getValue(), value);
+        let config = _.merge({}, this._config.getValue(), value);
 
         //Store the config in localstorage
         localStorage.setItem(SCRUTINY_CONFIG_LOCAL_STORAGE_KEY, JSON.stringify(config));
@@ -56,6 +57,10 @@ export class TreoConfigService
         return this._config.asObservable();
     }
 
+    // -----------------------------------------------------------------------------------------------------
+    // @ Private methods
+    // -----------------------------------------------------------------------------------------------------
+
     // -----------------------------------------------------------------------------------------------------
     // @ Public methods
     // -----------------------------------------------------------------------------------------------------

webapp/frontend/src/app/core/config/app.config.ts

@@ -1,7 +1,7 @@
 import { Layout } from "app/layout/layout.types";
 
 // Theme type
-export type Theme = "light" | "dark";
+export type Theme = "light" | "dark" | "system";
 
 /**
  * AppConfig interface. Update this interface to strictly type your config

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html

@@ -2,6 +2,18 @@
 <mat-dialog-content class="mat-typography">
 
     <div class="flex flex-col p-8 pb-0 overflow-hidden">
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Dark Mode</mat-label>
+                <mat-select [(ngModel)]="theme">
+                    <mat-option value="system">System</mat-option>
+                    <mat-option value="dark">Dark</mat-option>
+                    <mat-option value="light">Light</mat-option>
+                </mat-select>
+            </mat-form-field>
+        </div>
+
         <div class="flex flex-col mt-5 gt-md:flex-row">
             <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
                 <mat-label>Display Title</mat-label>
@@ -38,24 +50,24 @@
             <mat-tab-group mat-align-tabs="start">
                 <mat-tab label="Ata">
 
-                    <div matTooltip="not yet implemented" class="gray-200 flex flex-col mt-5 gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Critical Error Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
                             <input disabled matInput [value]="'10%'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Critical Warning Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
                             <input disabled matInput>
                         </mat-form-field>
                     </div>
 
-                    <div matTooltip="not yet implemented" class="gray-200 flex flex-col gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Error Threshold</mat-label>
+                            <mat-label class="text-hint">Error Threshold</mat-label>
                             <input disabled matInput [value]="'20%'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Warning Threshold</mat-label>
+                            <mat-label class="text-hint">Warning Threshold</mat-label>
                             <input disabled matInput [value]="'10%'">
                         </mat-form-field>
                     </div>
@@ -63,26 +75,26 @@
                 </mat-tab>
                 <mat-tab label="NVMe">
 
-                    <div matTooltip="not yet implemented" class="gray-200 flex flex-col mt-5 gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Critical Error Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
                             <input disabled matInput [value]="'enabled'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Critical Warning Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
                             <input disabled matInput>
                         </mat-form-field>
                     </div>
 
                 </mat-tab>
                 <mat-tab label="SCSI">
-                    <div matTooltip="not yet implemented" class="gray-200 flex flex-col mt-5 gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Critical Error Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
                             <input disabled matInput [value]="'enabled'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Critical Warning Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
                             <input disabled matInput>
                         </mat-form-field>
                     </div>

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts

@@ -13,7 +13,8 @@ export class DashboardSettingsComponent implements OnInit {
 
     dashboardDisplay: string;
     dashboardSort: string;
-    temperatureUnit: string
+    temperatureUnit: string;
+    theme: string;
 
     // Private
     private _unsubscribeAll: Subject<any>;
@@ -35,14 +36,20 @@ export class DashboardSettingsComponent implements OnInit {
               this.dashboardDisplay = config.dashboardDisplay;
               this.dashboardSort = config.dashboardSort;
               this.temperatureUnit = config.temperatureUnit;
+              this.theme = config.theme;
+
           });
+
   }
 
   saveSettings(): void {
+
+
         const newSettings = {
             dashboardDisplay: this.dashboardDisplay,
             dashboardSort: this.dashboardSort,
             temperatureUnit: this.temperatureUnit,
+            theme: this.theme
         }
         this._configService.config = newSettings
         console.log(`Saved Settings: ${JSON.stringify(newSettings)}`)

webapp/frontend/src/app/layout/layout.component.ts

@@ -23,6 +23,7 @@ export class LayoutComponent implements OnInit, OnDestroy
 
     // Private
     private _unsubscribeAll: Subject<any>;
+    private systemPrefersDark: boolean;
 
     /**
      * Constructor
@@ -43,6 +44,9 @@ export class LayoutComponent implements OnInit, OnDestroy
     {
         // Set the private defaults
         this._unsubscribeAll = new Subject();
+
+        this.systemPrefersDark = window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches;
+
     }
 
     // -----------------------------------------------------------------------------------------------------
@@ -66,7 +70,7 @@ export class LayoutComponent implements OnInit, OnDestroy
                 this.theme = config.theme;
 
                 // Update the selected theme class name on body
-                const themeName = 'treo-theme-' + config.theme;
+                const themeName = 'treo-theme-' + this.determineTheme(config);
                 this._document.body.classList.forEach((className) => {
                     if ( className.startsWith('treo-theme-') && className !== themeName )
                     {
@@ -105,6 +109,17 @@ export class LayoutComponent implements OnInit, OnDestroy
     // @ Private methods
     // -----------------------------------------------------------------------------------------------------
 
+    /**
+     * Checks if theme should be set to dark based on config & system settings
+     */
+    private determineTheme(config:AppConfig): string {
+        if (config.theme === 'system') {
+            return this.systemPrefersDark ? 'dark' : 'light'
+        } else {
+            return config.theme
+        }
+    }
+
     /**
      * Update the selected layout
      */

webapp/frontend/src/app/modules/detail/detail.component.html

@@ -71,6 +71,16 @@
                         <div>{{device?.host_id}}</div>
                         <div class="text-secondary text-md">Host ID</div>
                     </div>
+
+                    <div *ngIf="device?.device_uuid" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.device_uuid}}</div>
+                        <div class="text-secondary text-md">Device UUID</div>
+                    </div>
+                    <div *ngIf="device?.device_label" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.device_label}}</div>
+                        <div class="text-secondary text-md">Device Label</div>
+                    </div>
+
                     <div *ngIf="device?.device_type && device?.device_type != 'ata' && device?.device_type != 'scsi'" class="my-2 col-span-2 lt-md:col-span-1">
                         <div>{{device?.device_type | uppercase}}</div>
                         <div class="text-secondary text-md">Device Type</div>

webapp/frontend/src/app/modules/detail/detail.component.ts

@@ -121,25 +121,34 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
     // -----------------------------------------------------------------------------------------------------
     // @ Private methods
     // -----------------------------------------------------------------------------------------------------
-    getAttributeStatusName(attribute_status){
+    getAttributeStatusName(attributeStatus: number): string {
-        if(attribute_status == 0){
+        // tslint:disable:no-bitwise
-            return "passed"
+
-        } else if (attribute_status == 1){
+        // from Constants.go
-            return "failed"
+        // AttributeStatusPassed AttributeStatus = 0
-        } else if (attribute_status == 2){
+        // AttributeStatusFailedSmart AttributeStatus = 1
-            return "warn"
+        // AttributeStatusWarningScrutiny AttributeStatus = 2
+        // AttributeStatusFailedScrutiny AttributeStatus = 4
+
+        if(attributeStatus === 0){
+            return 'passed'
+
+        } else if ((attributeStatus & 1) !== 0 || (attributeStatus & 4) !== 0 ){
+            return 'failed'
+        } else if ((attributeStatus & 2) !== 0){
+            return 'warn'
         }
-        return
+        return ''
+        // tslint:enable:no-bitwise
     }
 
-    getAttributeName(attribute_data){
+    getAttributeName(attribute_data): string {
         let attribute_metadata = this.metadata[attribute_data.attribute_id]
         if(!attribute_metadata){
             return 'Unknown Attribute Name'
         } else {
             return attribute_metadata.display_name
         }
-        return
     }
     getAttributeDescription(attribute_data){
         let attribute_metadata = this.metadata[attribute_data.attribute_id]

webapp/frontend/src/styles/styles.scss

@@ -4,3 +4,15 @@
 // @ Styles from this file will override anything from 'vendors.scss' file allowing customizations and
 //  modifications of third party libraries.
 // -----------------------------------------------------------------------------------------------------
+
+.treo-theme-dark .yellow-50 {
+    background-color: #242b38 !important;
+
+    .mat-icon {
+        color: #0694a2 !important;
+    }
+
+    .text-secondary {
+        color: #0694a2 !important
+    }
+}