(v0.4.12) Automated packaging of release by Packagr

fixes #289

.dockerignore

@@ -1,4 +1,3 @@
-/dist
 /vendor
 /.idea
 /.github

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,20 +22,21 @@ See [/docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](docs/TROUBLESHOOTING_DEVICE_COLL
 
 ```
 docker run -it --rm -p 8080:8080 \
+-v `pwd`/config:/opt/scrutiny/config \
 -v /run/udev:/run/udev:ro \
 --cap-add SYS_RAWIO \
 --device=/dev/sda \
 --device=/dev/sdb \
 -e DEBUG=true \
--e COLLECTOR_LOG_FILE=/tmp/collector.log \
+-e COLLECTOR_LOG_FILE=/opt/scrutiny/config/collector.log \
--e SCRUTINY_LOG_FILE=/tmp/web.log \
+-e SCRUTINY_LOG_FILE=/opt/scrutiny/config/web.log \
 --name scrutiny \
 ghcr.io/analogj/scrutiny:master-omnibus
 
 # in another terminal trigger the collector
 docker exec scrutiny scrutiny-collector-metrics run
-
-# then use docker cp to copy the log files out of the container.
-docker cp scrutiny:/tmp/collector.log collector.log
-docker cp scrutiny:/tmp/web.log web.log
 ```
+
+The log files will be available on your host in the `config` directory. Please attach them to this issue. 
+
+Please also provide the output of `docker info`

.github/workflows/docker-build.yaml

@@ -1,7 +1,5 @@
 name: Docker
 on:
-  schedule:
-    - cron: '36 12 * * *'
   push:
     branches: [ master, beta ]
     # Publish semver tags as releases.
@@ -25,6 +23,8 @@ jobs:
           fetch-depth: 0
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       # Login against a Docker registry except on PR
@@ -60,8 +60,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
+#          cache-from: type=gha
-          cache-to: type=gha,mode=max
+#          cache-to: type=gha,mode=max
 
   web:
     runs-on: ubuntu-latest
@@ -72,8 +72,21 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: "Populate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
+      - name: "Generate frontend"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: node:lts
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            cd /work
+            make frontend && echo "print contents of /work/dist" && ls -alt /work/dist
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       # Login against a Docker registry except on PR
@@ -97,8 +110,6 @@ jobs:
             type=ref,enable=true,event=branch,suffix=-web
             type=ref,enable=true,event=tag,suffix=-web
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-      - name: "Generate frontend version information"
-        run: "cd webapp/frontend && ./git.version.sh"
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
@@ -110,8 +121,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
+#          cache-from: type=gha
-          cache-to: type=gha,mode=max
+#          cache-to: type=gha,mode=max
   omnibus:
     runs-on: ubuntu-latest
     permissions:
@@ -121,8 +132,22 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: "Populate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
+      - name: "Generate frontend & version information"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: node:lts
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            cd /work
+            make frontend && echo "print contents of /work/dist" && ls -alt /work/dist
+
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       # Login against a Docker registry except on PR
@@ -144,8 +169,6 @@ jobs:
             type=ref,enable=true,event=branch,suffix=-omnibus
             type=ref,enable=true,event=tag,suffix=-omnibus
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-      - name: "Generate frontend version information"
-        run: "cd webapp/frontend && ./git.version.sh"
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
@@ -157,5 +180,5 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
+#          cache-from: type=gha
-          cache-to: type=gha,mode=max
+#          cache-to: type=gha,mode=max

.github/workflows/docker-nightly.yaml

@@ -0,0 +1,69 @@
+name: Docker
+on:
+  schedule:
+    - cron: '36 12 * * *'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  omnibus:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: "Populate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
+      - name: "Generate frontend & version information"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: node:lts
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            cd /work
+            make frontend && echo "print contents of /work/dist" && ls -alt /work/dist
+
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          tags: |
+            type=ref,enable=true,event=branch,suffix=-omnibus-nightly
+            type=ref,enable=true,event=tag,suffix=-omnibus-nightly
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+          context: .
+          file: docker/Dockerfile
+          push: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+#          cache-from: type=gha
+#          cache-to: type=gha,mode=max

CONTRIBUTING.md

@@ -169,3 +169,19 @@ docker run -it --rm -p 8080:8080 \
 ghcr.io/analogj/scrutiny:master-omnibus
 /opt/scrutiny/bin/scrutiny-collector-metrics run
 ```
+
+
+# Running Tests
+
+```bash
+docker run -p 8086:8086 -d --rm \
+-e DOCKER_INFLUXDB_INIT_MODE=setup \
+-e DOCKER_INFLUXDB_INIT_USERNAME=admin \
+-e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
+-e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
+-e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
+-e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \
+influxdb:2.2
+go test ./...
+
+```

Makefile

@@ -9,6 +9,7 @@ BINARY=\
   linux/arm-7   \
   linux/arm64   \
 
+.ONESHELL: # Applies to every targets in the file! .ONESHELL instructs make to invoke a single instance of the shell and provide it with the entire recipe, regardless of how many lines it contains.
 .PHONY: all $(BINARY)
 all: $(BINARY) windows/amd64
 
@@ -38,5 +39,28 @@ windows/amd64:
 	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
 	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
 
+
+docker-collector:
+	@echo "building collector docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.collector -t analogj/scrutiny-dev:collector .
+
+docker-web:
+	@echo "building web docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.web -t analogj/scrutiny-dev:web .
+
+docker-omnibus:
+	@echo "building omnibus docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile -t analogj/scrutiny-dev:omnibus .
+
+# reduce logging, disable angular-cli analytics for ci environment
+frontend: export NPM_CONFIG_LOGLEVEL = warn
+frontend: export NG_CLI_ANALYTICS = false
+frontend:
+	cd webapp/frontend
+	npm install -g @angular/cli@9.1.4
+	mkdir -p $(CURDIR)/dist
+	npm ci
+	npm run build:prod -- --output-path=$(CURDIR)/dist
+
 # clean:
 # 	rm scrutiny-collector-metrics-* scrutiny-web-*

README.md

@@ -17,8 +17,6 @@
 WebUI for smartd S.M.A.R.T monitoring
 
 > NOTE: Scrutiny is a Work-in-Progress and still has some rough edges.
->
-> WARNING: Once the [InfluxDB](https://github.com/AnalogJ/scrutiny/tree/influxdb) branch is merged, Scrutiny will use both sqlite and InfluxDB for data storage. Unfortunately, this may not be backwards compatible with the database structures in the master (sqlite only) branch. 
 
 [![](docs/dashboard.png)](https://imgur.com/a/5k8qMzS)
 
@@ -60,11 +58,12 @@ Scrutiny uses `smartctl --scan` to detect devices/drives.
 - All RAID controllers supported by `smartctl` are automatically supported by Scrutiny.
     - While some RAID controllers support passing through the underlying SMART data to `smartctl` others do not.
     - In some cases `--scan` does not correctly detect the device type, returning [incomplete SMART data](https://github.com/AnalogJ/scrutiny/issues/45).
-    Scrutiny will eventually support overriding detected device type via the config file.
+    Scrutiny supports overriding detected device type via the config file: see [example.collector.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.collector.yaml)
 - If you use docker, you **must** pass though the RAID virtual disk to the container using `--device` (see below)
     - This device may be in `/dev/*` or `/dev/bus/*`.
     - If you're unsure, run `smartctl --scan` on your host, and pass all listed devices to the container.
 
+See [docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](./docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md) for help
 
 ## Docker
 

collector/pkg/collector/metrics.go

@@ -116,12 +116,13 @@ func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceT
 	}
 	mc.logger.Infof("Collecting smartctl results for %s\n", deviceName)
 
-	args := []string{"-x", "-j"}
+	fullDeviceName := fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName)
+	args := strings.Split(mc.config.GetCommandMetricsSmartArgs(fullDeviceName), " ")
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
 	if len(deviceType) > 0 && deviceType != "scsi" && deviceType != "ata" {
-		args = append(args, "-d", deviceType)
+		args = append(args, "--device", deviceType)
 	}
-	args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName))
+	args = append(args, fullDeviceName)
 
 	result, err := mc.shell.Command(mc.logger, "smartctl", args, "", os.Environ())
 	resultBytes := []byte(result)

collector/pkg/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"fmt"
 	"github.com/analogj/go-util/utils"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/collector/pkg/models"
@@ -8,6 +9,8 @@ import (
 	"github.com/spf13/viper"
 	"log"
 	"os"
+	"sort"
+	"strings"
 )
 
 // When initializing this class the following methods must be called:
@@ -16,6 +19,8 @@ import (
 // This is done automatically when created via the Factory.
 type configuration struct {
 	*viper.Viper
+
+	deviceOverrides []models.ScanOverride
 }
 
 //Viper uses the following precedence order. Each item takes precedence over the item below it:
@@ -38,6 +43,10 @@ func (c *configuration) Init() error {
 
 	c.SetDefault("api.endpoint", "http://localhost:8080")
 
+	c.SetDefault("commands.metrics_scan_args", "--scan --json")
+	c.SetDefault("commands.metrics_info_args", "--info --json")
+	c.SetDefault("commands.metrics_smart_args", "--xall --json")
+
 	//c.SetDefault("collect.short.command", "-a -o on -S on")
 
 	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
@@ -90,16 +99,89 @@ func (c *configuration) ValidateConfig() error {
 	// check that device prefix matches OS
 	// check that schema of config file is valid
 
-	return nil
+	// check that the collector commands are valid
+	commandArgStrings := map[string]string{
+		"commands.metrics_scan_args":  c.GetString("commands.metrics_scan_args"),
+		"commands.metrics_info_args":  c.GetString("commands.metrics_info_args"),
+		"commands.metrics_smart_args": c.GetString("commands.metrics_smart_args"),
+	}
+
+	errorStrings := []string{}
+	for configKey, commandArgString := range commandArgStrings {
+		args := strings.Split(commandArgString, " ")
+		//ensure that the args string contains `--json` or `-j` flag
+		containsJsonFlag := false
+		containsDeviceFlag := false
+		for _, flag := range args {
+			if strings.HasPrefix(flag, "--json") || strings.HasPrefix(flag, "-j") {
+				containsJsonFlag = true
+			}
+			if strings.HasPrefix(flag, "--device") || strings.HasPrefix(flag, "-d") {
+				containsDeviceFlag = true
+			}
+		}
+
+		if !containsJsonFlag {
+			errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' is missing '--json' flag", configKey))
+		}
+
+		if containsDeviceFlag {
+			errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' must not contain '--device' or '-d' flag", configKey))
+		}
+	}
+	//sort(errorStrings)
+	sort.Strings(errorStrings)
+
+	if len(errorStrings) == 0 {
+		return nil
+	} else {
+		return errors.ConfigValidationError(strings.Join(errorStrings, ", "))
+	}
 }
 
-func (c *configuration) GetScanOverrides() []models.ScanOverride {
+func (c *configuration) GetDeviceOverrides() []models.ScanOverride {
 	// we have to support 2 types of device types.
 	// - simple device type (device_type: 'sat')
 	// and list of device types (type: \n- 3ware,0 \n- 3ware,1 \n- 3ware,2)
 	// GetString will return "" if this is a list of device types.
 
-	overrides := []models.ScanOverride{}
+	if c.deviceOverrides == nil {
-	c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
+		overrides := []models.ScanOverride{}
-	return overrides
+		c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
+		c.deviceOverrides = overrides
+	}
+
+	return c.deviceOverrides
+}
+
+func (c *configuration) GetCommandMetricsInfoArgs(deviceName string) string {
+	overrides := c.GetDeviceOverrides()
+
+	for _, deviceOverrides := range overrides {
+		if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) {
+			//found matching device
+			if len(deviceOverrides.Commands.MetricsInfoArgs) > 0 {
+				return deviceOverrides.Commands.MetricsInfoArgs
+			} else {
+				return c.GetString("commands.metrics_info_args")
+			}
+		}
+	}
+	return c.GetString("commands.metrics_info_args")
+}
+
+func (c *configuration) GetCommandMetricsSmartArgs(deviceName string) string {
+	overrides := c.GetDeviceOverrides()
+
+	for _, deviceOverrides := range overrides {
+		if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) {
+			//found matching device
+			if len(deviceOverrides.Commands.MetricsSmartArgs) > 0 {
+				return deviceOverrides.Commands.MetricsSmartArgs
+			} else {
+				return c.GetString("commands.metrics_smart_args")
+			}
+		}
+	}
+	return c.GetString("commands.metrics_smart_args")
 }

collector/pkg/config/config_test.go

@@ -30,7 +30,7 @@ func TestConfiguration_GetScanOverrides_Simple(t *testing.T) {
 	//test
 	err := testConfig.ReadConfig(path.Join("testdata", "simple_device.yaml"))
 	require.NoError(t, err, "should correctly load simple device config")
-	scanOverrides := testConfig.GetScanOverrides()
+	scanOverrides := testConfig.GetDeviceOverrides()
 
 	//assert
 	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat"}, Ignore: false}}, scanOverrides)
@@ -45,7 +45,7 @@ func TestConfiguration_GetScanOverrides_Ignore(t *testing.T) {
 	//test
 	err := testConfig.ReadConfig(path.Join("testdata", "ignore_device.yaml"))
 	require.NoError(t, err, "should correctly load ignore device config")
-	scanOverrides := testConfig.GetScanOverrides()
+	scanOverrides := testConfig.GetDeviceOverrides()
 
 	//assert
 	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}, scanOverrides)
@@ -60,7 +60,7 @@ func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
 	//test
 	err := testConfig.ReadConfig(path.Join("testdata", "raid_device.yaml"))
 	require.NoError(t, err, "should correctly load ignore device config")
-	scanOverrides := testConfig.GetScanOverrides()
+	scanOverrides := testConfig.GetDeviceOverrides()
 
 	//assert
 	require.Equal(t, []models.ScanOverride{
@@ -75,3 +75,53 @@ func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
 			Ignore:     false,
 		}}, scanOverrides)
 }
+
+func TestConfiguration_InvalidCommands_MissingJson(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_missing_json.yaml"))
+	require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_scan_args' is missing '--json' flag"`, "should throw an error because json flag is missing")
+}
+
+func TestConfiguration_InvalidCommands_IncludesDevice(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_includes_device.yaml"))
+	require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_info_args' must not contain '--device' or '-d' flag, configuration key 'commands.metrics_smart_args' must not contain '--device' or '-d' flag"`, "should throw an error because device flags detected")
+}
+
+func TestConfiguration_OverrideCommands(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "override_commands.yaml"))
+	require.NoError(t, err, "should not throw an error")
+	require.Equal(t, "--xall --json -T permissive", testConfig.GetString("commands.metrics_smart_args"))
+}
+
+func TestConfiguration_OverrideDeviceCommands_MetricsInfoArgs(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "override_device_commands.yaml"))
+	require.NoError(t, err, "should correctly override device command")
+
+	//assert
+	require.Equal(t, "--info --json -T permissive", testConfig.GetCommandMetricsInfoArgs("/dev/sda"))
+	require.Equal(t, "--info --json", testConfig.GetCommandMetricsInfoArgs("/dev/sdb"))
+	//require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Commands: {MetricsInfoArgs: "--info --json -T "}}}, scanOverrides)
+}

collector/pkg/config/interface.go

@@ -22,5 +22,7 @@ type Interface interface {
 	GetStringSlice(key string) []string
 	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
 
-	GetScanOverrides() []models.ScanOverride
+	GetDeviceOverrides() []models.ScanOverride
+	GetCommandMetricsInfoArgs(deviceName string) string
+	GetCommandMetricsSmartArgs(deviceName string) string
 }

collector/pkg/config/mock/mock_config.go

@@ -5,88 +5,37 @@
 package mock_config
 
 import (
+	reflect "reflect"
+
 	models "github.com/analogj/scrutiny/collector/pkg/models"
 	gomock "github.com/golang/mock/gomock"
 	viper "github.com/spf13/viper"
-	reflect "reflect"
 )
 
-// MockInterface is a mock of Interface interface
+// MockInterface is a mock of Interface interface.
 type MockInterface struct {
 	ctrl     *gomock.Controller
 	recorder *MockInterfaceMockRecorder
 }
 
-// MockInterfaceMockRecorder is the mock recorder for MockInterface
+// MockInterfaceMockRecorder is the mock recorder for MockInterface.
 type MockInterfaceMockRecorder struct {
 	mock *MockInterface
 }
 
-// NewMockInterface creates a new mock instance
+// NewMockInterface creates a new mock instance.
 func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
 	mock := &MockInterface{ctrl: ctrl}
 	mock.recorder = &MockInterfaceMockRecorder{mock}
 	return mock
 }
 
-// EXPECT returns an object that allows the caller to indicate expected use
+// EXPECT returns an object that allows the caller to indicate expected use.
 func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
-// Init mocks base method
+// AllSettings mocks base method.
-func (m *MockInterface) Init() error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Init")
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Init indicates an expected call of Init
-func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
-}
-
-// ReadConfig mocks base method
-func (m *MockInterface) ReadConfig(configFilePath string) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// ReadConfig indicates an expected call of ReadConfig
-func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
-}
-
-// Set mocks base method
-func (m *MockInterface) Set(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Set", key, value)
-}
-
-// Set indicates an expected call of Set
-func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
-}
-
-// SetDefault mocks base method
-func (m *MockInterface) SetDefault(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "SetDefault", key, value)
-}
-
-// SetDefault indicates an expected call of SetDefault
-func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
-}
-
-// AllSettings mocks base method
 func (m *MockInterface) AllSettings() map[string]interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "AllSettings")
@@ -94,27 +43,13 @@ func (m *MockInterface) AllSettings() map[string]interface{} {
 	return ret0
 }
 
-// AllSettings indicates an expected call of AllSettings
+// AllSettings indicates an expected call of AllSettings.
 func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings))
 }
 
-// IsSet mocks base method
+// Get mocks base method.
-func (m *MockInterface) IsSet(key string) bool {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "IsSet", key)
-	ret0, _ := ret[0].(bool)
-	return ret0
-}
-
-// IsSet indicates an expected call of IsSet
-func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
-}
-
-// Get mocks base method
 func (m *MockInterface) Get(key string) interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Get", key)
@@ -122,13 +57,13 @@ func (m *MockInterface) Get(key string) interface{} {
 	return ret0
 }
 
-// Get indicates an expected call of Get
+// Get indicates an expected call of Get.
 func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key)
 }
 
-// GetBool mocks base method
+// GetBool mocks base method.
 func (m *MockInterface) GetBool(key string) bool {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetBool", key)
@@ -136,13 +71,55 @@ func (m *MockInterface) GetBool(key string) bool {
 	return ret0
 }
 
-// GetBool indicates an expected call of GetBool
+// GetBool indicates an expected call of GetBool.
 func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key)
 }
 
-// GetInt mocks base method
+// GetCommandMetricsInfoArgs mocks base method.
+func (m *MockInterface) GetCommandMetricsInfoArgs(deviceName string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCommandMetricsInfoArgs", deviceName)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetCommandMetricsInfoArgs indicates an expected call of GetCommandMetricsInfoArgs.
+func (mr *MockInterfaceMockRecorder) GetCommandMetricsInfoArgs(deviceName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsInfoArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsInfoArgs), deviceName)
+}
+
+// GetCommandMetricsSmartArgs mocks base method.
+func (m *MockInterface) GetCommandMetricsSmartArgs(deviceName string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCommandMetricsSmartArgs", deviceName)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetCommandMetricsSmartArgs indicates an expected call of GetCommandMetricsSmartArgs.
+func (mr *MockInterfaceMockRecorder) GetCommandMetricsSmartArgs(deviceName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsSmartArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsSmartArgs), deviceName)
+}
+
+// GetDeviceOverrides mocks base method.
+func (m *MockInterface) GetDeviceOverrides() []models.ScanOverride {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDeviceOverrides")
+	ret0, _ := ret[0].([]models.ScanOverride)
+	return ret0
+}
+
+// GetDeviceOverrides indicates an expected call of GetDeviceOverrides.
+func (mr *MockInterfaceMockRecorder) GetDeviceOverrides() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceOverrides", reflect.TypeOf((*MockInterface)(nil).GetDeviceOverrides))
+}
+
+// GetInt mocks base method.
 func (m *MockInterface) GetInt(key string) int {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetInt", key)
@@ -150,13 +127,13 @@ func (m *MockInterface) GetInt(key string) int {
 	return ret0
 }
 
-// GetInt indicates an expected call of GetInt
+// GetInt indicates an expected call of GetInt.
 func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
 }
 
-// GetString mocks base method
+// GetString mocks base method.
 func (m *MockInterface) GetString(key string) string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetString", key)
@@ -164,13 +141,13 @@ func (m *MockInterface) GetString(key string) string {
 	return ret0
 }
 
-// GetString indicates an expected call of GetString
+// GetString indicates an expected call of GetString.
 func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key)
 }
 
-// GetStringSlice mocks base method
+// GetStringSlice mocks base method.
 func (m *MockInterface) GetStringSlice(key string) []string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetStringSlice", key)
@@ -178,13 +155,79 @@ func (m *MockInterface) GetStringSlice(key string) []string {
 	return ret0
 }
 
-// GetStringSlice indicates an expected call of GetStringSlice
+// GetStringSlice indicates an expected call of GetStringSlice.
 func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key)
 }
 
-// UnmarshalKey mocks base method
+// Init mocks base method.
+func (m *MockInterface) Init() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Init")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Init indicates an expected call of Init.
+func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
+}
+
+// IsSet mocks base method.
+func (m *MockInterface) IsSet(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsSet", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsSet indicates an expected call of IsSet.
+func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
+}
+
+// ReadConfig mocks base method.
+func (m *MockInterface) ReadConfig(configFilePath string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReadConfig indicates an expected call of ReadConfig.
+func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
+}
+
+// Set mocks base method.
+func (m *MockInterface) Set(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Set", key, value)
+}
+
+// Set indicates an expected call of Set.
+func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
+}
+
+// SetDefault mocks base method.
+func (m *MockInterface) SetDefault(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetDefault", key, value)
+}
+
+// SetDefault indicates an expected call of SetDefault.
+func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
+}
+
+// UnmarshalKey mocks base method.
 func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{key, rawVal}
@@ -196,23 +239,9 @@ func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts
 	return ret0
 }
 
-// UnmarshalKey indicates an expected call of UnmarshalKey
+// UnmarshalKey indicates an expected call of UnmarshalKey.
 func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{key, rawVal}, decoderOpts...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...)
 }
-
-// GetScanOverrides mocks base method
-func (m *MockInterface) GetScanOverrides() []models.ScanOverride {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetScanOverrides")
-	ret0, _ := ret[0].([]models.ScanOverride)
-	return ret0
-}
-
-// GetScanOverrides indicates an expected call of GetScanOverrides
-func (mr *MockInterfaceMockRecorder) GetScanOverrides() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScanOverrides", reflect.TypeOf((*MockInterface)(nil).GetScanOverrides))
-}

collector/pkg/config/testdata/invalid_commands_includes_device.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan --json' # used to detect devices
+  metrics_info_args: '--info --json --device=sat' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json -d sat' # used to retrieve smart data for each device.

collector/pkg/config/testdata/invalid_commands_missing_json.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan' # used to detect devices
+  metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json' # used to retrieve smart data for each device.

collector/pkg/config/testdata/override_commands.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan --json' # used to detect devices
+  metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device.

collector/pkg/config/testdata/override_device_commands.yaml

@@ -0,0 +1,5 @@
+version: 1
+devices:
+  - device: /dev/sda
+    commands:
+      metrics_info_args: "--info --json -T permissive"

collector/pkg/detect/detect.go

@@ -28,7 +28,8 @@ type Detect struct {
 // models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
 func (d *Detect) SmartctlScan() ([]models.Device, error) {
 	//we use smartctl to detect all the drives available.
-	detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
+	args := strings.Split(d.Config.GetString("commands.metrics_scan_args"), " ")
+	detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ())
 	if err != nil {
 		d.Logger.Errorf("Error scanning for devices: %v", err)
 		return nil, err
@@ -51,13 +52,13 @@ func (d *Detect) SmartctlScan() ([]models.Device, error) {
 // - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
 // - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
 func (d *Detect) SmartCtlInfo(device *models.Device) error {
-
+	fullDeviceName := fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName)
-	args := []string{"--info", "-j"}
+	args := strings.Split(d.Config.GetCommandMetricsInfoArgs(fullDeviceName), " ")
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
 	if len(device.DeviceType) > 0 && device.DeviceType != "scsi" && device.DeviceType != "ata" {
-		args = append(args, "-d", device.DeviceType)
+		args = append(args, "--device", device.DeviceType)
 	}
-	args = append(args, fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName))
+	args = append(args, fullDeviceName)
 
 	availableDeviceInfoJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ())
 	if err != nil {
@@ -138,7 +139,7 @@ func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []mod
 
 	//now tha we've "grouped" all the devices, lets override any groups specified in the config file.
 
-	for _, overrideDevice := range d.Config.GetScanOverrides() {
+	for _, overrideDevice := range d.Config.GetDeviceOverrides() {
 		overrideDeviceFile := strings.ToLower(overrideDevice.Device)
 
 		if overrideDevice.Ignore {

collector/pkg/detect/detect_test.go

@@ -18,7 +18,8 @@ func TestDetect_SmartctlScan(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
 	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_simple.json")
@@ -45,7 +46,8 @@ func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
 	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_megaraid.json")
@@ -75,7 +77,8 @@ func TestDetect_SmartctlScan_Nvme(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
 	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_nvme.json")
@@ -104,7 +107,9 @@ func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -134,7 +139,9 @@ func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -163,7 +170,8 @@ func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{
 		{
 			Device:     "/dev/bus/0",
 			DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
@@ -202,7 +210,8 @@ func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{

collector/pkg/models/scan_override.go

@@ -4,4 +4,8 @@ type ScanOverride struct {
 	Device     string   `mapstructure:"device"`
 	DeviceType []string `mapstructure:"type"`
 	Ignore     bool     `mapstructure:"ignore"`
+	Commands   struct {
+		MetricsInfoArgs  string `mapstructure:"metrics_info_args"`
+		MetricsSmartArgs string `mapstructure:"metrics_smart_args"`
+	} `mapstructure:"commands"`
 }

docker/Dockerfile

@@ -1,49 +1,34 @@
 ########
-FROM golang:1.17.10-buster as backendbuild
+FROM golang:1.17-bullseye as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
 COPY . /go/src/github.com/analogj/scrutiny
 
 RUN go mod vendor && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go && \
+    go build -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-selftest collector/cmd/collector-selftest/collector-selftest.go && \
+    go build -o scrutiny-collector-selftest collector/cmd/collector-selftest/collector-selftest.go && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
+    go build -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
 
 ########
-FROM node:lts-slim as frontendbuild
+FROM debian:bullseye-slim as runtime
-
-#reduce logging, disable angular-cli analytics for ci environment
-ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
-
-WORKDIR /opt/scrutiny/src
-COPY webapp/frontend /opt/scrutiny/src
-
-RUN npm install -g @angular/cli@9.1.4 && \
-    mkdir -p /scrutiny/dist && \
-    npm install && \
-    npm run build:prod -- --output-path=/opt/scrutiny/dist
-
-
-########
-FROM ubuntu:bionic as runtime
 ARG TARGETARCH
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 ENV INFLUXD_CONFIG_PATH=/opt/scrutiny/influxdb
 
-RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates curl tzdata \
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates curl tzdata \
     && update-ca-certificates \
     &&  case ${TARGETARCH} in \
             "amd64")  S6_ARCH=amd64  ;; \
             "arm64")  S6_ARCH=aarch64  ;; \
         esac \
     && curl https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-${S6_ARCH}.tar.gz -L -s --output /tmp/s6-overlay-${S6_ARCH}.tar.gz \
-    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C /
+    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C / \
-
+    && rm -rf /tmp/s6-overlay-${S6_ARCH}.tar.gz \
-ADD https://dl.influxdata.com/influxdb/releases/influxdb2-2.2.0-${TARGETARCH}.deb /tmp/
+    && curl -L https://dl.influxdata.com/influxdb/releases/influxdb2-2.2.0-${TARGETARCH}.deb --output /tmp/influxdb2-2.2.0-${TARGETARCH}.deb \
-RUN dpkg -i /tmp/influxdb2-2.2.0-${TARGETARCH}.deb && rm -rf /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
+    && dpkg -i --force-all /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
 
 COPY /rootfs /
 
@@ -51,7 +36,7 @@ COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /opt/scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
-COPY --from=frontendbuild /opt/scrutiny/dist /opt/scrutiny/web
+COPY dist /opt/scrutiny/web
 RUN chmod +x /opt/scrutiny/bin/scrutiny && \
     chmod +x /opt/scrutiny/bin/scrutiny-collector-selftest && \
     chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \

docker/Dockerfile.collector

@@ -1,5 +1,5 @@
 ########
-FROM golang:1.17.10-buster as backendbuild
+FROM golang:1.17-bullseye as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
@@ -10,11 +10,11 @@ RUN go mod vendor && \
     go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
 
 ########
-FROM ubuntu:bionic as runtime
+FROM debian:bullseye-slim as runtime
 WORKDIR /scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates tzdata && update-ca-certificates
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && update-ca-certificates
 
 COPY /docker/entrypoint-collector.sh /entrypoint-collector.sh
 COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny

docker/Dockerfile.web

@@ -1,30 +1,15 @@
 ########
-FROM golang:1.17.10-buster as backendbuild
+FROM golang:1.17-bullseye as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
 COPY . /go/src/github.com/analogj/scrutiny
 
 RUN go mod vendor && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go
+    go build -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go
 
 ########
-FROM node:lts-slim as frontendbuild
+FROM debian:bullseye-slim as runtime
-
-#reduce logging, disable angular-cli analytics for ci environment
-ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
-
-WORKDIR /opt/scrutiny/src
-COPY webapp/frontend /opt/scrutiny/src
-
-RUN npm install -g @angular/cli@9.1.4 && \
-    mkdir -p /opt/scrutiny/dist && \
-    npm install && \
-    npm run build:prod -- --output-path=/opt/scrutiny/dist
-
-
-########
-FROM ubuntu:bionic as runtime
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
@@ -32,7 +17,7 @@ ENV PATH="/opt/scrutiny/bin:${PATH}"
 RUN apt-get update && apt-get install -y ca-certificates curl tzdata && update-ca-certificates
 
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
-COPY --from=frontendbuild /opt/scrutiny/dist /opt/scrutiny/web
+COPY dist /opt/scrutiny/web
 RUN chmod +x /opt/scrutiny/bin/scrutiny && \
     mkdir -p /opt/scrutiny/web && \
     mkdir -p /opt/scrutiny/config && \

docs/INSTALL_HUB_SPOKE.md

@@ -1 +1 @@
-> See [docker/example.hubspoke.docker-compose.yml](./docker/example.hubspoke.docker-compose.yml) for a docker-compose file.
+> See [docker/example.hubspoke.docker-compose.yml](https://github.com/AnalogJ/scrutiny/blob/master/docker/example.hubspoke.docker-compose.yml) for a docker-compose file.

docs/INSTALL_MANUAL.md

@@ -57,7 +57,7 @@ web:
   # and store the information in the config file. If you 're re-using an existing influxdb installation, you'll need to provide
   # the `token`
   influxdb:
-    host: 0.0.0.0
+    host: localhost
     port: 8086
 #    token: 'my-token'
 #    org: 'my-org'
@@ -83,9 +83,11 @@ Now that we have downloaded the required files, let's prepare the filesystem.
 chmod +x /opt/scrutiny/bin/scrutiny-web-linux-amd64
 
 # Next, lets extract the frontend files.
+# NOTE: after extraction, there **should not** be a `dist` subdirectory in `/opt/scrutiny/web` directory.
 cd /opt/scrutiny/web
 tar xvzf scrutiny-web-frontend.tar.gz --strip-components 1 -C .
 
+
 # Cleanup
 rm -rf scrutiny-web-frontend.tar.gz
 ```
@@ -113,7 +115,8 @@ Unlike the webapp, the collector does have some dependencies:
 Unfortunately the version of `smartmontools` (which contains `smartctl`) available in some of the base OS repositories is ancient.
 So you'll need to install the v7+ version using one of the following commands:
 
-- **Ubuntu:** `apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1`
+- **Ubuntu (22.04/Jammy/LTS):** `apt-get install -y smartmontools`
+- **Ubuntu (18.04/Bionic):** `apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1`
 - **Centos8:**
     - `dnf install https://extras.getpagespeed.com/release-el8-latest.rpm`
     - `dnf install smartmontools`

docs/INSTALL_PFSENSE.md

@@ -0,0 +1,72 @@
+# pfsense Install
+
+This bascially follows the [Manual collector instructions](https://github.com/AnalogJ/scrutiny/blob/master/docs/INSTALL_MANUAL.md#collector) and assumes you are running a hub and spoke deployment and already have the web app setup.
+
+
+### Dependencies
+
+SSH into pfsense, hit `8` for the shell and install the required dependencies.
+
+```
+pkg install smartmontools
+```
+
+Ensure smartmontools is v7+. This won't be a problem in pfsense 2.6.0+
+
+
+### Directory Structure
+
+Now let's create a directory structure to contain the Scrutiny collector binary.
+
+```
+mkdir -p /opt/scrutiny/bin
+```
+
+
+### Download Files
+
+Next, we'll download the Scrutiny collector binary from the [latest Github release](https://github.com/analogj/scrutiny/releases).
+
+> NOTE: Ensure you have the latest version in the below command
+
+```
+fetch -o /opt/scrutiny/bin https://github.com/AnalogJ/scrutiny/releases/download/vX.X.X/scrutiny-collector-metrics-freebsd-amd64
+```
+
+
+### Prepare Scrutiny
+
+Now that we have downloaded the required files, let's prepare the filesystem.
+
+```
+chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64
+```
+
+
+### Start Scrutiny Collector, Populate Webapp
+
+Next, we will manually trigger the collector, to populate the Scrutiny dashboard:
+
+> NOTE: if you need to pass a config file to the scrutiny collector, you can provide it using the `--config` flag.
+
+```
+/opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64 run --api-endpoint "http://localhost:8080"
+```
+> NOTE: change the IP address to that of your web app
+
+### Schedule Collector with Cron
+
+Finally you need to schedule the collector to run periodically.
+
+Login to the pfsense webGUI and head to `Services/Cron` add an entry with the following details:
+
+```
+Minute: */15
+Hour: *
+Day of the Month: *
+Month of the Year: *
+Day of the Week: *
+User: root
+Command: /opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64 run --api-endpoint "http://localhost:8080" >/dev/null 2>&1
+```
+> NOTE: `>/dev/null 2>&1` is used to stop cron confirmation emails being sent.

docs/SUPPORTED_NAS_OS.md

@@ -4,11 +4,13 @@ These are the officially supported NAS OS's (with documentation and setup guides
 Once a guide is created (in `docs/guides/`) it will be linked here.
 
 - [ ] freenas/truenas
-- [x] [unraid](https://github.com/AnalogJ/scrutiny/blob/master/docs/INSTALL_UNRAID.md)
+- [x] [unraid](./INSTALL_UNRAID.md)
 - [ ] ESXI
 - [ ] Proxmox
 - [ ] Synology
 - [ ] OMV
 - [ ] Amahi
 - [ ] Running in a LXC container
+- [x] [PFSense](./INSTALL_UNRAID.md)
+- [ ] QNAP
 

docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md

@@ -52,6 +52,8 @@ If the output is the same, your devices will be processed by Scrutiny.
 In some cases `--scan` does not correctly detect the device type, returning [incomplete SMART data](https://github.com/AnalogJ/scrutiny/issues/45).
 Scrutiny will supports overriding the detected device type via the config file.
 
+[example.collector.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.collector.yaml)
+
 ### RAID Controllers (Megaraid/3ware/HBA/Adaptec/HPE/etc)
 Smartctl has support for a large number of [RAID controllers](https://www.smartmontools.org/wiki/Supported_RAID-Controllers), however this 
 support is not automatic, and may require some additional device type hinting. You can provide this information to the Scrutiny collector
@@ -111,12 +113,60 @@ instead of the block device (`/dev/nvme0n1`). See [#209](https://github.com/Anal
 
 ### ATA
 
-### Standby/Sleeping Disks
+### Exit Codes
+
+If you see an error message similar to `smartctl returned an error code (2) while processing /dev/sda`, this means that
+`smartctl` (not Scrutiny) exited with an error code. Scrutiny will attempt to print a helpful error message to help you debug,
+but you can look at the table (and associated links) below to debug `smartctl`. 
+
+> smartctl Return Values
+> The return values of smartctl are defined by a bitmask. If all is well with the disk, the return value (exit status) of 
+> smartctl is 0 (all bits turned off). If a problem occurs, or an error, potential error, or fault is detected, then 
+> a non-zero status is returned. In this case, the eight different bits in the return value have the following meanings 
+> for ATA disks; some of these values may also be returned for SCSI disks.
+> 
+> source: http://www.linuxguide.it/command_line/linux-manpage/do.php?file=smartctl#sect7
+
+
+| Exit Code (Isolated) | Binary | Problem Message |
+| --- | --- | --- |
+| 1 | Bit 0 | Command line did not parse. |
+| 2 | Bit 1 | Device open failed, or device did not return an IDENTIFY DEVICE structure. |
+| 4 | Bit 2 | Some SMART command to the disk failed, or there was a checksum error in a SMART data structure (see В´-bВ´ option above). |
+| 8 | Bit 3 | SMART status check returned “DISK FAILING". |
+| 16 | Bit 4 | We found prefail Attributes <= threshold. |
+| 32 | Bit 5 | SMART status check returned “DISK OK” but we found that some (usage or prefail) Attributes have been <= threshold at some time in the past. |
+| 64 | Bit 6 | The device error log contains records of errors. |
+| 128 | Bit 7 | The device self-test log contains records of errors. |
+
+#### Standby/Sleeping Disks
+
+Disks in Standby/Sleep can also cause `smartctl` to exit abnormally, usually with `exit code: 2`. 
+
 - https://github.com/AnalogJ/scrutiny/issues/221
 - https://github.com/AnalogJ/scrutiny/issues/157
 
 ### Volume Mount All Devices (`/dev`) - Privileged
 
+> WARNING: This is an insecure/dangerous workaround. Running Scrutiny (or any Docker image) with `--privileged` is equivalent to running it with root access. 
+
+If you have exhausted all other mechanisms to get your disks working with `smartctl` running within a container, you can try running the docker image with the following additional flags:
+
+- `--privileged` (instead of `--cap-add`) - this gives the docker container full access to your system. Scrutiny does not require this permission, however it can be helpful for `smartctl`
+- `-v /dev:/dev:ro` (instead of `--device`) - this mounts the `/dev` folder (containing all your device files) into the container, allowing `smartctl` to see your disks, exactly as if it were running on your host directly. 
+
+With this workaround your `docker run` command would look similar to the following:
+
+```bash
+docker run -it --rm -p 8080:8080 -p 8086:8086 \
+  -v `pwd`/scrutiny:/opt/scrutiny/config \
+  -v `pwd`/influxdb2:/opt/scrutiny/influxdb \
+  -v /run/udev:/run/udev:ro \
+  --privileged \
+  -v /dev:/dev \
+  --name scrutiny \
+  ghcr.io/analogj/scrutiny:master-omnibus
+```
 
 ## Scrutiny detects Failure but SMART Passed?
 
@@ -138,3 +188,17 @@ Thankfully the collector has a special `--host-id` flag (or `COLLECTOR_HOST_ID`
 
 See the [docs/INSTALL_HUB_SPOKE.md](/docs/INSTALL_HUB_SPOKE.md) guide for more information. 
 
+## Collector DEBUG mode
+
+You can use environmental variables to enable debug logging and/or log files for the collector:
+
+```bash
+DEBUG=true
+COLLECTOR_LOG_FILE=/tmp/collector.log
+```
+
+Or if you're not using docker, you can pass CLI arguments to the collector during startup:
+
+```bash
+scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
+```

docs/TROUBLESHOOTING_DOCKER.md

@@ -0,0 +1,25 @@
+# Docker Images `master-omnibus` vs `latest`
+
+> TL;DR; The `master-omnibus` and `latest` tags are almost semantically identical, as I follow a `golden master` 
+development process. However if you want to ensure you're only using the latest release, you can change to `latest`
+
+The CI script used to orchestrate the docker image builds can be found here: https://github.com/AnalogJ/scrutiny/blob/master/.github/workflows/docker-build.yaml#L166-L184
+
+In general Scrutiny follows a `golden master` development process, which means that the `master` branch is not directly updated (unless its for documentation changes), 
+instead development is done in a feature branch, or committed to the `beta` branch. 
+
+As development progresses, and we're satisfied that a feature is complete, and the quality is acceptable, 
+I merge the changes to `master` and trigger the creation of a new release -- ie, when master is updated, a new release
+is almost immediately created (and tagged with `latest`)
+
+So changing from `master-omnibus -> latest` will be the same thing for all intents and purposes. 
+
+Having said that -- the one key difference is the `automated cron builds` that run on the `master` and `beta` branches. 
+They trigger a `nightly` build, even if nothing has changed on the branch. This has a couple of benefits, but one is to 
+ensure that there's no broken external dependencies in our (unchanged) code. 
+
+However, as everyone unfortunately found out recently, I had an error in my CI script, which caused failures to be 
+ignored -- https://github.com/AnalogJ/scrutiny/issues/287. That has since been fixed.
+
+Hope that gives you an understanding for how everything is wired up. 
+

docs/TROUBLESHOOTING_INFLUXDB.md

@@ -54,6 +54,15 @@ time="2022-05-13T14:38:05Z" level=info msg="Successfully connected to scrutiny s
 panic: a username and password is required for a setup
 ```
 
+or 
+
+```
+Start the scrutiny server
+time="2022-06-11T10:35:04-04:00" level=info msg="Trying to connect to scrutiny sqlite db: \n"
+time="2022-06-11T10:35:04-04:00" level=info msg="Successfully connected to scrutiny sqlite db: \n"
+panic: failed to check influxdb setup status - parse "://:": missing protocol scheme
+```
+
 As discussed in [#248](https://github.com/AnalogJ/scrutiny/issues/248) and [#234](https://github.com/AnalogJ/scrutiny/issues/234),
 this usually related to either:
 

example.collector.yaml

@@ -53,6 +53,13 @@ devices:
 #      - 3ware,3
 #      - 3ware,4
 #      - 3ware,5
+#
+#  # example to show how to override the smartctl command args (per device), see below for how to override these globally.
+#  - device: /dev/sda
+#    commands:
+#      metrics_info_args: '--info --json -T permissive' # used to determine device unique ID & register device with Scrutiny
+#      metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device.
+
 
 #log:
 #  file: '' #absolute or relative paths allowed, eg. web.log
@@ -64,6 +71,12 @@ devices:
 # if you need to use a custom base path (for a reverse proxy), you can add a suffix to the endpoint.
 #  See docs/TROUBLESHOOTING_REVERSE_PROXY.md for more info,
 
+# example to show how to override the smartctl command args globally
+#commands:
+#  metrics_scan_args: '--scan --json' # used to detect devices
+#  metrics_info_args: '--info --json' # used to determine device unique ID & register device with Scrutiny
+#  metrics_smart_args: '--xall --json' # used to retrieve smart data for each device.
+
 
 ########################################################################################################################
 # FEATURES COMING SOON

webapp/backend/pkg/constants.go

@@ -4,25 +4,34 @@ const DeviceProtocolAta = "ATA"
 const DeviceProtocolScsi = "SCSI"
 const DeviceProtocolNvme = "NVMe"
 
-const SmartAttributeStatusPassed = 0
+type AttributeStatus uint8
-const SmartAttributeStatusFailed = 1
-const SmartAttributeStatusWarning = 2
-
-const SmartWhenFailedFailingNow = "FAILING_NOW"
-const SmartWhenFailedInThePast = "IN_THE_PAST"
-
-//const SmartStatusPassed = "passed"
-//const SmartStatusFailed = "failed"
-
-type DeviceStatus int
 
 const (
-	DeviceStatusPassed         DeviceStatus = 0
+	// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
-	DeviceStatusFailedSmart    DeviceStatus = iota
+	AttributeStatusPassed          AttributeStatus = 0
-	DeviceStatusFailedScrutiny DeviceStatus = iota
+	AttributeStatusFailedSmart     AttributeStatus = 1
+	AttributeStatusWarningScrutiny AttributeStatus = 2
+	AttributeStatusFailedScrutiny  AttributeStatus = 4
 )
 
-func Set(b, flag DeviceStatus) DeviceStatus    { return b | flag }
+const AttributeWhenFailedFailingNow = "FAILING_NOW"
-func Clear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
+const AttributeWhenFailedInThePast = "IN_THE_PAST"
-func Toggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
+
-func Has(b, flag DeviceStatus) bool            { return b&flag != 0 }
+func AttributeStatusSet(b, flag AttributeStatus) AttributeStatus    { return b | flag }
+func AttributeStatusClear(b, flag AttributeStatus) AttributeStatus  { return b &^ flag }
+func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ flag }
+func AttributeStatusHas(b, flag AttributeStatus) bool               { return b&flag != 0 }
+
+type DeviceStatus uint8
+
+const (
+	// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
+	DeviceStatusPassed         DeviceStatus = 0
+	DeviceStatusFailedSmart    DeviceStatus = 1
+	DeviceStatusFailedScrutiny DeviceStatus = 2
+)
+
+func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus    { return b | flag }
+func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
+func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
+func DeviceStatusHas(b, flag DeviceStatus) bool            { return b&flag != 0 }

webapp/backend/pkg/database/helpers.go

@@ -0,0 +1,12 @@
+package database
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"sort"
+)
+
+func sortSmartMeasurementsDesc(smartResults []measurements.Smart) {
+	sort.SliceStable(smartResults, func(i, j int) bool {
+		return smartResults[i].Date.After(smartResults[j].Date)
+	})
+}

webapp/backend/pkg/database/helpers_test.go

@@ -0,0 +1,30 @@
+package database
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func Test_sortSmartMeasurementsDesc_LatestFirst(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	smartResults := []measurements.Smart{
+		{
+			Date: timeNow.AddDate(0, 0, -2),
+		},
+		{
+			Date: timeNow,
+		},
+		{
+			Date: timeNow.AddDate(0, 0, -1),
+		},
+	}
+
+	//test
+	sortSmartMeasurementsDesc(smartResults)
+
+	//assert
+	require.Equal(t, smartResults[0].Date, timeNow)
+}

webapp/backend/pkg/database/scrutiny_repository_device.go

@@ -58,7 +58,7 @@ func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string
 		return device, fmt.Errorf("Could not get device from DB: %v", err)
 	}
 
-	device.DeviceStatus = pkg.Set(device.DeviceStatus, status)
+	device.DeviceStatus = pkg.DeviceStatusSet(device.DeviceStatus, status)
 	return device, sr.gormClient.Model(&device).Updates(device).Error
 }
 

webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go

@@ -29,6 +29,7 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin
 	return deviceSmartData, sr.saveDatapoint(sr.influxWriteApi, "smart", tags, fields, deviceSmartData.Date, ctx)
 }
 
+// GetSmartAttributeHistory MUST return in sorted order, where newest entries are at the beginning of the list, and oldest are at the end.
 func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) {
 	// Get SMartResults from InfluxDB
 
@@ -64,6 +65,9 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn
 		return nil, err
 	}
 
+	//we have to sort the smartResults again, because the `union` command will return multiple 'tables' and only sort the records in each table.
+	sortSmartMeasurementsDesc(smartResults)
+
 	return smartResults, nil
 
 	//if err := device.SquashHistory(); err != nil {

webapp/backend/pkg/database/scrutiny_repository_tasks.go

@@ -109,7 +109,7 @@ func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
   |> toInt()
 
   temp_data
-  |> aggregateWindow(fn: mean, every: aggWindow)
+  |> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
   |> to(bucket: destBucket, org: destOrg)
 		`,
 		sourceBucket,

webapp/backend/pkg/models/device.go

@@ -21,9 +21,9 @@ type Device struct {
 	WWN string `json:"wwn" gorm:"primary_key"`
 
 	DeviceName     string `json:"device_name"`
-	DeviceUUID	   string `json:"device_uuid"`
+	DeviceUUID     string `json:"device_uuid"`
-	DeviceSerialID	   string `json:"device_serial_id"`
+	DeviceSerialID string `json:"device_serial_id"`
-	DeviceLabel	   string `json:"device_label"`
+	DeviceLabel    string `json:"device_label"`
 
 	Manufacturer   string `json:"manufacturer"`
 	ModelName      string `json:"model_name"`
@@ -166,7 +166,7 @@ func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
 	dv.DeviceProtocol = info.Device.Protocol
 
 	if !info.SmartStatus.Passed {
-		dv.DeviceStatus = pkg.Set(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
+		dv.DeviceStatus = pkg.DeviceStatusSet(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
 	}
 
 	return nil

webapp/backend/pkg/models/measurements/smart.go

@@ -110,7 +110,7 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er
 	sm.PowerCycleCount = info.PowerCycleCount
 	sm.PowerOnHours = info.PowerOnTime.Hours
 	if !info.SmartStatus.Passed {
-		sm.Status = pkg.DeviceStatusFailedSmart
+		sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedSmart)
 	}
 
 	sm.DeviceProtocol = info.Device.Protocol
@@ -148,8 +148,9 @@ func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTa
 		}
 		attrModel.PopulateAttributeStatus()
 		sm.Attributes[strconv.Itoa(collectorAttr.ID)] = &attrModel
-		if attrModel.Status == pkg.SmartAttributeStatusFailed {
+
-			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		if pkg.AttributeStatusHas(attrModel.Status, pkg.AttributeStatusFailedScrutiny) {
+			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
 		}
 	}
 }
@@ -178,8 +179,8 @@ func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.Nv
 
 	//find analyzed attribute status
 	for _, val := range sm.Attributes {
-		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
+		if pkg.AttributeStatusHas(val.GetStatus(), pkg.AttributeStatusFailedScrutiny) {
-			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
 		}
 	}
 }
@@ -204,8 +205,8 @@ func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog
 
 	//find analyzed attribute status
 	for _, val := range sm.Attributes {
-		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
+		if pkg.AttributeStatusHas(val.GetStatus(), pkg.AttributeStatusFailedScrutiny) {
-			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
 		}
 	}
 }

webapp/backend/pkg/models/measurements/smart_ata_attribute.go

@@ -18,13 +18,13 @@ type SmartAtaAttribute struct {
 	WhenFailed  string `json:"when_failed"`
 
 	//Generated data
-	TransformedValue int64   `json:"transformed_value"`
+	TransformedValue int64               `json:"transformed_value"`
-	Status           int64   `json:"status"`
+	Status           pkg.AttributeStatus `json:"status"`
-	StatusReason     string  `json:"status_reason,omitempty"`
+	StatusReason     string              `json:"status_reason,omitempty"`
-	FailureRate      float64 `json:"failure_rate,omitempty"`
+	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
-func (sa *SmartAtaAttribute) GetStatus() int64 {
+func (sa *SmartAtaAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }
 
@@ -43,7 +43,7 @@ func (sa *SmartAtaAttribute) Flatten() map[string]interface{} {
 
 		//Generated Data
 		fmt.Sprintf("attr.%s.transformed_value", idString): sa.TransformedValue,
-		fmt.Sprintf("attr.%s.status", idString):            sa.Status,
+		fmt.Sprintf("attr.%s.status", idString):            int64(sa.Status),
 		fmt.Sprintf("attr.%s.status_reason", idString):     sa.StatusReason,
 		fmt.Sprintf("attr.%s.failure_rate", idString):      sa.FailureRate,
 	}
@@ -77,7 +77,7 @@ func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
 	case "transformed_value":
 		sa.TransformedValue = val.(int64)
 	case "status":
-		sa.Status = val.(int64)
+		sa.Status = pkg.AttributeStatus(val.(int64))
 	case "status_reason":
 		sa.StatusReason = val.(string)
 	case "failure_rate":
@@ -89,16 +89,16 @@ func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
 //populate attribute status, using SMART Thresholds & Observed Metadata
 // Chainable
 func (sa *SmartAtaAttribute) PopulateAttributeStatus() *SmartAtaAttribute {
-	if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedFailingNow {
+	if strings.ToUpper(sa.WhenFailed) == pkg.AttributeWhenFailedFailingNow {
 		//this attribute has previously failed
-		sa.Status = pkg.SmartAttributeStatusFailed
+		sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedSmart)
-		sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
+		sa.StatusReason += "Attribute is failing manufacturer SMART threshold"
 		//if the Smart Status is failed, we should exit early, no need to look at thresholds.
 		return sa
 
-	} else if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedInThePast {
+	} else if strings.ToUpper(sa.WhenFailed) == pkg.AttributeWhenFailedInThePast {
-		sa.Status = pkg.SmartAttributeStatusWarning
+		sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
-		sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
+		sa.StatusReason += "Attribute has previously failed manufacturer SMART threshold"
 	}
 
 	if smartMetadata, ok := thresholds.AtaMetadata[sa.AttributeId]; ok {
@@ -138,16 +138,16 @@ func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttri
 
 			if smartMetadata.Critical {
 				if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = pkg.SmartAttributeStatusFailed
+					sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
-					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
+					sa.StatusReason += "Observed Failure Rate for Critical Attribute is greater than 10%"
 				}
 			} else {
 				if obsThresh.AnnualFailureRate >= 0.20 {
-					sa.Status = pkg.SmartAttributeStatusFailed
+					sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
+					sa.StatusReason += "Observed Failure Rate for Non-Critical Attribute is greater than 20%"
 				} else if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = pkg.SmartAttributeStatusWarning
+					sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
+					sa.StatusReason += "Observed Failure Rate for Non-Critical Attribute is greater than 10%"
 				}
 			}
 
@@ -157,7 +157,7 @@ func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttri
 	}
 	// no bucket found
 	if smartMetadata.Critical {
-		sa.Status = pkg.SmartAttributeStatusWarning
+		sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
 		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
 	}
 

webapp/backend/pkg/models/measurements/smart_attribute.go

@@ -1,7 +1,9 @@
 package measurements
 
+import "github.com/analogj/scrutiny/webapp/backend/pkg"
+
 type SmartAttribute interface {
 	Flatten() (fields map[string]interface{})
 	Inflate(key string, val interface{})
-	GetStatus() int64
+	GetStatus() pkg.AttributeStatus
 }

webapp/backend/pkg/models/measurements/smart_nvme_attribute.go

@@ -12,13 +12,13 @@ type SmartNvmeAttribute struct {
 	Value       int64  `json:"value"`
 	Threshold   int64  `json:"thresh"`
 
-	TransformedValue int64   `json:"transformed_value"`
+	TransformedValue int64               `json:"transformed_value"`
-	Status           int64   `json:"status"`
+	Status           pkg.AttributeStatus `json:"status"`
-	StatusReason     string  `json:"status_reason,omitempty"`
+	StatusReason     string              `json:"status_reason,omitempty"`
-	FailureRate      float64 `json:"failure_rate,omitempty"`
+	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
-func (sa *SmartNvmeAttribute) GetStatus() int64 {
+func (sa *SmartNvmeAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }
 
@@ -30,7 +30,7 @@ func (sa *SmartNvmeAttribute) Flatten() map[string]interface{} {
 
 		//Generated Data
 		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
-		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            int64(sa.Status),
 		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
 		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
 	}
@@ -54,7 +54,7 @@ func (sa *SmartNvmeAttribute) Inflate(key string, val interface{}) {
 	case "transformed_value":
 		sa.TransformedValue = val.(int64)
 	case "status":
-		sa.Status = val.(int64)
+		sa.Status = pkg.AttributeStatus(val.(int64))
 	case "status_reason":
 		sa.StatusReason = val.(string)
 	case "failure_rate":
@@ -72,8 +72,8 @@ func (sa *SmartNvmeAttribute) PopulateAttributeStatus() *SmartNvmeAttribute {
 			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
 			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
 				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = pkg.SmartAttributeStatusFailed
+				sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
-				sa.StatusReason = "Attribute is failing recommended SMART threshold"
+				sa.StatusReason += "Attribute is failing recommended SMART threshold"
 			}
 		}
 	}

webapp/backend/pkg/models/measurements/smart_scsci_attribute.go

@@ -12,13 +12,13 @@ type SmartScsiAttribute struct {
 	Value       int64  `json:"value"`
 	Threshold   int64  `json:"thresh"`
 
-	TransformedValue int64   `json:"transformed_value"`
+	TransformedValue int64               `json:"transformed_value"`
-	Status           int64   `json:"status"`
+	Status           pkg.AttributeStatus `json:"status"`
-	StatusReason     string  `json:"status_reason,omitempty"`
+	StatusReason     string              `json:"status_reason,omitempty"`
-	FailureRate      float64 `json:"failure_rate,omitempty"`
+	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
-func (sa *SmartScsiAttribute) GetStatus() int64 {
+func (sa *SmartScsiAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }
 
@@ -30,7 +30,7 @@ func (sa *SmartScsiAttribute) Flatten() map[string]interface{} {
 
 		//Generated Data
 		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
-		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            int64(sa.Status),
 		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
 		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
 	}
@@ -54,7 +54,7 @@ func (sa *SmartScsiAttribute) Inflate(key string, val interface{}) {
 	case "transformed_value":
 		sa.TransformedValue = val.(int64)
 	case "status":
-		sa.Status = val.(int64)
+		sa.Status = pkg.AttributeStatus(val.(int64))
 	case "status_reason":
 		sa.StatusReason = val.(string)
 	case "failure_rate":
@@ -73,7 +73,7 @@ func (sa *SmartScsiAttribute) PopulateAttributeStatus() *SmartScsiAttribute {
 			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
 			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
 				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = pkg.SmartAttributeStatusFailed
+				sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
 				sa.StatusReason = "Attribute is failing recommended SMART threshold"
 			}
 		}

webapp/backend/pkg/models/measurements/smart_test.go

@@ -328,9 +328,12 @@ func TestFromCollectorSmartInfo(t *testing.T) {
 	require.Equal(t, 18, len(smartMdl.Attributes))
 
 	//check that temperature was correctly parsed
-
 	require.Equal(t, int64(163210330144), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).RawValue)
 	require.Equal(t, int64(32), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).TransformedValue)
+
+	//ensure that Scrutiny warning for a non critical attribute does not set device status to failed.
+	require.Equal(t, pkg.AttributeStatusWarningScrutiny, smartMdl.Attributes["3"].GetStatus())
+
 }
 
 func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) {
@@ -402,7 +405,7 @@ func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
 	require.Equal(t, pkg.DeviceStatusFailedScrutiny, smartMdl.Status)
-	require.Equal(t, int64(pkg.SmartAttributeStatusFailed), smartMdl.Attributes["199"].GetStatus(),
+	require.Equal(t, pkg.AttributeStatusFailedScrutiny, smartMdl.Attributes["199"].GetStatus(),
 		"scrutiny should detect that %d failed (status: %d, %s)",
 		smartMdl.Attributes["199"].(*measurements.SmartAtaAttribute).AttributeId,
 		smartMdl.Attributes["199"].GetStatus(), smartMdl.Attributes["199"].(*measurements.SmartAtaAttribute).StatusReason,
@@ -435,7 +438,7 @@ func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
 	require.Equal(t, pkg.DeviceStatusFailedScrutiny, smartMdl.Status)
-	require.Equal(t, int64(pkg.SmartAttributeStatusFailed), smartMdl.Attributes["media_errors"].GetStatus(),
+	require.Equal(t, pkg.AttributeStatusFailedScrutiny, smartMdl.Attributes["media_errors"].GetStatus(),
 		"scrutiny should detect that %s failed (status: %d, %s)",
 		smartMdl.Attributes["media_errors"].(*measurements.SmartNvmeAttribute).AttributeId,
 		smartMdl.Attributes["media_errors"].GetStatus(),

webapp/backend/pkg/thresholds/scsi_attribute_metadata.go

@@ -19,7 +19,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
-		Description: "",
+		Description: "The grown defect count shows the amount of swapped (defective) blocks since the drive was shipped by it's vendor. Each additional defective block increases the count by one.",
 	},
 	"read_errors_corrected_by_eccfast": {
 		ID:          "read_errors_corrected_by_eccfast",
@@ -27,7 +27,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
-		Description: "",
+		Description: "An error correction was applied to get perfect data (a.k.a. ECC on-the-fly). \"Without substantial delay\" means the correction did not postpone reading of later sectors (e.g. a revolution was not lost). The counter is incremented once for each logical block that requires correction. Two different blocks corrected during the same command are counted as two events.",
 	},
 	"read_errors_corrected_by_eccdelayed": {
 		ID:          "read_errors_corrected_by_eccdelayed",
@@ -35,7 +35,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
-		Description: "",
+		Description: "An error code or algorithm (e.g. ECC, checksum) is applied in order to get perfect data with substantial delay. \"With possible delay\" means the correction took longer than a sector time so that reading/writing of subsequent sectors was delayed (e.g. a lost revolution). The counter is incremented once for each logical block that requires correction. A block with a double error that is correctable counts as one event and two different blocks corrected during the same command count as two events. ",
 	},
 	"read_errors_corrected_by_rereads_rewrites": {
 		ID:          "read_errors_corrected_by_rereads_rewrites",
@@ -43,7 +43,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
-		Description: "",
+		Description: "This parameter code specifies the counter counting the number of errors that are corrected by applying retries. This counts errors recovered, not the number of retries. If five retries were required to recover one block of data, the counter increments by one, not five. The counter is incremented once for each logical block that is recovered using retries. If an error is not recoverable while applying retries and is recovered by ECC, it isn't counted by this counter; it will be counted by the counter specified by parameter code 01h - Errors Corrected With Possible Delays. ",
 	},
 	"read_total_errors_corrected": {
 		ID:          "read_total_errors_corrected",
@@ -51,7 +51,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
-		Description: "",
+		Description: "This counter counts the total of parameter code errors 00h, 01h and 02h (i.e. error corrected by ECC: fast and delayed plus errors corrected by rereads and rewrites). There is no \"double counting\" of data errors among these three counters. The sum of all correctable errors can be reached by adding parameter code 01h and 02h errors, not by using this total.",
 	},
 	"read_correction_algorithm_invocations": {
 		ID:          "read_correction_algorithm_invocations",
@@ -59,7 +59,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
-		Description: "",
+		Description: "This parameter code specifies the counter that counts the total number of retries, or \"times the retry algorithm is invoked\". If after five attempts a counter 02h type error is recovered, then five is added to this counter. If three retries are required to get stable ECC syndrome before a counter 01h type error is corrected, then those three retries are also counted here. The number of retries applied to unsuccessfully recover an error (counter 06h type error) are also counted by this counter. ",
 	},
 	"read_total_uncorrected_errors": {
 		ID:          "read_total_uncorrected_errors",
@@ -67,7 +67,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
-		Description: "",
+		Description: "This parameter code specifies the counter that contains the total number of blocks for which an uncorrected data error has occurred. ",
 	},
 	"write_errors_corrected_by_eccfast": {
 		ID:          "write_errors_corrected_by_eccfast",
@@ -75,7 +75,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
-		Description: "",
+		Description: "An error correction was applied to get perfect data (a.k.a. ECC on-the-fly). \"Without substantial delay\" means the correction did not postpone reading of later sectors (e.g. a revolution was not lost). The counter is incremented once for each logical block that requires correction. Two different blocks corrected during the same command are counted as two events. ",
 	},
 	"write_errors_corrected_by_eccdelayed": {
 		ID:          "write_errors_corrected_by_eccdelayed",
@@ -83,7 +83,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
-		Description: "",
+		Description: "An error code or algorithm (e.g. ECC, checksum) is applied in order to get perfect data with substantial delay. \"With possible delay\" means the correction took longer than a sector time so that reading/writing of subsequent sectors was delayed (e.g. a lost revolution). The counter is incremented once for each logical block that requires correction. A block with a double error that is correctable counts as one event and two different blocks corrected during the same command count as two events. ",
 	},
 	"write_errors_corrected_by_rereads_rewrites": {
 		ID:          "write_errors_corrected_by_rereads_rewrites",
@@ -91,7 +91,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
-		Description: "",
+		Description: "This parameter code specifies the counter counting the number of errors that are corrected by applying retries. This counts errors recovered, not the number of retries. If five retries were required to recover one block of data, the counter increments by one, not five. The counter is incremented once for each logical block that is recovered using retries. If an error is not recoverable while applying retries and is recovered by ECC, it isn't counted by this counter; it will be counted by the counter specified by parameter code 01h - Errors Corrected With Possible Delays.",
 	},
 	"write_total_errors_corrected": {
 		ID:          "write_total_errors_corrected",
@@ -99,7 +99,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
-		Description: "",
+		Description: "This counter counts the total of parameter code errors 00h, 01h and 02h (i.e. error corrected by ECC: fast and delayed plus errors corrected by rereads and rewrites). There is no \"double counting\" of data errors among these three counters. The sum of all correctable errors can be reached by adding parameter code 01h and 02h errors, not by using this total.",
 	},
 	"write_correction_algorithm_invocations": {
 		ID:          "write_correction_algorithm_invocations",
@@ -107,7 +107,7 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
-		Description: "",
+		Description: "This parameter code specifies the counter that counts the total number of retries, or \"times the retry algorithm is invoked\". If after five attempts a counter 02h type error is recovered, then five is added to this counter. If three retries are required to get stable ECC syndrome before a counter 01h type error is corrected, then those three retries are also counted here. The number of retries applied to unsuccessfully recover an error (counter 06h type error) are also counted by this counter. ",
 	},
 	"write_total_uncorrected_errors": {
 		ID:          "write_total_uncorrected_errors",
@@ -115,6 +115,6 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
-		Description: "",
+		Description: " This parameter code specifies the counter that contains the total number of blocks for which an uncorrected data error has occurred.",
 	},
 }

webapp/backend/pkg/version/version.go

@@ -2,4 +2,4 @@ package version
 
 // VERSION is the app-global version string, which will be replaced with a
 // new value during packaging
-const VERSION = "0.4.8"
+const VERSION = "0.4.12"

webapp/backend/pkg/web/server.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 type AppEngine struct {
@@ -68,6 +69,11 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 }
 
 func (ae *AppEngine) Start() error {
+	//set the gin mode
+	gin.SetMode(gin.ReleaseMode)
+	if strings.ToLower(ae.Config.GetString("log.level")) == "debug" {
+		gin.SetMode(gin.DebugMode)
+	}
 
 	logger := logrus.New()
 	//set default log level

webapp/frontend/package-lock.json

@@ -29,13 +29,13 @@
                 "@fullcalendar/rrule": "4.4.0",
                 "@fullcalendar/timegrid": "4.4.0",
                 "@types/humanize-duration": "^3.18.1",
-                "apexcharts": "3.19.0",
+                "apexcharts": "3.19.2",
                 "crypto-js": "3.3.0",
                 "highlight.js": "10.0.1",
                 "humanize-duration": "^3.24.0",
                 "lodash": "4.17.15",
                 "moment": "2.24.0",
-                "ng-apexcharts": "1.2.3",
+                "ng-apexcharts": "1.5.12",
                 "ngx-markdown": "9.0.0",
                 "ngx-quill": "9.1.0",
                 "perfect-scrollbar": "1.5.0",
@@ -2964,9 +2964,9 @@
             }
         },
         "node_modules/apexcharts": {
-            "version": "3.19.0",
+            "version": "3.19.2",
-            "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.19.0.tgz",
+            "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.19.2.tgz",
-            "integrity": "sha512-fzupCGVDvOoU6kEzguLAfgRgrlHynHM5fnkkyCL85tYf9U8bw1hCijs4A+kWXurC/SNytJrArBc21kA/2wuHYg==",
+            "integrity": "sha512-hMFLRE2Lyx4WrN9pYfQLvBDcn+HOodZrqRwc+kucxM+hcUmI2NHY4z+GI14+VcSFmD4aKiMbS3z3Q2jiBxUrcg==",
             "dependencies": {
                 "svg.draggable.js": "^2.2.2",
                 "svg.easing.js": "^2.0.0",
@@ -9958,17 +9958,17 @@
             "dev": true
         },
         "node_modules/ng-apexcharts": {
-            "version": "1.2.3",
+            "version": "1.5.12",
-            "resolved": "https://registry.npmjs.org/ng-apexcharts/-/ng-apexcharts-1.2.3.tgz",
+            "resolved": "https://registry.npmjs.org/ng-apexcharts/-/ng-apexcharts-1.5.12.tgz",
-            "integrity": "sha512-4G+JRCWp8uSSBJKvYP9vKHEZIC0w6YuRLasumZS35fCCc7bzLY+L907n8khG9Xeoo4LBt7pVbmjb9P+lSWs/5g==",
+            "integrity": "sha512-k82AdWNbZs5yqGCjiX7PGS11Cy1+1Oo/RGt2lT89xReD9N9Vvo1t34p1dmzS+U6W5wOFlLEKKVLGNQqENW8cTQ==",
             "dependencies": {
-                "tslib": "^1.9.0"
+                "tslib": "^1.10.0"
             },
             "peerDependencies": {
-                "@angular/common": "^8.0.0",
+                "@angular/common": ">=9.0.0 <13.0.0",
-                "@angular/core": "^8.0.0",
+                "@angular/core": ">=9.0.0 <13.0.0",
-                "apexcharts": "^3.11.2",
+                "apexcharts": "^3.19.2",
-                "rxjs": "^6.4.0"
+                "rxjs": "^6.5.5"
             }
         },
         "node_modules/ngx-markdown": {
@@ -20708,9 +20708,9 @@
             }
         },
         "apexcharts": {
-            "version": "3.19.0",
+            "version": "3.19.2",
-            "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.19.0.tgz",
+            "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.19.2.tgz",
-            "integrity": "sha512-fzupCGVDvOoU6kEzguLAfgRgrlHynHM5fnkkyCL85tYf9U8bw1hCijs4A+kWXurC/SNytJrArBc21kA/2wuHYg==",
+            "integrity": "sha512-hMFLRE2Lyx4WrN9pYfQLvBDcn+HOodZrqRwc+kucxM+hcUmI2NHY4z+GI14+VcSFmD4aKiMbS3z3Q2jiBxUrcg==",
             "requires": {
                 "svg.draggable.js": "^2.2.2",
                 "svg.easing.js": "^2.0.0",
@@ -26486,11 +26486,11 @@
             "dev": true
         },
         "ng-apexcharts": {
-            "version": "1.2.3",
+            "version": "1.5.12",
-            "resolved": "https://registry.npmjs.org/ng-apexcharts/-/ng-apexcharts-1.2.3.tgz",
+            "resolved": "https://registry.npmjs.org/ng-apexcharts/-/ng-apexcharts-1.5.12.tgz",
-            "integrity": "sha512-4G+JRCWp8uSSBJKvYP9vKHEZIC0w6YuRLasumZS35fCCc7bzLY+L907n8khG9Xeoo4LBt7pVbmjb9P+lSWs/5g==",
+            "integrity": "sha512-k82AdWNbZs5yqGCjiX7PGS11Cy1+1Oo/RGt2lT89xReD9N9Vvo1t34p1dmzS+U6W5wOFlLEKKVLGNQqENW8cTQ==",
             "requires": {
-                "tslib": "^1.9.0"
+                "tslib": "^1.10.0"
             }
         },
         "ngx-markdown": {

webapp/frontend/package.json

@@ -40,13 +40,13 @@
         "@fullcalendar/rrule": "4.4.0",
         "@fullcalendar/timegrid": "4.4.0",
         "@types/humanize-duration": "^3.18.1",
-        "apexcharts": "3.19.0",
+        "apexcharts": "3.19.2",
         "crypto-js": "3.3.0",
         "highlight.js": "10.0.1",
         "humanize-duration": "^3.24.0",
         "lodash": "4.17.15",
         "moment": "2.24.0",
-        "ng-apexcharts": "1.2.3",
+        "ng-apexcharts": "1.5.12",
         "ngx-markdown": "9.0.0",
         "ngx-quill": "9.1.0",
         "perfect-scrollbar": "1.5.0",

webapp/frontend/src/@treo/services/config/config.service.ts

@@ -2,6 +2,7 @@ import { Inject, Injectable } from '@angular/core';
 import { BehaviorSubject, Observable } from 'rxjs';
 import * as _ from 'lodash';
 import { TREO_APP_CONFIG } from '@treo/services/config/config.constants';
+import { AppConfig } from 'app/core/config/app.config';
 
 const SCRUTINY_CONFIG_LOCAL_STORAGE_KEY = 'scrutiny';
 
@@ -12,6 +13,7 @@ export class TreoConfigService
 {
     // Private
     private _config: BehaviorSubject<any>;
+
     /**
      * Constructor
      */
@@ -20,12 +22,11 @@ export class TreoConfigService
         let currentScrutinyConfig = defaultConfig
 
         let localConfigStr = localStorage.getItem(SCRUTINY_CONFIG_LOCAL_STORAGE_KEY)
-        if(localConfigStr){
+        if (localConfigStr){
             //check localstorage for a value
             let localConfig = JSON.parse(localConfigStr)
-            currentScrutinyConfig = localConfig
+            currentScrutinyConfig = Object.assign({}, currentScrutinyConfig, localConfig) // make sure defaults are available if missing from localStorage.
         }
-
         // Set the private defaults
         this._config = new BehaviorSubject(currentScrutinyConfig);
     }
@@ -41,7 +42,7 @@ export class TreoConfigService
     set config(value: any)
     {
         // Merge the new config over to the current config
-        const config = _.merge({}, this._config.getValue(), value);
+        let config = _.merge({}, this._config.getValue(), value);
 
         //Store the config in localstorage
         localStorage.setItem(SCRUTINY_CONFIG_LOCAL_STORAGE_KEY, JSON.stringify(config));
@@ -56,6 +57,10 @@ export class TreoConfigService
         return this._config.asObservable();
     }
 
+    // -----------------------------------------------------------------------------------------------------
+    // @ Private methods
+    // -----------------------------------------------------------------------------------------------------
+
     // -----------------------------------------------------------------------------------------------------
     // @ Public methods
     // -----------------------------------------------------------------------------------------------------

webapp/frontend/src/app/core/config/app.config.ts

@@ -1,7 +1,7 @@
 import { Layout } from "app/layout/layout.types";
 
 // Theme type
-export type Theme = "light" | "dark";
+export type Theme = "light" | "dark" | "system";
 
 /**
  * AppConfig interface. Update this interface to strictly type your config

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html

@@ -2,6 +2,18 @@
 <mat-dialog-content class="mat-typography">
 
     <div class="flex flex-col p-8 pb-0 overflow-hidden">
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Dark Mode</mat-label>
+                <mat-select [(ngModel)]="theme">
+                    <mat-option value="system">System</mat-option>
+                    <mat-option value="dark">Dark</mat-option>
+                    <mat-option value="light">Light</mat-option>
+                </mat-select>
+            </mat-form-field>
+        </div>
+
         <div class="flex flex-col mt-5 gt-md:flex-row">
             <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
                 <mat-label>Display Title</mat-label>
@@ -38,24 +50,24 @@
             <mat-tab-group mat-align-tabs="start">
                 <mat-tab label="Ata">
 
-                    <div matTooltip="not yet implemented" class="gray-200 flex flex-col mt-5 gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Critical Error Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
                             <input disabled matInput [value]="'10%'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Critical Warning Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
                             <input disabled matInput>
                         </mat-form-field>
                     </div>
 
-                    <div matTooltip="not yet implemented" class="gray-200 flex flex-col gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Error Threshold</mat-label>
+                            <mat-label class="text-hint">Error Threshold</mat-label>
                             <input disabled matInput [value]="'20%'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Warning Threshold</mat-label>
+                            <mat-label class="text-hint">Warning Threshold</mat-label>
                             <input disabled matInput [value]="'10%'">
                         </mat-form-field>
                     </div>
@@ -63,26 +75,26 @@
                 </mat-tab>
                 <mat-tab label="NVMe">
 
-                    <div matTooltip="not yet implemented" class="gray-200 flex flex-col mt-5 gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Critical Error Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
                             <input disabled matInput [value]="'enabled'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Critical Warning Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
                             <input disabled matInput>
                         </mat-form-field>
                     </div>
 
                 </mat-tab>
                 <mat-tab label="SCSI">
-                    <div matTooltip="not yet implemented" class="gray-200 flex flex-col mt-5 gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Critical Error Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
                             <input disabled matInput [value]="'enabled'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Critical Warning Threshold</mat-label>
+                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
                             <input disabled matInput>
                         </mat-form-field>
                     </div>

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts

@@ -13,7 +13,8 @@ export class DashboardSettingsComponent implements OnInit {
 
     dashboardDisplay: string;
     dashboardSort: string;
-    temperatureUnit: string
+    temperatureUnit: string;
+    theme: string;
 
     // Private
     private _unsubscribeAll: Subject<any>;
@@ -35,14 +36,20 @@ export class DashboardSettingsComponent implements OnInit {
               this.dashboardDisplay = config.dashboardDisplay;
               this.dashboardSort = config.dashboardSort;
               this.temperatureUnit = config.temperatureUnit;
+              this.theme = config.theme;
+
           });
+
   }
 
   saveSettings(): void {
+
+
         const newSettings = {
             dashboardDisplay: this.dashboardDisplay,
             dashboardSort: this.dashboardSort,
             temperatureUnit: this.temperatureUnit,
+            theme: this.theme
         }
         this._configService.config = newSettings
         console.log(`Saved Settings: ${JSON.stringify(newSettings)}`)

webapp/frontend/src/app/layout/layout.component.ts

@@ -23,6 +23,7 @@ export class LayoutComponent implements OnInit, OnDestroy
 
     // Private
     private _unsubscribeAll: Subject<any>;
+    private systemPrefersDark: boolean;
 
     /**
      * Constructor
@@ -43,6 +44,9 @@ export class LayoutComponent implements OnInit, OnDestroy
     {
         // Set the private defaults
         this._unsubscribeAll = new Subject();
+
+        this.systemPrefersDark = window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches;
+
     }
 
     // -----------------------------------------------------------------------------------------------------
@@ -66,7 +70,7 @@ export class LayoutComponent implements OnInit, OnDestroy
                 this.theme = config.theme;
 
                 // Update the selected theme class name on body
-                const themeName = 'treo-theme-' + config.theme;
+                const themeName = 'treo-theme-' + this.determineTheme(config);
                 this._document.body.classList.forEach((className) => {
                     if ( className.startsWith('treo-theme-') && className !== themeName )
                     {
@@ -105,6 +109,17 @@ export class LayoutComponent implements OnInit, OnDestroy
     // @ Private methods
     // -----------------------------------------------------------------------------------------------------
 
+    /**
+     * Checks if theme should be set to dark based on config & system settings
+     */
+    private determineTheme(config:AppConfig): string {
+        if (config.theme === 'system') {
+            return this.systemPrefersDark ? 'dark' : 'light'
+        } else {
+            return config.theme
+        }
+    }
+
     /**
      * Update the selected layout
      */

webapp/frontend/src/app/modules/detail/detail.component.html

@@ -71,6 +71,16 @@
                         <div>{{device?.host_id}}</div>
                         <div class="text-secondary text-md">Host ID</div>
                     </div>
+
+                    <div *ngIf="device?.device_uuid" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.device_uuid}}</div>
+                        <div class="text-secondary text-md">Device UUID</div>
+                    </div>
+                    <div *ngIf="device?.device_label" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.device_label}}</div>
+                        <div class="text-secondary text-md">Device Label</div>
+                    </div>
+
                     <div *ngIf="device?.device_type && device?.device_type != 'ata' && device?.device_type != 'scsi'" class="my-2 col-span-2 lt-md:col-span-1">
                         <div>{{device?.device_type | uppercase}}</div>
                         <div class="text-secondary text-md">Device Type</div>
@@ -133,6 +143,7 @@
                     <table class="w-full bg-transparent"
                            mat-table
                            matSort
+                           multiTemplateDataRows
                            [dataSource]="smartAttributeDataSource"
                            [trackBy]="trackByFn"
                            #smartAttributeTable>
@@ -193,7 +204,7 @@
                             </th>
                             <td mat-cell
                                 *matCellDef="let attribute">
-                                <span class="pr-6 whitespace-no-wrap" matTooltip="{{getAttributeDescription(attribute)}}">
+                                <span class="pr-6 whitespace-no-wrap" matTooltip="click for more details.">
                                     {{getAttributeName(attribute)}} <mat-icon *ngIf="getAttributeDescription(attribute)" class="icon-size-10" [svgIcon]="'info'"></mat-icon>
                                 </span>
                             </td>
@@ -314,6 +325,72 @@
                             </td>
                         </ng-container>
 
+
+                        <!-- Expanded Content Column - The detail row is made up of this one column that spans across all columns -->
+                        <ng-container matColumnDef="expandedDetail">
+                            <td mat-cell *matCellDef="let attribute" [attr.colspan]="smartAttributeTableColumns.length">
+
+
+
+                                <div class="attribute-detail"
+                                     [@detailExpand]="attribute == expandedAttribute ? 'expanded' : 'collapsed'">
+
+                                    <div class="flex flex-auto w-1/3 min-w-80 py-4">
+                                        <div class="flex flex-col flex-auto justify-end text-md pb-3">
+                                            {{getAttributeDescription(attribute)}}
+                                        </div>
+                                    </div>
+                                    <div class="flex flex-auto w-2/3 min-w-80">
+                                        <div class="flex flex-col flex-auto justify-end text-md px-6 pb-3">
+                                            <div class="flex items-center justify-between py-3 border-b last:border-b-0 ng-star-inserted">
+                                                <div class="flex items-center w-1/4">Type</div>
+                                                <div class="flex items-center w-1/4">Value</div>
+                                                <div class="flex items-center w-1/4">Worst/Thresh</div>
+                                                <div class="flex items-center w-1/4">Failure %</div>
+                                            </div>
+
+                                            <div class="flex items-center justify-between py-3 border-b last:border-b-0 ng-star-inserted">
+                                                <div class="flex items-center w-1/4">
+                                                    <div class="flex-shrink-0 w-2 h-2 mr-3 rounded-full"
+                                                         [ngClass]="{'bg-red': getAttributeScrutinyStatusName(attribute.status) === 'failed',
+                                                          'bg-green': getAttributeScrutinyStatusName(attribute.status) === 'passed',
+                                                          'bg-yellow': getAttributeScrutinyStatusName(attribute.status) === 'warn'}"></div>
+                                                    <div class="truncate">Scrutiny</div>
+                                                </div>
+                                                <div class="w-1/4 items-center font-medium">{{getAttributeValue(attribute)}}</div>
+                                                <div class="w-1/4 items-center text-secondary">--</div>
+                                                <div class="w-1/4 items-center text-secondary">{{(attribute.failure_rate | percent) || '--'}}</div>
+                                            </div>
+
+                                            <div class="flex items-center justify-between py-3 border-b last:border-b-0 ng-star-inserted">
+                                                <div class="flex items-center w-1/4">
+                                                    <div class="flex-shrink-0 w-2 h-2 mr-3 rounded-full"
+                                                         [ngClass]="{'bg-red': getAttributeSmartStatusName(attribute.status) === 'failed',
+                                                          'bg-green': getAttributeSmartStatusName(attribute.status) === 'passed'}"
+                                                    ></div>
+                                                    <div class="truncate">Normalized</div>
+                                                </div>
+                                                <div class="w-1/4 items-center font-medium">{{attribute.value}}</div>
+                                                <div class="w-1/4 items-center text-secondary">{{getAttributeWorst(attribute) || '--' }}/{{getAttributeThreshold(attribute)}}</div>
+                                                <div class="w-1/4 items-center text-secondary">--</div>
+                                            </div>
+
+                                            <div class="flex items-center justify-between py-3 border-b last:border-b-0 ng-star-inserted">
+                                                <div class="flex items-center w-1/4">
+                                                    <div class="flex-shrink-0 w-2 h-2 mr-3 rounded-full"></div>
+                                                    <div class="truncate">Raw</div>
+                                                </div>
+                                                <div class="w-1/4 items-center font-medium">{{attribute.raw_value}}</div>
+                                                <div class="w-1/4 items-center text-secondary">--</div>
+                                                <div class="w-1/4 items-center text-secondary">--</div>
+                                            </div>
+                                        </div>
+                                    </div>
+                                </div>
+                            </td>
+                        </ng-container>
+
+
                         <!-- Footer -->
                         <ng-container matColumnDef="recentOrdersTableFooter">
                             <td class="px-3 border-none"
@@ -334,7 +411,10 @@
                         <tr class="attribute-row h-16"
                             mat-row
                             [ngClass]="{'yellow-50': getAttributeCritical(row)}"
+                            [class.attribute-expanded-row]="expandedAttribute === row"
+                            (click)="expandedAttribute = expandedAttribute === row ? null : row"
                             *matRowDef="let row; columns: smartAttributeTableColumns;"></tr>
+                        <tr mat-row *matRowDef="let row; columns: ['expandedDetail']" class="attribute-detail-row"></tr>
                         <tr class="h-16"
                             mat-footer-row
                             *matFooterRowDef="['recentOrdersTableFooter']"></tr>

webapp/frontend/src/app/modules/detail/detail.component.scss

@@ -1,7 +1,6 @@
 @import 'treo';
 
 detail {
-
 }
 
 // -----------------------------------------------------------------------------------------------------
@@ -20,5 +19,35 @@ detail {
     }
 
 
-
+}
+
+//table {
+//    width: 100%;
+//}
+
+$primary: map-get($theme, primary);
+$is-dark: map-get($theme, is-dark);
+tr.attribute-detail-row {
+    height: 0;
+}
+
+//tr.attribute-row:not(.attribute-expanded-row):hover {
+//    @if ($is-dark) {
+//        background: rgba(0, 0, 0, 0.05);
+//    } @else {
+//        background: map-get($primary, 50);
+//    }
+//}
+
+tr.attribute-row:not(.attribute-expanded-row):active {
+    background: #efefef;
+}
+
+.attribute-row td {
+    border-bottom-width: 0;
+}
+
+.attribute-detail {
+    overflow: hidden;
+    display: flex;
 }

webapp/frontend/src/app/modules/detail/detail.component.ts

@@ -11,11 +11,28 @@ import {MatDialog} from "@angular/material/dialog";
 import humanizeDuration from 'humanize-duration';
 import {TreoConfigService} from "../../../@treo/services/config";
 import {AppConfig} from "../../core/config/app.config";
+import {animate, state, style, transition, trigger} from '@angular/animations';
+import {formatDate} from "@angular/common";
+import { LOCALE_ID, Inject } from '@angular/core';
+
+// from Constants.go - these must match
+const AttributeStatusPassed = 0
+const AttributeStatusFailedSmart = 1
+const AttributeStatusWarningScrutiny = 2
+const AttributeStatusFailedScrutiny = 4
+
 
 @Component({
   selector: 'detail',
   templateUrl: './detail.component.html',
-  styleUrls: ['./detail.component.scss']
+  styleUrls: ['./detail.component.scss'],
+    animations: [
+        trigger('detailExpand', [
+            state('collapsed', style({height: '0px', minHeight: '0'})),
+            state('expanded', style({height: '*'})),
+            transition('expanded <=> collapsed', animate('225ms cubic-bezier(0.4, 0.0, 0.2, 1)')),
+        ]),
+    ],
 })
 
 export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
@@ -24,6 +41,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     onlyCritical: boolean = true;
     // data: any;
+    expandedAttribute: any | null;
 
     metadata: any;
     device: any;
@@ -33,12 +51,12 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
     smartAttributeDataSource: MatTableDataSource<any>;
     smartAttributeTableColumns: string[];
 
-
     @ViewChild('smartAttributeTable', {read: MatSort})
     smartAttributeTableMatSort: MatSort;
 
     // Private
     private _unsubscribeAll: Subject<any>;
+    private systemPrefersDark: boolean;
 
     /**
      * Constructor
@@ -49,7 +67,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
         private _detailService: DetailService,
         public dialog: MatDialog,
         private _configService: TreoConfigService,
-
+        @Inject(LOCALE_ID) public locale: string
 
     )
     {
@@ -60,6 +78,9 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
         this.smartAttributeDataSource = new MatTableDataSource();
         // this.recentTransactionsTableColumns = ['status', 'id', 'name', 'value', 'worst', 'thresh'];
         this.smartAttributeTableColumns = ['status', 'id', 'name', 'value', 'worst', 'thresh','ideal', 'failure', 'history'];
+
+        this.systemPrefersDark = window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches;
+
     }
 
     // -----------------------------------------------------------------------------------------------------
@@ -121,25 +142,51 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
     // -----------------------------------------------------------------------------------------------------
     // @ Private methods
     // -----------------------------------------------------------------------------------------------------
-    getAttributeStatusName(attribute_status){
+
-        if(attribute_status == 0){
+    getAttributeStatusName(attributeStatus: number): string {
-            return "passed"
+        // tslint:disable:no-bitwise
-        } else if (attribute_status == 1){
+
-            return "failed"
+        if(attributeStatus === AttributeStatusPassed){
-        } else if (attribute_status == 2){
+            return 'passed'
-            return "warn"
+
+        } else if ((attributeStatus & AttributeStatusFailedScrutiny) !== 0 || (attributeStatus & AttributeStatusFailedSmart) !== 0 ){
+            return 'failed'
+        } else if ((attributeStatus & AttributeStatusWarningScrutiny) !== 0){
+            return 'warn'
         }
-        return
+        return ''
+        // tslint:enable:no-bitwise
+    }
+    getAttributeScrutinyStatusName(attributeStatus: number): string {
+        // tslint:disable:no-bitwise
+        if ((attributeStatus & AttributeStatusFailedScrutiny) !== 0){
+            return 'failed'
+        } else if ((attributeStatus & AttributeStatusWarningScrutiny) !== 0){
+            return 'warn'
+        } else {
+            return 'passed'
+        }
+        // tslint:enable:no-bitwise
     }
 
-    getAttributeName(attribute_data){
+    getAttributeSmartStatusName(attributeStatus: number): string {
+        // tslint:disable:no-bitwise
+        if ((attributeStatus & AttributeStatusFailedSmart) !== 0){
+            return 'failed'
+        } else {
+            return 'passed'
+        }
+        // tslint:enable:no-bitwise
+    }
+
+
+    getAttributeName(attribute_data): string {
         let attribute_metadata = this.metadata[attribute_data.attribute_id]
         if(!attribute_metadata){
             return 'Unknown Attribute Name'
         } else {
             return attribute_metadata.display_name
         }
-        return
     }
     getAttributeDescription(attribute_data){
         let attribute_metadata = this.metadata[attribute_data.attribute_id]
@@ -261,7 +308,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
         } else {
             //ATA
             attributes = latest_smart_result.attrs
-            this.smartAttributeTableColumns = ['status', 'id', 'name', 'value', 'worst', 'thresh','ideal', 'failure', 'history'];
+            this.smartAttributeTableColumns = ['status', 'id', 'name', 'value', 'thresh','ideal', 'failure', 'history'];
         }
 
         for(const attrId in attributes){
@@ -273,7 +320,21 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
                 var attrHistory = []
                 for (let smart_result of smart_results){
-                    attrHistory.push(this.getAttributeValue(smart_result.attrs[attrId]))
+                    // attrHistory.push(this.getAttributeValue(smart_result.attrs[attrId]))
+
+                    const chartDatapoint = {
+                        x: formatDate(smart_result.date, 'MMMM dd, yyyy - HH:mm', this.locale),
+                        y: this.getAttributeValue(smart_result.attrs[attrId])
+                    }
+                    const attributeStatusName = this.getAttributeStatusName(smart_result.attrs[attrId].status)
+                    if(attributeStatusName === 'failed') {
+                        chartDatapoint['strokeColor'] = '#F05252'
+                        chartDatapoint['fillColor'] = '#F05252'
+                    } else if (attributeStatusName === 'warn'){
+                        chartDatapoint['strokeColor'] = '#C27803'
+                        chartDatapoint['fillColor'] = '#C27803'
+                    }
+                    attrHistory.push(chartDatapoint)
                 }
 
                 // var rawHistory = (attr.history || []).map(hist_attr => this.getAttributeValue(hist_attr)).reverse()
@@ -316,12 +377,17 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
                     enabled: false
                 }
             },
+            // theme:{
+            //     // @ts-ignore
+            //     // mode:
+            //     mode: 'dark',
+            // },
             tooltip: {
                 fixed: {
                     enabled: false
                 },
                 x: {
-                    show: false
+                    show: true
                 },
                 y: {
                     title: {
@@ -332,7 +398,9 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
                 },
                 marker: {
                     show: false
-                }
+                },
+                theme: this.determineTheme(this.config)
+
             },
             stroke: {
                 width: 2,
@@ -341,6 +409,13 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
         };
     }
 
+    private determineTheme(config:AppConfig): string {
+        if (config.theme === 'system') {
+            return this.systemPrefersDark ? 'dark' : 'light'
+        } else {
+            return config.theme
+        }
+    }
     // -----------------------------------------------------------------------------------------------------
     // @ Public methods
     // -----------------------------------------------------------------------------------------------------

webapp/frontend/src/styles/styles.scss

@@ -4,3 +4,17 @@
 // @ Styles from this file will override anything from 'vendors.scss' file allowing customizations and
 //  modifications of third party libraries.
 // -----------------------------------------------------------------------------------------------------
+
+.treo-theme-dark {
+    .yellow-50 {
+        background-color: #242b38 !important;
+
+        .mat-icon {
+            color: #0694a2 !important;
+        }
+
+        .text-secondary {
+            color: #0694a2 !important
+        }
+    }
+}