(v0.8.0) Automated packaging of release by Packagr

Use exec on scrutiny-collector cron to ensure signal handling

.github/workflows/docker-build.yaml

@@ -74,13 +74,6 @@ jobs:
         uses: actions/checkout@v2
       - name: "Populate frontend version information"
         run: "cd webapp/frontend && ./git.version.sh"
-      - name: "Install Node"
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16
-      - name: "Generate frontend"
-        run: |
-          make binary-frontend && echo "print contents of ./dist" && ls -alt ./dist
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
         with:
@@ -132,13 +125,6 @@ jobs:
         uses: actions/checkout@v2
       - name: "Populate frontend version information"
         run: "cd webapp/frontend && ./git.version.sh"
-      - name: "Install Node"
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16
-      - name: "Generate frontend"
-        run: |
-          make binary-frontend && echo "print contents of ./dist" && ls -alt ./dist
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
         with:

.github/workflows/docker-nightly.yaml

@@ -19,13 +19,6 @@ jobs:
         uses: actions/checkout@v2
       - name: "Populate frontend version information"
         run: "cd webapp/frontend && ./git.version.sh"
-      - name: "Install Node"
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16
-      - name: "Generate frontend"
-        run: |
-          make binary-frontend && echo "print contents of ./dist" && ls -alt ./dist
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
         with:

.gitignore

@@ -65,4 +65,6 @@ scrutiny_test.db
 scrutiny.yaml
 coverage.txt
 /config
-/influxdb
+/influxdb
+.angular
+web.log

Makefile

@@ -100,7 +100,7 @@ binary-frontend: export NPM_CONFIG_LOGLEVEL = warn
 binary-frontend: export NG_CLI_ANALYTICS = false
 binary-frontend:
 	cd webapp/frontend
-	npm install -g @angular/cli@9.1.4
+	npm install -g @angular/cli@v13-lts
 	mkdir -p $(CURDIR)/dist
 	npm ci
 	npm run build:prod -- --output-path=$(CURDIR)/dist

README.md

@@ -26,7 +26,7 @@ If you run a server with more than a couple of hard drives, you're probably alre
 
 > smartd is a daemon that monitors the Self-Monitoring, Analysis and Reporting Technology (SMART) system built into many ATA, IDE and SCSI-3 hard drives. The purpose of SMART is to monitor the reliability of the hard drive and predict drive failures, and to carry out different types of drive self-tests.
 
-Theses S.M.A.R.T hard drive self-tests can help you detect and replace failing hard drives before they cause permanent data loss. However, there's a couple issues with `smartd`:
+These S.M.A.R.T hard drive self-tests can help you detect and replace failing hard drives before they cause permanent data loss. However, there's a couple issues with `smartd`:
 
 - There are more than a hundred S.M.A.R.T attributes, however `smartd` does not differentiate between critical and informational metrics
 - `smartd` does not record S.M.A.R.T attribute history, so it can be hard to determine if an attribute is degrading slowly over time.
@@ -69,7 +69,7 @@ See [docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](./docs/TROUBLESHOOTING_DEVICE_COL
 
 If you're using Docker, getting started is as simple as running the following command:
 
-> See [docker/example.omnibus.docker-compose.yml](./docker/example.omnibus.docker-compose.yml) for a docker-compose file. 
+> See [docker/example.omnibus.docker-compose.yml](https://github.com/AnalogJ/scrutiny/blob/master/docker/example.omnibus.docker-compose.yml) for a docker-compose file.
 
 ```bash
 docker run -it --rm -p 8080:8080 -p 8086:8086 \
@@ -100,7 +100,7 @@ other Docker images:
 - `influxdb:2.2` - InfluxDB image, used by the Web container to persist SMART data. Only one container necessary
   See [docs/TROUBLESHOOTING_INFLUXDB.md](./docs/TROUBLESHOOTING_INFLUXDB.md)
 
-> See [docker/example.hubspoke.docker-compose.yml](./docker/example.hubspoke.docker-compose.yml) for a docker-compose file.
+> See [docker/example.hubspoke.docker-compose.yml](https://github.com/AnalogJ/scrutiny/blob/master/docker/example.hubspoke.docker-compose.yml) for a docker-compose file.
 
 ```bash
 docker run --rm -p 8086:8086 \
@@ -157,7 +157,7 @@ Neither file is required, however if provided, it allows you to configure how Sc
 
 ## Cron Schedule
 Unfortunately the Cron schedule cannot be configured via the `collector.yaml` (as the collector binary needs to be trigged by a scheduler/cron).
-However, if you are using the official `ghcr.io/analogj/scrutiny:master-collector` or `ghcr.io/analogj/scrutiny:master-omnibus` docker images, 
+However, if you are using the official `ghcr.io/analogj/scrutiny:master-collector` or `ghcr.io/analogj/scrutiny:master-omnibus` docker images,
 you can use the `COLLECTOR_CRON_SCHEDULE` environmental variable to override the default cron schedule (daily @ midnight - `0 0 * * *`).
 
 `docker run -e COLLECTOR_CRON_SCHEDULE="0 0 * * *" ...`
@@ -174,6 +174,7 @@ Scrutiny supports sending SMART device failure notifications via the following s
 - IFTTT
 - Join
 - Mattermost
+- ntfy
 - Pushbullet
 - Pushover
 - Slack
@@ -243,7 +244,7 @@ scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
 | linux-arm-6 | :white_check_mark: |  |
 | linux-arm-7 | :white_check_mark: | web/collector only. see [#236](https://github.com/AnalogJ/scrutiny/issues/236)  |
 | linux-arm64 | :white_check_mark: | :white_check_mark: |
-| freebsd-amd64 | :white_check_mark: |  | 
+| freebsd-amd64 | :white_check_mark: |  |
 | macos-amd64 | :white_check_mark: | :white_check_mark: |
 | macos-arm64 | :white_check_mark: | :white_check_mark: |
 | windows-amd64 | :white_check_mark: | WIP, see [#15](https://github.com/AnalogJ/scrutiny/issues/15) |

collector/cmd/collector-metrics/collector-metrics.go

@@ -30,8 +30,14 @@ func main() {
 		os.Exit(1)
 	}
 
+	configFilePath := "/opt/scrutiny/config/collector.yaml"
+	configFilePathAlternative := "/opt/scrutiny/config/collector.yml"
+	if !utils.FileExists(configFilePath) && utils.FileExists(configFilePathAlternative) {
+		configFilePath = configFilePathAlternative
+	}
+
 	//we're going to load the config file manually, since we need to validate it.
-	err = config.ReadConfig("/opt/scrutiny/config/collector.yaml") // Find and read the config file
+	err = config.ReadConfig(configFilePath) // Find and read the config file
 	if _, ok := err.(errors.ConfigFileMissingError); ok {          // Handle errors reading the config file
 		//ignore "could not find config file"
 	} else if err != nil {

collector/pkg/detect/detect.go

@@ -3,13 +3,14 @@ package detect
 import (
 	"encoding/json"
 	"fmt"
+	"os"
+	"strings"
+
 	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/sirupsen/logrus"
-	"os"
-	"strings"
 )
 
 type Detect struct {
@@ -47,7 +48,7 @@ func (d *Detect) SmartctlScan() ([]models.Device, error) {
 	return detectedDevices, nil
 }
 
-//updates a device model with information from smartctl --scan
+// updates a device model with information from smartctl --scan
 // It has a couple of issues however:
 // - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
 // - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
@@ -81,8 +82,9 @@ func (d *Detect) SmartCtlInfo(device *models.Device) error {
 	device.SerialNumber = availableDeviceInfo.SerialNumber
 	device.Firmware = availableDeviceInfo.FirmwareVersion
 	device.RotationSpeed = availableDeviceInfo.RotationRate
-	device.Capacity = availableDeviceInfo.UserCapacity.Bytes
+	device.Capacity = availableDeviceInfo.Capacity()
 	device.FormFactor = availableDeviceInfo.FormFactor.Name
+	device.DeviceType = availableDeviceInfo.Device.Type
 	device.DeviceProtocol = availableDeviceInfo.Device.Protocol
 	if len(availableDeviceInfo.Vendor) > 0 {
 		device.Manufacturer = availableDeviceInfo.Vendor

collector/pkg/detect/detect_test.go

@@ -1,19 +1,22 @@
 package detect_test
 
 import (
+	"os"
+	"strings"
+	"testing"
+
 	mock_shell "github.com/analogj/scrutiny/collector/pkg/common/shell/mock"
 	mock_config "github.com/analogj/scrutiny/collector/pkg/config/mock"
 	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/golang/mock/gomock"
 	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"io/ioutil"
-	"testing"
 )
 
 func TestDetect_SmartctlScan(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -23,7 +26,7 @@ func TestDetect_SmartctlScan(t *testing.T) {
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
-	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_simple.json")
+	testScanResults, err := os.ReadFile("testdata/smartctl_scan_simple.json")
 	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
 
 	d := detect.Detect{
@@ -32,17 +35,17 @@ func TestDetect_SmartctlScan(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	scannedDevices, err := d.SmartctlScan()
 
-	//assert
+	// assert
 	require.NoError(t, err)
 	require.Equal(t, 7, len(scannedDevices))
 	require.Equal(t, "scsi", scannedDevices[0].DeviceType)
 }
 
 func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -52,7 +55,7 @@ func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
-	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_megaraid.json")
+	testScanResults, err := os.ReadFile("testdata/smartctl_scan_megaraid.json")
 	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
 
 	d := detect.Detect{
@@ -61,20 +64,20 @@ func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	scannedDevices, err := d.SmartctlScan()
 
-	//assert
+	// assert
 	require.NoError(t, err)
 	require.Equal(t, 2, len(scannedDevices))
 	require.Equal(t, []models.Device{
-		models.Device{DeviceName: "bus/0", DeviceType: "megaraid,0"},
-		models.Device{DeviceName: "bus/0", DeviceType: "megaraid,1"},
+		{DeviceName: "bus/0", DeviceType: "megaraid,0"},
+		{DeviceName: "bus/0", DeviceType: "megaraid,1"},
 	}, scannedDevices)
 }
 
 func TestDetect_SmartctlScan_Nvme(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -84,7 +87,7 @@ func TestDetect_SmartctlScan_Nvme(t *testing.T) {
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
-	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_nvme.json")
+	testScanResults, err := os.ReadFile("testdata/smartctl_scan_nvme.json")
 	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
 
 	d := detect.Detect{
@@ -93,19 +96,19 @@ func TestDetect_SmartctlScan_Nvme(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	scannedDevices, err := d.SmartctlScan()
 
-	//assert
+	// assert
 	require.NoError(t, err)
 	require.Equal(t, 1, len(scannedDevices))
 	require.Equal(t, []models.Device{
-		models.Device{DeviceName: "nvme0", DeviceType: "nvme"},
+		{DeviceName: "nvme0", DeviceType: "nvme"},
 	}, scannedDevices)
 }
 
 func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -129,16 +132,16 @@ func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, "sda", transformedDevices[0].DeviceName)
 	require.Equal(t, "scsi", transformedDevices[0].DeviceType)
 }
 
 func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -162,15 +165,15 @@ func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, []models.Device{}, transformedDevices)
 }
 
 func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -187,7 +190,8 @@ func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
 			Device:     "/dev/twa0",
 			DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
 			Ignore:     false,
-		}})
+		},
+	})
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -203,15 +207,15 @@ func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, 12, len(transformedDevices))
 }
 
 func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -234,17 +238,17 @@ func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, 1, len(transformedDevices))
 	require.Equal(t, "sat+megaraid", transformedDevices[0].DeviceType)
 }
 
 // test https://github.com/AnalogJ/scrutiny/issues/255#issuecomment-1164024126
 func TestDetect_TransformDetectedDevices_WithoutDeviceTypeOverride(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -267,16 +271,16 @@ func TestDetect_TransformDetectedDevices_WithoutDeviceTypeOverride(t *testing.T)
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, 1, len(transformedDevices))
 	require.Equal(t, "scsi", transformedDevices[0].DeviceType)
 }
 
 func TestDetect_TransformDetectedDevices_WhenDeviceNotDetected(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -290,10 +294,69 @@ func TestDetect_TransformDetectedDevices_WhenDeviceNotDetected(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, 1, len(transformedDevices))
 	require.Equal(t, "ata", transformedDevices[0].DeviceType)
 }
+
+func TestDetect_SmartCtlInfo(t *testing.T) {
+	t.Run("should report nvme info", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+
+		const (
+			someArgs = "--info --json"
+
+			// device info
+			someDeviceName           = "some-device-name"
+			someModelName            = "KCD61LUL3T84"
+			someSerialNumber         = "61Q0A05UT7B8"
+			someFirmware             = "8002"
+			someDeviceProtocol       = "NVMe"
+			someDeviceType           = "nvme"
+			someCapacity       int64 = 3840755982336
+		)
+
+		fakeConfig := mock_config.NewMockInterface(ctrl)
+		fakeConfig.EXPECT().
+			GetCommandMetricsInfoArgs("/dev/" + someDeviceName).
+			Return(someArgs)
+		fakeConfig.EXPECT().
+			GetString("commands.metrics_smartctl_bin").
+			Return("smartctl")
+
+		someLogger := logrus.WithFields(logrus.Fields{})
+
+		smartctlInfoResults, err := os.ReadFile("testdata/smartctl_info_nvme.json")
+		require.NoError(t, err)
+
+		fakeShell := mock_shell.NewMockInterface(ctrl)
+		fakeShell.EXPECT().
+			Command(someLogger, "smartctl", append(strings.Split(someArgs, " "), "/dev/"+someDeviceName), "", gomock.Any()).
+			Return(string(smartctlInfoResults), err)
+
+		d := detect.Detect{
+			Logger: someLogger,
+			Shell:  fakeShell,
+			Config: fakeConfig,
+		}
+
+		someDevice := &models.Device{
+			WWN:        "some wwn",
+			DeviceName: someDeviceName,
+		}
+
+		require.NoError(t, d.SmartCtlInfo(someDevice))
+
+		assert.Equal(t, someDeviceName, someDevice.DeviceName)
+		assert.Equal(t, someModelName, someDevice.ModelName)
+		assert.Equal(t, someSerialNumber, someDevice.SerialNumber)
+		assert.Equal(t, someFirmware, someDevice.Firmware)
+		assert.Equal(t, someDeviceProtocol, someDevice.DeviceProtocol)
+		assert.Equal(t, someDeviceType, someDevice.DeviceType)
+		assert.Equal(t, someCapacity, someDevice.Capacity)
+	})
+}

collector/pkg/detect/testdata/smartctl_info_nvme.json

@@ -0,0 +1,48 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      2
+    ],
+    "svn_revision": "5155",
+    "platform_info": "x86_64-linux-6.1.69-talos",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "--info",
+      "--json",
+      "/dev/nvme4"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/nvme4",
+    "info_name": "/dev/nvme4",
+    "type": "nvme",
+    "protocol": "NVMe"
+  },
+  "model_name": "KCD61LUL3T84",
+  "serial_number": "61Q0A05UT7B8",
+  "firmware_version": "8002",
+  "nvme_pci_vendor": {
+    "id": 7695,
+    "subsystem_id": 7695
+  },
+  "nvme_ieee_oui_identifier": 9233294,
+  "nvme_total_capacity": 3840755982336,
+  "nvme_unallocated_capacity": 0,
+  "nvme_controller_id": 1,
+  "nvme_version": {
+    "string": "1.4",
+    "value": 66560
+  },
+  "nvme_number_of_namespaces": 16,
+  "local_time": {
+    "time_t": 1706045146,
+    "asctime": "Tue Jan 23 21:25:46 2024 UTC"
+  }
+}

docker/Dockerfile

@@ -1,50 +1,71 @@
+# syntax=docker/dockerfile:1.4
 ########################################################################################################################
 # Omnibus Image
-# NOTE: this image requires the `make binary-frontend` target to have been run before `docker build` The `dist` directory must exist.
 ########################################################################################################################
 
+######## Build the frontend
+FROM --platform=${BUILDPLATFORM} node AS frontendbuild
+WORKDIR /go/src/github.com/analogj/scrutiny
+COPY --link . /go/src/github.com/analogj/scrutiny
 
-########
-FROM golang:1.20-bullseye as backendbuild
+RUN make binary-frontend
+
+
+######## Build the backend
+FROM golang:1.20-bookworm as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
-COPY . /go/src/github.com/analogj/scrutiny
+COPY --link . /go/src/github.com/analogj/scrutiny
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
+    apt-get install -y --no-install-recommends \
+    file \
+    && rm -rf /var/lib/apt/lists/*
 RUN make binary-clean binary-all WEB_BINARY_NAME=scrutiny
 
 
-########
-FROM debian:bullseye-slim as runtime
+######## Combine build artifacts in runtime image
+FROM debian:bookworm-slim as runtime
 ARG TARGETARCH
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 ENV INFLUXD_CONFIG_PATH=/opt/scrutiny/influxdb
+ENV S6VER="1.21.8.0"
+ENV INFLUXVER="2.2.0"
+SHELL ["/usr/bin/sh", "-c"]
 
-RUN apt-get update && apt-get install -y cron smartmontools ca-certificates curl tzdata \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
+    apt-get install -y --no-install-recommends \
+    ca-certificates \
+    cron \
+    curl \
+    smartmontools \
+    tzdata \
+    && rm -rf /var/lib/apt/lists/* \
     && update-ca-certificates \
-    &&  case ${TARGETARCH} in \
-            "amd64")  S6_ARCH=amd64  ;; \
-            "arm64")  S6_ARCH=aarch64  ;; \
-        esac \
-    && curl https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-${S6_ARCH}.tar.gz -L -s --output /tmp/s6-overlay-${S6_ARCH}.tar.gz \
+    && case ${TARGETARCH} in \
+           "amd64")  S6_ARCH=amd64  ;; \
+           "arm64")  S6_ARCH=aarch64  ;; \
+       esac \
+    && curl https://github.com/just-containers/s6-overlay/releases/download/v${S6VER}/s6-overlay-${S6_ARCH}.tar.gz -L -s --output /tmp/s6-overlay-${S6_ARCH}.tar.gz \
     && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C / \
     && rm -rf /tmp/s6-overlay-${S6_ARCH}.tar.gz \
-    && curl -L https://dl.influxdata.com/influxdb/releases/influxdb2-2.2.0-${TARGETARCH}.deb --output /tmp/influxdb2-2.2.0-${TARGETARCH}.deb \
-    && dpkg -i --force-all /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
+    && curl -L https://dl.influxdata.com/influxdb/releases/influxdb2-${INFLUXVER}-${TARGETARCH}.deb --output /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb \
+    && ln -s /usr/bin/false /bin/false \
+    && ln -s /usr/bin/bash /bin/bash \
+    && dpkg -i --force-all /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb \
+    && rm -f /bin/bash \
+    && rm -rf /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb
 
 COPY /rootfs /
 
-COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
-COPY dist /opt/scrutiny/web
-RUN chmod +x /opt/scrutiny/bin/scrutiny && \
-    chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \
-    chmod 0644 /etc/cron.d/scrutiny && \
+COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
+COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
+COPY --link --from=frontendbuild --chmod=644 /go/src/github.com/analogj/scrutiny/dist /opt/scrutiny/web
+RUN chmod 0644 /etc/cron.d/scrutiny && \
     rm -f /etc/cron.daily/* && \
     mkdir -p /opt/scrutiny/web && \
     mkdir -p /opt/scrutiny/config && \
     chmod -R ugo+rwx /opt/scrutiny/config
 
-
 CMD ["/init"]

docker/Dockerfile.collector

@@ -4,20 +4,21 @@
 
 
 ########
-FROM golang:1.20-bullseye as backendbuild
+FROM golang:1.20-bookworm as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
 COPY . /go/src/github.com/analogj/scrutiny
 
+RUN apt-get update && apt-get install -y file && rm -rf /var/lib/apt/lists/*
 RUN make binary-clean binary-collector
 
 ########
-FROM debian:bullseye-slim as runtime
+FROM debian:bookworm-slim as runtime
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && update-ca-certificates
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && rm -rf /var/lib/apt/lists/* && update-ca-certificates
 
 COPY /docker/entrypoint-collector.sh /entrypoint-collector.sh
 COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny

docker/Dockerfile.web

@@ -1,31 +1,37 @@
+# syntax=docker/dockerfile:1.4
 ########################################################################################################################
 # Web Image
-# NOTE: this image requires the `make binary-frontend` target to have been run before `docker build` The `dist` directory must exist.
 ########################################################################################################################
 
+######## Build the frontend
+FROM --platform=${BUILDPLATFORM} node AS frontendbuild
+WORKDIR /go/src/github.com/analogj/scrutiny
+COPY --link . /go/src/github.com/analogj/scrutiny
 
-########
-FROM golang:1.20-bullseye as backendbuild
+RUN make binary-frontend
+
+######## Build the backend
+FROM golang:1.20-bookworm as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
+COPY --link . /go/src/github.com/analogj/scrutiny
 
-COPY . /go/src/github.com/analogj/scrutiny
-
+RUN apt-get update && apt-get install -y file && rm -rf /var/lib/apt/lists/*
 RUN make binary-clean binary-all WEB_BINARY_NAME=scrutiny
 
 
-########
-FROM debian:bullseye-slim as runtime
+######## Combine build artifacts in runtime image
+FROM debian:bookworm-slim as runtime
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-RUN apt-get update && apt-get install -y ca-certificates curl tzdata && update-ca-certificates
+RUN apt-get update && apt-get install -y ca-certificates curl tzdata && rm -rf /var/lib/apt/lists/* && update-ca-certificates
 
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
-COPY dist /opt/scrutiny/web
-RUN chmod +x /opt/scrutiny/bin/scrutiny && \
-    mkdir -p /opt/scrutiny/web && \
+COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
+COPY --link --from=frontendbuild --chmod=644 /go/src/github.com/analogj/scrutiny/dist /opt/scrutiny/web
+RUN mkdir -p /opt/scrutiny/web && \
     mkdir -p /opt/scrutiny/config && \
-    chmod -R ugo+rwx /opt/scrutiny/config
+    chmod -R a+rX /opt/scrutiny && \
+    chmod -R a+w /opt/scrutiny/config
 CMD ["/opt/scrutiny/bin/scrutiny", "start"]

docker/entrypoint-collector.sh

@@ -25,4 +25,4 @@ fi
 
 # now that we have the env start cron in the foreground
 echo "starting cron"
-su -c "cron -f -L 15" root
+exec su -c "cron -f -L 15" root

docker/example.hubspoke.docker-compose.yml

@@ -40,9 +40,10 @@ services:
       - '/run/udev:/run/udev:ro'
     environment:
       COLLECTOR_API_ENDPOINT: 'http://web:8080'
+      COLLECTOR_HOST_ID: 'scrutiny-collector-hostname'
     depends_on:
       web:
         condition: service_healthy
     devices:
       - "/dev/sda"
-      - "/dev/sdb"
+      - "/dev/sdb"

docs/INSTALL_HUB_SPOKE.md

@@ -91,7 +91,7 @@ services:
       - SCRUTINY_WEB_INFLUXDB_ORG=homelab
       - SCRUTINY_WEB_INFLUXDB_BUCKET=scrutiny
       # Optional but highly recommended to notify you in case of a problem
-      - SCRUTINY_WEB_NOTIFY_URLS=["http://gotify:80/message?token=a-gotify-token"]
+      - SCRUTINY_NOTIFY_URLS=["http://gotify:80/message?token=a-gotify-token"]
     depends_on:
       - influxdb
     restart: unless-stopped

docs/SUPPORTED_NAS_OS.md

@@ -9,7 +9,8 @@ in `docs/guides/` or elsewhere) it will be linked here.
 - [ ] Proxmox
 - [x] Synology 
   - [Hub/Spoke Deployment - Collector](./INSTALL_SYNOLOGY_COLLECTOR.md)
-  - [Omnibus Deployment](https://drfrankenstein.co.uk/2022/07/28/scrutiny-in-docker-on-a-synology-nas)
+  - [Omnibus Deployment](https://drfrankenstein.co.uk/2022/07/28/scrutiny-in-docker-on-a-synology-nas) -  Docker Package
+  - [Omnibus Deployment](https://drfrankenstein.co.uk/scrutiny-in-container-manager-on-a-synology-nas/) -  Container Manager Package
 - [ ] OMV
 - [ ] Amahi
 - [ ] Running in a LXC container

docs/TROUBLESHOOTING_NOTIFICATIONS.md

@@ -24,8 +24,23 @@ SCRUTINY_MESSAGE - eg. "Scrutiny SMART error notification for device: %s\nFailur
 SCRUTINY_HOST_ID - (optional) eg. "my-custom-host-id"
 ```
 
+# Special Characters
+
+`Shoutrrr` supports special characters in the username and password fields, however you'll need to url-encode the
+username and the password separately.
+
+- if your username is: `myname@example.com`
+- if your password is `124@34$1`
+
+Then your `shoutrrr` url will look something like:
+
+- `smtp://myname%40example%2Ecom:124%4034%241@ms.my.domain.com:587`
+
 # Testing Notifications
-You can test that your notifications are configured correctly by posting an empty payload to the notifications health check API.
+
+You can test that your notifications are configured correctly by posting an empty payload to the notifications health
+check API.
+
 ```
 curl -X POST http://localhost:8080/api/health/notify
 ```

docs/TROUBLESHOOTING_REVERSE_PROXY.md

@@ -103,4 +103,37 @@ You may also configure these values using the following environmental variables
           - "9090:9090"
     ```
 3. run `docker-compose up`
-4. visit [http://localhost:9090/custom/web](http://localhost:9090/custom/web) - access the scrutiny container via caddy reverse proxy
+4. visit [http://localhost:9090/custom/web](http://localhost:9090/custom/web) - access the scrutiny container via caddy reverse proxy
+ 
+## Traefik
+
+Assuming, that you have Traefik up and running with [AutoDiscovery Using Traefik For Docker ](https://doc.traefik.io/traefik/providers/docker/),
+here is an example of a `docker-compose.yml` file, with labels to enable Traefik reverse proxy and basic auth
+```yaml
+version: '3.5'
+services:
+  scrutiny:
+    container_name: scrutiny
+    image: ghcr.io/analogj/scrutiny:master-omnibus
+    cap_add:
+      - SYS_RAWIO
+      - SYS_ADMIN
+    volumes:
+      - /run/udev:/run/udev:ro
+      - ./config:/opt/scrutiny/config
+      - ./influxdb:/opt/scrutiny/influxdb
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.scrutiny.rule=Host(`example.com`)
+      - traefik.http.services.scrutiny.loadbalancer.server.port=8080
+        # 2 labels below are optional, in case you want basic auth in Traefik:
+      - traefik.http.routers.scrutiny.middlewares=auth
+      - "traefik.http.middlewares.auth.basicauth.users=user:$$2y$$05$$G11Wm/dlWpXHENK..m8se.zxvaE8USJBp1Ws56sSCrOcwWDjsYHni"
+        # Note: when used in docker-compose.yml all dollar signs in the hash need to be doubled for escaping.
+        # To create user:password pair, it's possible to use this command:
+        # echo $(htpasswd -nB user) | sed -e s/\\$/\\$\\$/g
+    devices:
+      - "/dev/sda"
+      - "/dev/sdb"
+      - "/dev/nvme0"
+```

example.scrutiny.yaml

@@ -60,6 +60,10 @@ log:
 
 # Notification "urls" look like the following. For more information about service specific configuration see
 # Shoutrrr's documentation: https://containrrr.dev/shoutrrr/services/overview/
+#
+# note, usernames and passwords containing special characters will need to be urlencoded.
+# if your username is: "myname@example.com" and your password is "124@34$1"
+# your shoutrrr url will look like: "smtp://myname%40example%2Ecom:124%4034%241@ms.my.domain.com:587"
 
 #notify:
 #  urls:
@@ -73,6 +77,7 @@ log:
 #    - "pushbullet://api-token[/device/#channel/email]"
 #    - "ifttt://key/?events=event1[,event2,...]&value1=value1&value2=value2&value3=value3"
 #    - "mattermost://[username@]mattermost-host/token[/channel]"
+#    - "ntfy://username:password@host:port/topic"
 #    - "hangouts://chat.googleapis.com/v1/spaces/FOO/messages?key=bar&token=baz"
 #    - "zulip://bot-mail:bot-key@zulip-domain/?stream=name-or-id&topic=name"
 #    - "join://shoutrrr:api-key@join/?devices=device1[,device2, ...][&icon=icon][&title=title]"

go.mod

@@ -4,30 +4,30 @@ go 1.20
 
 require (
 	github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
-	github.com/containrrr/shoutrrr v0.6.1
-	github.com/fatih/color v1.10.0
+	github.com/containrrr/shoutrrr v0.7.1
+	github.com/fatih/color v1.15.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/glebarez/sqlite v1.4.5
 	github.com/go-gormigrate/gormigrate/v2 v2.0.0
-	github.com/golang/mock v1.4.3
+	github.com/golang/mock v1.6.0
 	github.com/influxdata/influxdb-client-go/v2 v2.9.0
 	github.com/jaypipes/ghw v0.6.1
-	github.com/mitchellh/mapstructure v1.2.2
+	github.com/mitchellh/mapstructure v1.5.0
 	github.com/samber/lo v1.25.0
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/viper v1.7.0
-	github.com/stretchr/testify v1.7.1
+	github.com/sirupsen/logrus v1.6.0
+	github.com/spf13/viper v1.14.0
+	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.2.0
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	golang.org/x/sync v0.1.0
 	gorm.io/gorm v1.23.5
 )
 
 require (
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/glebarez/go-sqlite v1.17.2 // indirect
@@ -35,44 +35,45 @@ require (
 	github.com/go-playground/locales v0.13.0 // indirect
 	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/go-playground/validator/v10 v10.2.0 // indirect
-	github.com/golang/protobuf v1.4.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
 	github.com/jaypipes/pcidb v0.5.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.4 // indirect
-	github.com/json-iterator/go v1.1.9 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/kvz/logstreamer v0.0.0-20201023134116-02d20f4338f5 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
-	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/pelletier/go-toml v1.7.0 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/afero v1.9.2 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/ugorji/go/codec v1.1.7 // indirect
-	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
+	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
-	golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
-	golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 // indirect
-	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
-	golang.org/x/text v0.3.5 // indirect
-	google.golang.org/protobuf v1.23.0 // indirect
-	gopkg.in/ini.v1 v1.55.0 // indirect
+	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/term v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
 	modernc.org/libc v1.16.8 // indirect
 	modernc.org/mathutil v1.4.1 // indirect

webapp/backend/cmd/scrutiny/scrutiny.go

@@ -29,8 +29,14 @@ func main() {
 		os.Exit(1)
 	}
 
+	configFilePath := "/opt/scrutiny/config/scrutiny.yaml"
+	configFilePathAlternative := "/opt/scrutiny/config/scrutiny.yml"
+	if !utils.FileExists(configFilePath) && utils.FileExists(configFilePathAlternative) {
+		configFilePath = configFilePathAlternative
+	}
+
 	//we're going to load the config file manually, since we need to validate it.
-	err = config.ReadConfig("/opt/scrutiny/config/scrutiny.yaml") // Find and read the config file
+	err = config.ReadConfig(configFilePath) // Find and read the config file
 	if _, ok := err.(errors.ConfigFileMissingError); ok {         // Handle errors reading the config file
 		//ignore "could not find config file"
 	} else if err != nil {

webapp/backend/pkg/database/helpers.go

@@ -1,12 +0,0 @@
-package database
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
-	"sort"
-)
-
-func sortSmartMeasurementsDesc(smartResults []measurements.Smart) {
-	sort.SliceStable(smartResults, func(i, j int) bool {
-		return smartResults[i].Date.After(smartResults[j].Date)
-	})
-}

webapp/backend/pkg/database/helpers_test.go

@@ -1,30 +0,0 @@
-package database
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
-	"github.com/stretchr/testify/require"
-	"testing"
-	"time"
-)
-
-func Test_sortSmartMeasurementsDesc_LatestFirst(t *testing.T) {
-	//setup
-	timeNow := time.Now()
-	smartResults := []measurements.Smart{
-		{
-			Date: timeNow.AddDate(0, 0, -2),
-		},
-		{
-			Date: timeNow,
-		},
-		{
-			Date: timeNow.AddDate(0, 0, -1),
-		},
-	}
-
-	//test
-	sortSmartMeasurementsDesc(smartResults)
-
-	//assert
-	require.Equal(t, smartResults[0].Date, timeNow)
-}

webapp/backend/pkg/database/interface.go

@@ -2,12 +2,15 @@ package database
 
 import (
 	"context"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 )
 
+// Create mock using:
+// mockgen -source=webapp/backend/pkg/database/interface.go -destination=webapp/backend/pkg/database/mock/mock_database.go
 type DeviceRepo interface {
 	Close() error
 	HealthCheck(ctx context.Context) error
@@ -20,7 +23,7 @@ type DeviceRepo interface {
 	DeleteDevice(ctx context.Context, wwn string) error
 
 	SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error)
-	GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error)
+	GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error)
 
 	SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error
 

webapp/backend/pkg/database/mock/mock_database.go

@@ -0,0 +1,258 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: webapp/backend/pkg/database/interface.go
+
+// Package mock_database is a generated GoMock package.
+package mock_database
+
+import (
+	context "context"
+	reflect "reflect"
+
+	pkg "github.com/analogj/scrutiny/webapp/backend/pkg"
+	models "github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	collector "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	measurements "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockDeviceRepo is a mock of DeviceRepo interface.
+type MockDeviceRepo struct {
+	ctrl     *gomock.Controller
+	recorder *MockDeviceRepoMockRecorder
+}
+
+// MockDeviceRepoMockRecorder is the mock recorder for MockDeviceRepo.
+type MockDeviceRepoMockRecorder struct {
+	mock *MockDeviceRepo
+}
+
+// NewMockDeviceRepo creates a new mock instance.
+func NewMockDeviceRepo(ctrl *gomock.Controller) *MockDeviceRepo {
+	mock := &MockDeviceRepo{ctrl: ctrl}
+	mock.recorder = &MockDeviceRepoMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockDeviceRepo) EXPECT() *MockDeviceRepoMockRecorder {
+	return m.recorder
+}
+
+// Close mocks base method.
+func (m *MockDeviceRepo) Close() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Close")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Close indicates an expected call of Close.
+func (mr *MockDeviceRepoMockRecorder) Close() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockDeviceRepo)(nil).Close))
+}
+
+// DeleteDevice mocks base method.
+func (m *MockDeviceRepo) DeleteDevice(ctx context.Context, wwn string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteDevice", ctx, wwn)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteDevice indicates an expected call of DeleteDevice.
+func (mr *MockDeviceRepoMockRecorder) DeleteDevice(ctx, wwn interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDevice", reflect.TypeOf((*MockDeviceRepo)(nil).DeleteDevice), ctx, wwn)
+}
+
+// GetDeviceDetails mocks base method.
+func (m *MockDeviceRepo) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDeviceDetails", ctx, wwn)
+	ret0, _ := ret[0].(models.Device)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDeviceDetails indicates an expected call of GetDeviceDetails.
+func (mr *MockDeviceRepoMockRecorder) GetDeviceDetails(ctx, wwn interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceDetails", reflect.TypeOf((*MockDeviceRepo)(nil).GetDeviceDetails), ctx, wwn)
+}
+
+// GetDevices mocks base method.
+func (m *MockDeviceRepo) GetDevices(ctx context.Context) ([]models.Device, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDevices", ctx)
+	ret0, _ := ret[0].([]models.Device)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDevices indicates an expected call of GetDevices.
+func (mr *MockDeviceRepoMockRecorder) GetDevices(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDevices", reflect.TypeOf((*MockDeviceRepo)(nil).GetDevices), ctx)
+}
+
+// GetSmartAttributeHistory mocks base method.
+func (m *MockDeviceRepo) GetSmartAttributeHistory(ctx context.Context, wwn, durationKey string, selectEntries, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetSmartAttributeHistory", ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes)
+	ret0, _ := ret[0].([]measurements.Smart)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetSmartAttributeHistory indicates an expected call of GetSmartAttributeHistory.
+func (mr *MockDeviceRepoMockRecorder) GetSmartAttributeHistory(ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSmartAttributeHistory", reflect.TypeOf((*MockDeviceRepo)(nil).GetSmartAttributeHistory), ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes)
+}
+
+// GetSmartTemperatureHistory mocks base method.
+func (m *MockDeviceRepo) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetSmartTemperatureHistory", ctx, durationKey)
+	ret0, _ := ret[0].(map[string][]measurements.SmartTemperature)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetSmartTemperatureHistory indicates an expected call of GetSmartTemperatureHistory.
+func (mr *MockDeviceRepoMockRecorder) GetSmartTemperatureHistory(ctx, durationKey interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSmartTemperatureHistory", reflect.TypeOf((*MockDeviceRepo)(nil).GetSmartTemperatureHistory), ctx, durationKey)
+}
+
+// GetSummary mocks base method.
+func (m *MockDeviceRepo) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetSummary", ctx)
+	ret0, _ := ret[0].(map[string]*models.DeviceSummary)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetSummary indicates an expected call of GetSummary.
+func (mr *MockDeviceRepoMockRecorder) GetSummary(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSummary", reflect.TypeOf((*MockDeviceRepo)(nil).GetSummary), ctx)
+}
+
+// HealthCheck mocks base method.
+func (m *MockDeviceRepo) HealthCheck(ctx context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "HealthCheck", ctx)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// HealthCheck indicates an expected call of HealthCheck.
+func (mr *MockDeviceRepoMockRecorder) HealthCheck(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HealthCheck", reflect.TypeOf((*MockDeviceRepo)(nil).HealthCheck), ctx)
+}
+
+// LoadSettings mocks base method.
+func (m *MockDeviceRepo) LoadSettings(ctx context.Context) (*models.Settings, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "LoadSettings", ctx)
+	ret0, _ := ret[0].(*models.Settings)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// LoadSettings indicates an expected call of LoadSettings.
+func (mr *MockDeviceRepoMockRecorder) LoadSettings(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadSettings", reflect.TypeOf((*MockDeviceRepo)(nil).LoadSettings), ctx)
+}
+
+// RegisterDevice mocks base method.
+func (m *MockDeviceRepo) RegisterDevice(ctx context.Context, dev models.Device) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RegisterDevice", ctx, dev)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RegisterDevice indicates an expected call of RegisterDevice.
+func (mr *MockDeviceRepoMockRecorder) RegisterDevice(ctx, dev interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterDevice", reflect.TypeOf((*MockDeviceRepo)(nil).RegisterDevice), ctx, dev)
+}
+
+// SaveSettings mocks base method.
+func (m *MockDeviceRepo) SaveSettings(ctx context.Context, settings models.Settings) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SaveSettings", ctx, settings)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SaveSettings indicates an expected call of SaveSettings.
+func (mr *MockDeviceRepoMockRecorder) SaveSettings(ctx, settings interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSettings", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSettings), ctx, settings)
+}
+
+// SaveSmartAttributes mocks base method.
+func (m *MockDeviceRepo) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SaveSmartAttributes", ctx, wwn, collectorSmartData)
+	ret0, _ := ret[0].(measurements.Smart)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// SaveSmartAttributes indicates an expected call of SaveSmartAttributes.
+func (mr *MockDeviceRepoMockRecorder) SaveSmartAttributes(ctx, wwn, collectorSmartData interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartAttributes", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartAttributes), ctx, wwn, collectorSmartData)
+}
+
+// SaveSmartTemperature mocks base method.
+func (m *MockDeviceRepo) SaveSmartTemperature(ctx context.Context, wwn, deviceProtocol string, collectorSmartData collector.SmartInfo) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SaveSmartTemperature", ctx, wwn, deviceProtocol, collectorSmartData)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SaveSmartTemperature indicates an expected call of SaveSmartTemperature.
+func (mr *MockDeviceRepoMockRecorder) SaveSmartTemperature(ctx, wwn, deviceProtocol, collectorSmartData interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartTemperature", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartTemperature), ctx, wwn, deviceProtocol, collectorSmartData)
+}
+
+// UpdateDevice mocks base method.
+func (m *MockDeviceRepo) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateDevice", ctx, wwn, collectorSmartData)
+	ret0, _ := ret[0].(models.Device)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// UpdateDevice indicates an expected call of UpdateDevice.
+func (mr *MockDeviceRepoMockRecorder) UpdateDevice(ctx, wwn, collectorSmartData interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDevice", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDevice), ctx, wwn, collectorSmartData)
+}
+
+// UpdateDeviceStatus mocks base method.
+func (m *MockDeviceRepo) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateDeviceStatus", ctx, wwn, status)
+	ret0, _ := ret[0].(models.Device)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// UpdateDeviceStatus indicates an expected call of UpdateDeviceStatus.
+func (mr *MockDeviceRepoMockRecorder) UpdateDeviceStatus(ctx, wwn, status interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceStatus", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceStatus), ctx, wwn, status)
+}

webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go

@@ -3,13 +3,14 @@ package database
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
 	"github.com/influxdata/influxdb-client-go/v2/api"
 	log "github.com/sirupsen/logrus"
-	"strings"
-	"time"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -30,14 +31,17 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin
 }
 
 // GetSmartAttributeHistory MUST return in sorted order, where newest entries are at the beginning of the list, and oldest are at the end.
-func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) {
+// When selectEntries is > 0, only the most recent selectEntries database entries are returned, starting from the selectEntriesOffset entry.
+// For example, with selectEntries = 5, selectEntries = 0, the most recent 5 are returned. With selectEntries = 3, selectEntries = 2, entries
+// 2 to 4 are returned (2 being the third newest, since it is zero-indexed)
+func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) {
 	// Get SMartResults from InfluxDB
 
 	//TODO: change the filter startrange to a real number.
 
 	// Get parser flux query result
 	//appConfig.GetString("web.influxdb.bucket")
-	queryStr := sr.aggregateSmartAttributesQuery(wwn, durationKey)
+	queryStr := sr.aggregateSmartAttributesQuery(wwn, durationKey, selectEntries, selectEntriesOffset, attributes)
 	log.Infoln(queryStr)
 
 	smartResults := []measurements.Smart{}
@@ -65,9 +69,6 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn
 		return nil, err
 	}
 
-	//we have to sort the smartResults again, because the `union` command will return multiple 'tables' and only sort the records in each table.
-	sortSmartMeasurementsDesc(smartResults)
-
 	return smartResults, nil
 
 	//if err := device.SquashHistory(); err != nil {
@@ -99,7 +100,7 @@ func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking,
 	return influxWriteApi.WritePoint(ctx, p)
 }
 
-func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string) string {
+func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string {
 
 	/*
 
@@ -108,28 +109,34 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration
 		|> range(start: -1w, stop: now())
 		|> filter(fn: (r) => r["_measurement"] == "smart" )
 		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> tail(n: 10, offset: 0)
 		|> schema.fieldsAsCols()
 
 		monthData = from(bucket: "metrics_weekly")
 		|> range(start: -1mo, stop: -1w)
 		|> filter(fn: (r) => r["_measurement"] == "smart" )
 		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> tail(n: 10, offset: 0)
 		|> schema.fieldsAsCols()
 
 		yearData = from(bucket: "metrics_monthly")
 		|> range(start: -1y, stop: -1mo)
 		|> filter(fn: (r) => r["_measurement"] == "smart" )
 		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> tail(n: 10, offset: 0)
 		|> schema.fieldsAsCols()
 
 		foreverData = from(bucket: "metrics_yearly")
 		|> range(start: -10y, stop: -1y)
 		|> filter(fn: (r) => r["_measurement"] == "smart" )
 		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> tail(n: 10, offset: 0)
 		|> schema.fieldsAsCols()
 
 		union(tables: [weekData, monthData, yearData, foreverData])
-		|> sort(columns: ["_time"], desc: false)
+		|> group()
+		|> sort(columns: ["_time"], desc: true)
+		|> tail(n: 6, offset: 4)
 		|> yield(name: "last")
 
 	*/
@@ -140,34 +147,57 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration
 
 	nestedDurationKeys := sr.lookupNestedDurationKeys(durationKey)
 
-	subQueryNames := []string{}
-	for _, nestedDurationKey := range nestedDurationKeys {
-		bucketName := sr.lookupBucketName(nestedDurationKey)
-		durationRange := sr.lookupDuration(nestedDurationKey)
-
-		subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey))
-		partialQueryStr = append(partialQueryStr, []string{
-			fmt.Sprintf(`%sData = from(bucket: "%s")`, nestedDurationKey, bucketName),
-			fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
-			`|> filter(fn: (r) => r["_measurement"] == "smart" )`,
-			fmt.Sprintf(`|> filter(fn: (r) => r["device_wwn"] == "%s" )`, wwn),
-			"|> schema.fieldsAsCols()",
-		}...)
-	}
-
-	if len(subQueryNames) == 1 {
+	if len(nestedDurationKeys) == 1 {
 		//there's only one bucket being queried, no need to union, just aggregate the dataset and return
 		partialQueryStr = append(partialQueryStr, []string{
-			subQueryNames[0],
+			sr.generateSmartAttributesSubquery(wwn, nestedDurationKeys[0], selectEntries, selectEntriesOffset, attributes),
+			fmt.Sprintf(`%sData`, nestedDurationKeys[0]),
+			`|> sort(columns: ["_time"], desc: true)`,
 			`|> yield()`,
 		}...)
-	} else {
-		partialQueryStr = append(partialQueryStr, []string{
-			fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")),
-			`|> sort(columns: ["_time"], desc: false)`,
-			`|> yield(name: "last")`,
-		}...)
+		return strings.Join(partialQueryStr, "\n")
 	}
 
+	subQueries := []string{}
+	subQueryNames := []string{}
+	for _, nestedDurationKey := range nestedDurationKeys {
+		subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey))
+		if selectEntries > 0 {
+			// We only need the last `n + offset` # of entries from each table to guarantee we can
+			// get the last `n` # of entries starting from `offset` of the union
+			subQueries = append(subQueries, sr.generateSmartAttributesSubquery(wwn, nestedDurationKey, selectEntries+selectEntriesOffset, 0, attributes))
+		} else {
+			subQueries = append(subQueries, sr.generateSmartAttributesSubquery(wwn, nestedDurationKey, 0, 0, attributes))
+		}
+	}
+	partialQueryStr = append(partialQueryStr, subQueries...)
+	partialQueryStr = append(partialQueryStr, []string{
+		fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")),
+		`|> group()`,
+		`|> sort(columns: ["_time"], desc: true)`,
+	}...)
+	if selectEntries > 0 {
+		partialQueryStr = append(partialQueryStr, fmt.Sprintf(`|> tail(n: %d, offset: %d)`, selectEntries, selectEntriesOffset))
+	}
+	partialQueryStr = append(partialQueryStr, `|> yield(name: "last")`)
+
+	return strings.Join(partialQueryStr, "\n")
+}
+
+func (sr *scrutinyRepository) generateSmartAttributesSubquery(wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string {
+	bucketName := sr.lookupBucketName(durationKey)
+	durationRange := sr.lookupDuration(durationKey)
+
+	partialQueryStr := []string{
+		fmt.Sprintf(`%sData = from(bucket: "%s")`, durationKey, bucketName),
+		fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
+		`|> filter(fn: (r) => r["_measurement"] == "smart" )`,
+		fmt.Sprintf(`|> filter(fn: (r) => r["device_wwn"] == "%s" )`, wwn),
+	}
+	if selectEntries > 0 {
+		partialQueryStr = append(partialQueryStr, fmt.Sprintf(`|> tail(n: %d, offset: %d)`, selectEntries, selectEntriesOffset))
+	}
+	partialQueryStr = append(partialQueryStr, "|> schema.fieldsAsCols()")
+
 	return strings.Join(partialQueryStr, "\n")
 }

webapp/backend/pkg/database/scrutiny_repository_migrations.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strconv"
+	"time"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
@@ -17,8 +20,6 @@ import (
 	"github.com/influxdata/influxdb-client-go/v2/api/http"
 	log "github.com/sirupsen/logrus"
 	"gorm.io/gorm"
-	"strconv"
-	"time"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -369,6 +370,21 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 				return tx.Create(&defaultSettings).Error
 			},
 		},
+		{
+			ID: "m20231123123300", // add repeat_notifications setting.
+			Migrate: func(tx *gorm.DB) error {
+				//add repeat_notifications setting default.
+				var defaultSettings = []m20220716214900.Setting{
+					{
+						SettingKeyName:        "metrics.repeat_notifications",
+						SettingKeyDescription: "Whether to repeat all notifications or just when values change (true | false)",
+						SettingDataType:       "bool",
+						SettingValueBool:      true,
+					},
+				}
+				return tx.Create(&defaultSettings).Error
+			},
+		},
 	})
 
 	if err := m.Migrate(); err != nil {

webapp/backend/pkg/models/collector/smart.go

@@ -27,14 +27,11 @@ type SmartInfo struct {
 		Oui uint64 `json:"oui"`
 		ID  uint64 `json:"id"`
 	} `json:"wwn"`
-	FirmwareVersion string `json:"firmware_version"`
-	UserCapacity    struct {
-		Blocks int64 `json:"blocks"`
-		Bytes  int64 `json:"bytes"`
-	} `json:"user_capacity"`
-	LogicalBlockSize  int `json:"logical_block_size"`
-	PhysicalBlockSize int `json:"physical_block_size"`
-	RotationRate      int `json:"rotation_rate"`
+	FirmwareVersion   string       `json:"firmware_version"`
+	UserCapacity      UserCapacity `json:"user_capacity"`
+	LogicalBlockSize  int          `json:"logical_block_size"`
+	PhysicalBlockSize int          `json:"physical_block_size"`
+	RotationRate      int          `json:"rotation_rate"`
 	FormFactor        struct {
 		AtaValue int    `json:"ata_value"`
 		Name     string `json:"name"`
@@ -210,9 +207,10 @@ type SmartInfo struct {
 		ID          int `json:"id"`
 		SubsystemID int `json:"subsystem_id"`
 	} `json:"nvme_pci_vendor"`
-	NvmeIeeeOuiIdentifier  int `json:"nvme_ieee_oui_identifier"`
-	NvmeControllerID       int `json:"nvme_controller_id"`
-	NvmeNumberOfNamespaces int `json:"nvme_number_of_namespaces"`
+	NvmeIeeeOuiIdentifier  int   `json:"nvme_ieee_oui_identifier"`
+	NvmeTotalCapacity      int64 `json:"nvme_total_capacity"`
+	NvmeControllerID       int   `json:"nvme_controller_id"`
+	NvmeNumberOfNamespaces int   `json:"nvme_number_of_namespaces"`
 	NvmeNamespaces         []struct {
 		ID   int `json:"id"`
 		Size struct {
@@ -239,7 +237,23 @@ type SmartInfo struct {
 	ScsiErrorCounterLog ScsiErrorCounterLog `json:"scsi_error_counter_log"`
 }
 
-//Primary Attribute Structs
+// Capacity finds the total capacity of the device in bytes, or 0 if unknown.
+func (s *SmartInfo) Capacity() int64 {
+	switch {
+	case s.NvmeTotalCapacity > 0:
+		return s.NvmeTotalCapacity
+	case s.UserCapacity.Bytes > 0:
+		return s.UserCapacity.Bytes
+	}
+	return 0
+}
+
+type UserCapacity struct {
+	Blocks int64 `json:"blocks"`
+	Bytes  int64 `json:"bytes"`
+}
+
+// Primary Attribute Structs
 type AtaSmartAttributesTableItem struct {
 	ID         int    `json:"id"`
 	Name       string `json:"name"`

webapp/backend/pkg/models/collector/smart_test.go

@@ -0,0 +1,33 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSmartInfo_Capacity(t *testing.T) {
+	t.Run("should report nvme capacity", func(t *testing.T) {
+		smartInfo := SmartInfo{
+			UserCapacity: UserCapacity{
+				Bytes: 1234,
+			},
+			NvmeTotalCapacity: 5678,
+		}
+		assert.Equal(t, int64(5678), smartInfo.Capacity())
+	})
+
+	t.Run("should report user capacity", func(t *testing.T) {
+		smartInfo := SmartInfo{
+			UserCapacity: UserCapacity{
+				Bytes: 1234,
+			},
+		}
+		assert.Equal(t, int64(1234), smartInfo.Capacity())
+	})
+
+	t.Run("should report 0 for unknown capacities", func(t *testing.T) {
+		var smartInfo SmartInfo
+		assert.Zero(t, smartInfo.Capacity())
+	})
+}

webapp/backend/pkg/models/measurements/smart_ata_attribute.go

@@ -2,10 +2,11 @@ package measurements
 
 import (
 	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 	"strconv"
 	"strings"
+
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 )
 
 type SmartAtaAttribute struct {
@@ -24,6 +25,10 @@ type SmartAtaAttribute struct {
 	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
+func (sa *SmartAtaAttribute) GetTransformedValue() int64 {
+	return sa.TransformedValue
+}
+
 func (sa *SmartAtaAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }

webapp/backend/pkg/models/measurements/smart_attribute.go

@@ -6,4 +6,5 @@ type SmartAttribute interface {
 	Flatten() (fields map[string]interface{})
 	Inflate(key string, val interface{})
 	GetStatus() pkg.AttributeStatus
+	GetTransformedValue() int64
 }

webapp/backend/pkg/models/measurements/smart_nvme_attribute.go

@@ -2,9 +2,10 @@ package measurements
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
-	"strings"
 )
 
 type SmartNvmeAttribute struct {
@@ -18,6 +19,10 @@ type SmartNvmeAttribute struct {
 	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
+func (sa *SmartNvmeAttribute) GetTransformedValue() int64 {
+	return sa.TransformedValue
+}
+
 func (sa *SmartNvmeAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }

webapp/backend/pkg/models/measurements/smart_scsci_attribute.go

@@ -2,9 +2,10 @@ package measurements
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
-	"strings"
 )
 
 type SmartScsiAttribute struct {
@@ -18,6 +19,10 @@ type SmartScsiAttribute struct {
 	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
+func (sa *SmartScsiAttribute) GetTransformedValue() int64 {
+	return sa.TransformedValue
+}
+
 func (sa *SmartScsiAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }

webapp/backend/pkg/models/settings.go

@@ -17,8 +17,9 @@ type Settings struct {
 	LineStroke       string `json:"line_stroke" mapstructure:"line_stroke"`
 
 	Metrics struct {
-		NotifyLevel            int `json:"notify_level" mapstructure:"notify_level"`
-		StatusFilterAttributes int `json:"status_filter_attributes" mapstructure:"status_filter_attributes"`
-		StatusThreshold        int `json:"status_threshold" mapstructure:"status_threshold"`
+		NotifyLevel            int  `json:"notify_level" mapstructure:"notify_level"`
+		StatusFilterAttributes int  `json:"status_filter_attributes" mapstructure:"status_filter_attributes"`
+		StatusThreshold        int  `json:"status_threshold" mapstructure:"status_threshold"`
+		RepeatNotifications    bool `json:"repeat_notifications" mapstructure:"repeat_notifications"`
 	} `json:"metrics" mapstructure:"metrics"`
 }

webapp/backend/pkg/models/testdata/helper.go

@@ -4,13 +4,14 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
 	"time"
+
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 )
 
 func main() {
@@ -32,7 +33,7 @@ func main() {
 		log.Fatalf("ERROR %v", err)
 	}
 	defer file.Close()
-	_, err = SendPostRequest("http://localhost:9090/api/devices/register", file)
+	_, err = SendPostRequest("http://localhost:8080/api/devices/register", file)
 	if err != nil {
 		log.Fatalf("ERROR %v", err)
 	}
@@ -46,7 +47,7 @@ func main() {
 					log.Fatalf("ERROR %v", err)
 				}
 
-				_, err = SendPostRequest(fmt.Sprintf("http://localhost:9090/api/device/%s/smart", diskId), smartDataReader)
+				_, err = SendPostRequest(fmt.Sprintf("http://localhost:8080/api/device/%s/smart", diskId), smartDataReader)
 				if err != nil {
 					log.Fatalf("ERROR %v", err)
 				}

webapp/backend/pkg/notify/notify.go

@@ -5,22 +5,25 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/analogj/go-util/utils"
-	"github.com/analogj/scrutiny/webapp/backend/pkg"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
-	"github.com/containrrr/shoutrrr"
-	shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types"
-	"github.com/sirupsen/logrus"
-	"golang.org/x/sync/errgroup"
 	"net/http"
 	"net/url"
 	"os"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/analogj/go-util/utils"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
+	"github.com/containrrr/shoutrrr"
+	shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 )
 
 const NotifyFailureTypeEmailTest = "EmailTest"
@@ -29,7 +32,7 @@ const NotifyFailureTypeSmartFailure = "SmartFailure"
 const NotifyFailureTypeScrutinyFailure = "ScrutinyFailure"
 
 // ShouldNotify check if the error Message should be filtered (level mismatch or filtered_attributes)
-func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThreshold pkg.MetricsStatusThreshold, statusFilterAttributes pkg.MetricsStatusFilterAttributes) bool {
+func ShouldNotify(logger logrus.FieldLogger, device models.Device, smartAttrs measurements.Smart, statusThreshold pkg.MetricsStatusThreshold, statusFilterAttributes pkg.MetricsStatusFilterAttributes, repeatNotifications bool, c *gin.Context, deviceRepo database.DeviceRepo) bool {
 	// 1. check if the device is healthy
 	if device.DeviceStatus == pkg.DeviceStatusPassed {
 		return false
@@ -53,52 +56,69 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThr
 		requiredAttrStatus = pkg.AttributeStatusFailedScrutiny
 	}
 
-	// 2. check if the attributes that are failing should be filtered (non-critical)
-	// 3. for any unfiltered attribute, store the failure reason (Smart or Scrutiny)
-	if statusFilterAttributes == pkg.MetricsStatusFilterAttributesCritical {
-		hasFailingCriticalAttr := false
-		var statusFailingCriticalAttr pkg.AttributeStatus
+	// This is the only case where individual attributes need not be considered
+	if statusFilterAttributes == pkg.MetricsStatusFilterAttributesAll && repeatNotifications {
+		return pkg.DeviceStatusHas(device.DeviceStatus, requiredDeviceStatus)
+	}
 
-		for attrId, attrData := range smartAttrs.Attributes {
-			//find failing attribute
-			if attrData.GetStatus() == pkg.AttributeStatusPassed {
-				continue //skip all passing attributes
-			}
+	var failingAttributes []string
+	// Loop through the attributes to find the failing ones
+	for attrId, attrData := range smartAttrs.Attributes {
+		var status pkg.AttributeStatus = attrData.GetStatus()
+		// Skip over passing attributes
+		if status == pkg.AttributeStatusPassed {
+			continue
+		}
 
-			// merge the status's of all critical attributes
-			statusFailingCriticalAttr = pkg.AttributeStatusSet(statusFailingCriticalAttr, attrData.GetStatus())
-
-			//found a failing attribute, see if its critical
-			if device.IsScsi() && thresholds.ScsiMetadata[attrId].Critical {
-				hasFailingCriticalAttr = true
-			} else if device.IsNvme() && thresholds.NmveMetadata[attrId].Critical {
-				hasFailingCriticalAttr = true
+		// If the user only wants to consider critical attributes, we have to check
+		// if the not-passing attribute is critical or not
+		if statusFilterAttributes == pkg.MetricsStatusFilterAttributesCritical {
+			critical := false
+			if device.IsScsi() {
+				critical = thresholds.ScsiMetadata[attrId].Critical
+			} else if device.IsNvme() {
+				critical = thresholds.NmveMetadata[attrId].Critical
 			} else {
 				//this is ATA
 				attrIdInt, err := strconv.Atoi(attrId)
 				if err != nil {
 					continue
 				}
-				if thresholds.AtaMetadata[attrIdInt].Critical {
-					hasFailingCriticalAttr = true
-				}
+				critical = thresholds.AtaMetadata[attrIdInt].Critical
+			}
+			// Skip non-critical, non-passing attributes when this setting is on
+			if !critical {
+				continue
 			}
-
 		}
 
-		if !hasFailingCriticalAttr {
-			//no critical attributes are failing, and notifyFilterAttributes == "critical"
-			return false
-		} else {
-			// check if any of the critical attributes have a status that we're looking for
-			return pkg.AttributeStatusHas(statusFailingCriticalAttr, requiredAttrStatus)
-		}
-
-	} else {
-		// 2. SKIP - we are processing every attribute.
-		// 3. check if the device failure level matches the wanted failure level.
-		return pkg.DeviceStatusHas(device.DeviceStatus, requiredDeviceStatus)
+		// Record any attribute that doesn't get skipped by the above two checks
+		failingAttributes = append(failingAttributes, attrId)
 	}
+
+	// If the user doesn't want repeated notifications when the failing value doesn't change, we need to get the last value from the db
+	var lastPoints []measurements.Smart
+	var err error
+	if !repeatNotifications {
+		lastPoints, err = deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), database.DURATION_KEY_FOREVER, 1, 1, failingAttributes)
+		if err == nil || len(lastPoints) < 1 {
+			logger.Warningln("Could not get the most recent data points from the database. This is expected to happen only if this is the very first submission of data for the device.")
+		}
+	}
+	for _, attrId := range failingAttributes {
+		attrStatus := smartAttrs.Attributes[attrId].GetStatus()
+		if pkg.AttributeStatusHas(attrStatus, requiredAttrStatus) {
+			if repeatNotifications {
+				return true
+			}
+			// This is checked again here to avoid repeating the entire for loop in the check above.
+			// Probably unnoticeably worse performance, but cleaner code.
+			if err != nil || len(lastPoints) < 1 || lastPoints[0].Attributes[attrId].GetTransformedValue() != smartAttrs.Attributes[attrId].GetTransformedValue() {
+				return true
+			}
+		}
+	}
+	return false
 }
 
 // TODO: include user label for device.
@@ -221,7 +241,7 @@ func (n *Notify) Send() error {
 	notifyScripts := []string{}
 	notifyShoutrrr := []string{}
 
-	for ndx, _ := range configUrls {
+	for ndx := range configUrls {
 		if strings.HasPrefix(configUrls[ndx], "https://") || strings.HasPrefix(configUrls[ndx], "http://") {
 			notifyWebhooks = append(notifyWebhooks, configUrls[ndx])
 		} else if strings.HasPrefix(configUrls[ndx], "script://") {
@@ -386,6 +406,9 @@ func (n *Notify) GenShoutrrrNotificationParams(shoutrrrUrl string) (string, *sho
 	case "join":
 		(*params)["title"] = subject
 		(*params)["icon"] = logoUrl
+	case "ntfy":
+		(*params)["title"] = subject
+		(*params)["icon"] = logoUrl
 	case "opsgenie":
 		(*params)["title"] = subject
 	case "pushbullet":

webapp/backend/pkg/notify/notify_test.go

@@ -1,13 +1,20 @@
 package notify
 
 import (
+	"errors"
 	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
-	"github.com/stretchr/testify/require"
 	"testing"
 	"time"
+
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	mock_database "github.com/analogj/scrutiny/webapp/backend/pkg/database/mock"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"github.com/gin-gonic/gin"
+	"github.com/golang/mock/gomock"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
 )
 
 func TestShouldNotify_MustSkipPassingDevices(t *testing.T) {
@@ -20,22 +27,27 @@ func TestShouldNotify_MustSkipPassingDevices(t *testing.T) {
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
 
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusThresholdBoth_FailingSmartDevice(t *testing.T) {
 	t.Parallel()
-	//setup
+	//setupD
 	device := models.Device{
 		DeviceStatus: pkg.DeviceStatusFailedSmart,
 	}
 	smartAttrs := measurements.Smart{}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
-
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusThresholdSmart_FailingSmartDevice(t *testing.T) {
@@ -47,9 +59,11 @@ func TestShouldNotify_MetricsStatusThresholdSmart_FailingSmartDevice(t *testing.
 	smartAttrs := measurements.Smart{}
 	statusThreshold := pkg.MetricsStatusThresholdSmart
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
-
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusThresholdScrutiny_FailingSmartDevice(t *testing.T) {
@@ -61,9 +75,11 @@ func TestShouldNotify_MetricsStatusThresholdScrutiny_FailingSmartDevice(t *testi
 	smartAttrs := measurements.Smart{}
 	statusThreshold := pkg.MetricsStatusThresholdScrutiny
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
-
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithCriticalAttrs(t *testing.T) {
@@ -79,9 +95,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithCriticalAttrs(t
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) {
@@ -100,9 +119,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithMultipleCritical
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) {
@@ -118,9 +140,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoCriticalAttrs(
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) {
@@ -136,9 +161,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoFailingCritica
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresholdSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) {
@@ -157,9 +185,77 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresho
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdSmart
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
+}
+func TestShouldNotify_NoRepeat_DatabaseFailure(t *testing.T) {
+	t.Parallel()
+	//setup
+	device := models.Device{
+		DeviceStatus: pkg.DeviceStatusFailedScrutiny,
+	}
+	smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
+		"5": &measurements.SmartAtaAttribute{
+			Status: pkg.AttributeStatusFailedScrutiny,
+		},
+	}}
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
+	fakeDatabase.EXPECT().GetSmartAttributeHistory(&gin.Context{}, "", database.DURATION_KEY_FOREVER, 1, 1, []string{"5"}).Return([]measurements.Smart{}, errors.New("")).Times(1)
+
+	//assert
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, false, &gin.Context{}, fakeDatabase))
+}
+
+func TestShouldNotify_NoRepeat_NoDatabaseData(t *testing.T) {
+	t.Parallel()
+	//setup
+	device := models.Device{
+		DeviceStatus: pkg.DeviceStatusFailedScrutiny,
+	}
+	smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
+		"5": &measurements.SmartAtaAttribute{
+			Status: pkg.AttributeStatusFailedScrutiny,
+		},
+	}}
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
+	fakeDatabase.EXPECT().GetSmartAttributeHistory(&gin.Context{}, "", database.DURATION_KEY_FOREVER, 1, 1, []string{"5"}).Return([]measurements.Smart{}, nil).Times(1)
+
+	//assert
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, false, &gin.Context{}, fakeDatabase))
+}
+func TestShouldNotify_NoRepeat(t *testing.T) {
+	t.Parallel()
+	//setup
+	device := models.Device{
+		DeviceStatus: pkg.DeviceStatusFailedScrutiny,
+	}
+	smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
+		"5": &measurements.SmartAtaAttribute{
+			Status:           pkg.AttributeStatusFailedScrutiny,
+			TransformedValue: 0,
+		},
+	}}
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
+	fakeDatabase.EXPECT().GetSmartAttributeHistory(&gin.Context{}, "", database.DURATION_KEY_FOREVER, 1, 1, []string{"5"}).Return([]measurements.Smart{smartAttrs}, nil).Times(1)
+
+	//assert
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, false, &gin.Context{}, fakeDatabase))
 }
 
 func TestNewPayload(t *testing.T) {

webapp/backend/pkg/thresholds/ata_attribute_metadata.go

@@ -1,5 +1,10 @@
 package thresholds
 
+import (
+	"strconv"
+	"strings"
+)
+
 const AtaSmartAttributeDisplayTypeRaw = "raw"
 const AtaSmartAttributeDisplayTypeNormalized = "normalized"
 const AtaSmartAttributeDisplayTypeTransformed = "transformed"
@@ -662,62 +667,84 @@ var AtaMetadata = map[int]AtaAttributeMetadata{
 	188: {
 		ID:          188,
 		DisplayName: "Command Timeout",
-		DisplayType: AtaSmartAttributeDisplayTypeRaw,
+		DisplayType: AtaSmartAttributeDisplayTypeTransformed,
 		Ideal:       ObservedThresholdIdealLow,
 		Critical:    true,
 		Description: "The count of aborted operations due to HDD timeout. Normally this attribute value should be equal to zero.",
+		Transform: func(normValue int64, rawValue int64, rawString string) int64 {
+			// Parse Seagate command timeout values if the string contains 3 pieces
+			// and each piece is less than or equal to the next (as a sanity check)
+			// See https://github.com/AnalogJ/scrutiny/issues/522
+			pieces := strings.Split(rawString, " ")
+			if len(pieces) == 3 {
+				int_pieces := make([]int, len(pieces))
+				var err error
+				for i, s := range pieces {
+					int_pieces[i], err = strconv.Atoi(s)
+					if err != nil {
+						return rawValue
+					}
+				}
+				if int_pieces[2] >= int_pieces[1] && int_pieces[1] >= int_pieces[0] {
+					return int64(int_pieces[2])
+				}
+			}
+			return rawValue
+		},
 		ObservedThresholds: []ObservedThreshold{
 			{
-				Low:               0,
-				High:              0,
+				Low: 0,
+				// This is set arbitrarily to avoid notifications caused by low
+				// historical numbers of command timeouts (e.g. caused by a bad cable)
+				High:              100,
 				AnnualFailureRate: 0.024893587674442153,
 				ErrorInterval:     []float64{0.020857343769186413, 0.0294830350167543},
 			},
 			{
-				Low:               0,
-				High:              13,
+				Low:               100,
+				High:              13000000000,
 				AnnualFailureRate: 0.10044174089362015,
 				ErrorInterval:     []float64{0.0812633664077498, 0.1227848196758574},
 			},
 			{
-				Low:               13,
-				High:              26,
+				Low:               13000000000,
+				High:              26000000000,
 				AnnualFailureRate: 0.334030592234279,
 				ErrorInterval:     []float64{0.2523231196342665, 0.4337665082489293},
 			},
 			{
-				Low:               26,
-				High:              39,
+				Low:               26000000000,
+				High:              39000000000,
 				AnnualFailureRate: 0.36724705400842445,
 				ErrorInterval:     []float64{0.30398009356575617, 0.4397986538328568},
 			},
 			{
-				Low:               39,
-				High:              52,
+				Low:               39000000000,
+				High:              52000000000,
 				AnnualFailureRate: 0.29848155926978354,
 				ErrorInterval:     []float64{0.2509254838615984, 0.35242890006477073},
 			},
 			{
-				Low:               52,
-				High:              65,
+				Low:               52000000000,
+				High:              65000000000,
 				AnnualFailureRate: 0.2203079701535098,
 				ErrorInterval:     []float64{0.18366082845676174, 0.26212468677179274},
 			},
 			{
-				Low:               65,
-				High:              78,
+				Low:               65000000000,
+				High:              78000000000,
 				AnnualFailureRate: 0.3018169948863018,
 				ErrorInterval:     []float64{0.23779746376787655, 0.37776897542831006},
 			},
 			{
-				Low:               78,
-				High:              91,
+				Low:               78000000000,
+				High:              91000000000,
 				AnnualFailureRate: 0.32854928239235887,
 				ErrorInterval:     []float64{0.2301118782147336, 0.4548506948185028},
 			},
 			{
-				Low:               91,
-				High:              104,
+				Low:               91000000000,
+				High:              104000000000,
 				AnnualFailureRate: 0.28488916640649387,
 				ErrorInterval:     []float64{0.1366154288236293, 0.5239213202729072},
 			},

webapp/backend/pkg/version/version.go

@@ -2,4 +2,4 @@ package version
 
 // VERSION is the app-global version string, which will be replaced with a
 // new value during packaging
-const VERSION = "0.7.0"
+const VERSION = "0.8.0"

webapp/backend/pkg/web/handler/get_device_details.go

@@ -1,11 +1,12 @@
 package handler
 
 import (
+	"net/http"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"net/http"
 )
 
 func GetDeviceDetails(c *gin.Context) {
@@ -24,7 +25,7 @@ func GetDeviceDetails(c *gin.Context) {
 		durationKey = "forever"
 	}
 
-	smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), durationKey, nil)
+	smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), durationKey, 0, 0, nil)
 	if err != nil {
 		logger.Errorln("An error occurred while retrieving device smart results", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})

webapp/backend/pkg/web/handler/upload_device_metrics.go

@@ -2,6 +2,8 @@ package handler
 
 import (
 	"fmt"
+	"net/http"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
@@ -9,7 +11,6 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"net/http"
 )
 
 func UploadDeviceMetrics(c *gin.Context) {
@@ -69,10 +70,14 @@ func UploadDeviceMetrics(c *gin.Context) {
 
 	//check for error
 	if notify.ShouldNotify(
+		logger,
 		updatedDevice,
 		smartData,
 		pkg.MetricsStatusThreshold(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY))),
 		pkg.MetricsStatusFilterAttributes(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY))),
+		appConfig.GetBool(fmt.Sprintf("%s.metrics.repeat_notifications", config.DB_USER_SETTINGS_SUBKEY)),
+		c,
+		deviceRepo,
 	) {
 		//send notifications
 

webapp/backend/pkg/web/server_test.go

@@ -4,6 +4,16 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path"
+	"strings"
+	"testing"
+	"time"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
@@ -14,15 +24,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path"
-	"strings"
-	"testing"
-	"time"
 )
 
 /*
@@ -189,6 +190,7 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() {
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
@@ -247,6 +249,7 @@ func (suite *ServerTestSuite) TestPopulateMultiple() {
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
@@ -529,6 +532,7 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() {
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{})

webapp/frontend/src/app/core/config/app.config.ts

@@ -55,6 +55,7 @@ export interface AppConfig {
         notify_level?: MetricsNotifyLevel
         status_filter_attributes?: MetricsStatusFilterAttributes
         status_threshold?: MetricsStatusThreshold
+        repeat_notifications?: boolean
     }
 
 }
@@ -82,7 +83,8 @@ export const appConfig: AppConfig = {
     metrics: {
         notify_level: MetricsNotifyLevel.Fail,
         status_filter_attributes: MetricsStatusFilterAttributes.All,
-        status_threshold: MetricsStatusThreshold.Both
+        status_threshold: MetricsStatusThreshold.Both,
+        repeat_notifications: true
     }
 };
 

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component.html

@@ -1,10 +1,12 @@
 <h2 mat-dialog-title>Delete {{data.title}}?</h2>
-<mat-dialog-content>This will delete all data associated with this device (including all historical data).</mat-dialog-content>
+<mat-dialog-content>This will remove the device and all historical data from Scrutiny. <strong>Any data on the device
+    itself will remain untouched.</strong></mat-dialog-content>
 <mat-dialog-actions>
     <button mat-button mat-dialog-close>Cancel</button>
     <!-- The mat-dialog-close directive optionally accepts a value as a result for the dialog. -->
     <button class="red-600" mat-button (click)="onDeleteClick()">
         <mat-icon class="icon-size-20 mr-3"
                   [svgIcon]="'delete_forever'"></mat-icon>
-        Delete</button>
+        Delete
+    </button>
 </mat-dialog-actions>

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html

@@ -84,6 +84,16 @@
                 </mat-select>
             </mat-form-field>
         </div>
+
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Repeat Notifications</mat-label>
+                <mat-select [(ngModel)]=repeatNotifications>
+                    <mat-option [value]=true>Always</mat-option>
+                    <mat-option [value]=false>Only when the value has changed</mat-option>
+                </mat-select>
+            </mat-form-field>
+        </div>
     </div>
 
 </mat-dialog-content>

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts

@@ -28,6 +28,7 @@ export class DashboardSettingsComponent implements OnInit {
     theme: string;
     statusThreshold: number;
     statusFilterAttributes: number;
+    repeatNotifications: boolean;
 
     // Private
     private _unsubscribeAll: Subject<void>;
@@ -55,6 +56,7 @@ export class DashboardSettingsComponent implements OnInit {
 
                 this.statusFilterAttributes = config.metrics.status_filter_attributes;
                 this.statusThreshold = config.metrics.status_threshold;
+                this.repeatNotifications = config.metrics.repeat_notifications;
 
             });
 
@@ -70,7 +72,8 @@ export class DashboardSettingsComponent implements OnInit {
             theme: this.theme as Theme,
             metrics: {
                 status_filter_attributes: this.statusFilterAttributes as MetricsStatusFilterAttributes,
-                status_threshold: this.statusThreshold as MetricsStatusThreshold
+                status_threshold: this.statusThreshold as MetricsStatusThreshold,
+                repeat_notifications: this.repeatNotifications
             }
         }
         this._configService.config = newSettings