Bump webpack-dev-middleware and @angular-devkit/build-angular

Bumps [webpack-dev-middleware](https://github.com/webpack/webpack-dev-middleware) to 6.1.2 and updates ancestor dependency [@angular-devkit/build-angular](https://github.com/angular/angular-cli). These dependencies need to be updated together.


Updates `webpack-dev-middleware` from 5.3.0 to 6.1.2
- [Release notes](https://github.com/webpack/webpack-dev-middleware/releases)
- [Changelog](https://github.com/webpack/webpack-dev-middleware/blob/v6.1.2/CHANGELOG.md)
- [Commits](https://github.com/webpack/webpack-dev-middleware/compare/v5.3.0...v6.1.2)

Updates `@angular-devkit/build-angular` from 13.3.11 to 17.3.2
- [Release notes](https://github.com/angular/angular-cli/releases)
- [Changelog](https://github.com/angular/angular-cli/blob/main/CHANGELOG.md)
- [Commits](https://github.com/angular/angular-cli/compare/13.3.11...17.3.2)

---
updated-dependencies:
- dependency-name: webpack-dev-middleware
  dependency-type: indirect
- dependency-name: "@angular-devkit/build-angular"
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/workflows/ci.yaml

@@ -13,7 +13,7 @@ jobs:
         run: |
           make binary-frontend-test-coverage
       - name: Upload coverage
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage
           path: ${{ github.workspace }}/webapp/frontend/coverage/lcov.info
@@ -49,7 +49,7 @@ jobs:
         run: |
           make binary-clean binary-test-coverage
       - name: Upload coverage
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage
           path: ${{ github.workspace }}/coverage.txt
@@ -64,12 +64,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Download coverage reports
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: coverage
       - name: Upload coverage reports
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v5
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: ${{ github.workspace }}/coverage.txt,${{ github.workspace }}/lcov.info
           flags: unittests
           fail_ci_if_error: true
@@ -106,9 +107,9 @@ jobs:
         run: |
           make binary-clean binary-all
       - name: Archive
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
-          name: binaries.zip
+          name: binaries-${{ matrix.cfg.on }}-${{ matrix.cfg.goos }}-${{ matrix.cfg.goarch }}-${{ matrix.cfg.goarm || 'na' }}.zip
           path: |
             scrutiny-web-*
             scrutiny-collector-metrics-*

.github/workflows/docker-build.yaml

@@ -46,7 +46,9 @@ jobs:
             latest=false
           tags: |
             type=ref,enable=true,event=branch,suffix=-collector
-            type=ref,enable=true,event=tag,suffix=-collector
+            type=semver,pattern=v{{major}}.{{minor}}.{{patch}},suffix=-collector
+            type=semver,pattern=v{{major}}.{{minor}},suffix=-collector
+            type=semver,pattern=v{{major}},suffix=-collector
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       # Build and push Docker image with Buildx (don't push on PR)
@@ -99,7 +101,9 @@ jobs:
             latest=false
           tags: |
             type=ref,enable=true,event=branch,suffix=-web
-            type=ref,enable=true,event=tag,suffix=-web
+            type=semver,pattern=v{{major}}.{{minor}}.{{patch}},suffix=-web
+            type=semver,pattern=v{{major}}.{{minor}},suffix=-web
+            type=semver,pattern=v{{major}},suffix=-web
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
@@ -148,7 +152,9 @@ jobs:
         with:
           tags: |
             type=ref,enable=true,event=branch,suffix=-omnibus
-            type=ref,enable=true,event=tag,suffix=-omnibus
+            type=semver,pattern=v{{major}}.{{minor}}.{{patch}},suffix=-omnibus
+            type=semver,pattern=v{{major}}.{{minor}},suffix=-omnibus
+            type=semver,pattern=v{{major}},suffix=-omnibus
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action

.github/workflows/release.yaml

@@ -61,7 +61,7 @@ jobs:
         with:
           version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
       - name: Upload workspace
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: workspace
           path: ${{ github.workspace }}/**/*
@@ -91,7 +91,7 @@ jobs:
           - { on: windows-latest, goos: windows,    goarch: arm64 }
     steps:
       - name: Download workspace
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: workspace
       - uses: actions/setup-go@v3
@@ -101,7 +101,7 @@ jobs:
         run: |
           make binary-clean binary-all
       - name: Archive
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: binaries.zip
           path: |
@@ -114,11 +114,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download workspace
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: workspace
       - name: Download binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: binaries.zip
       - name: List

.gitignore

@@ -65,4 +65,6 @@ scrutiny_test.db
 scrutiny.yaml
 coverage.txt
 /config
-/influxdb
+/influxdb
+.angular
+web.log

AI_POLICY.md

@@ -0,0 +1,73 @@
+# AI Usage Policy
+
+scrutiny has strict rules for AI usage:
+
+- **All AI usage in any form must be disclosed.** You must state
+  the tool you used (e.g. Claude Code, Cursor, Amp) along with
+  the extent that the work was AI-assisted.
+
+- **Pull requests created in any way by AI can only be for accepted issues.**
+  Drive-by pull requests that do not reference an accepted issue will be
+  closed. If AI isn't disclosed but a maintainer suspects its use, the
+  PR will be closed. If you want to share code for a non-accepted issue,
+  open a discussion or attach it to an existing discussion.
+
+- **Pull requests created by AI must have been fully verified with
+  human use.** AI must not create hypothetically correct code that
+  hasn't been tested. Importantly, you must not allow AI to write
+  code for platforms or environments you don't have access to manually
+  test on.
+
+- **Issues and discussions can use AI assistance but must have a full
+  human-in-the-loop.** This means that any content generated with AI
+  must have been reviewed _and edited_ by a human before submission.
+  AI is very good at being overly verbose and including noise that
+  distracts from the main point. Humans must do their research and
+  trim this down.
+
+- **No AI-generated media is allowed (art, images, videos, audio, etc.).**
+  Text and code are the only acceptable AI-generated content, per the
+  other rules in this policy.
+
+- **Bad AI drivers will be banned and ridiculed in public.** You've
+  been warned. We love to help junior developers learn and grow, but
+  if you're interested in that then don't use AI, and we'll help you.
+  I'm sorry that bad AI drivers have ruined this for you.
+
+These rules apply only to outside contributions to scrutiny. Maintainers
+and repeat contributors (with explicit permission) are exempt from these
+rules and may use AI tools at their discretion; they've proven themselves
+trustworthy to apply good judgment.
+
+## There are Humans Here
+
+Please remember that scrutiny is maintained by humans.
+
+Every discussion, issue, and pull request is read and reviewed by
+humans (and sometimes machines, too). It is a boundary point at which
+people interact with each other and the work done. It is rude and
+disrespectful to approach this boundary with low-effort, unqualified
+work, since it puts the burden of validation on the maintainer.
+
+In a perfect world, AI would produce high-quality, accurate work
+every time. But today, that reality depends on the driver of the AI.
+And today, most drivers of AI are just not good enough. So, until either
+the people get better, the AI gets better, or both, we have to have
+strict rules to protect maintainers.
+
+## AI is Welcome Here
+
+Many maintainers embrace AI tools as a productive tool in their workflow.
+As a project, scrutiny welcomes AI as a tool!
+
+**Our reason for the strict AI policy is not due to an anti-AI stance**, but
+instead due to the number of highly unqualified people using AI. It's the
+people, not the tools, that are the problem.
+
+This section is included to be transparent about the project's usage about
+AI for people who may disagree with it, and to address the misconception
+that this policy is anti-AI in nature.
+
+# Credit
+
+Adopted from [ghostty's AI policy](https://github.com/ghostty-org/ghostty/blob/1b7a15899ad40fba4ce020f537055d30eaf99ee8/AI_POLICY.md)

CONTRIBUTING.md

@@ -1,5 +1,7 @@
 # Contributing
 
+**Please see our [AI policy](./AI_POLICY.md).**
+
 The Scrutiny repository is a [monorepo](https://en.wikipedia.org/wiki/Monorepo) containing source code for:
 - Scrutiny Backend Server (API)
 - Scrutiny Frontend Angular SPA
@@ -161,7 +163,7 @@ docker cp scrutiny:/tmp/web.log web.log
 # Docker Development
 
 ```
-docker build -f docker/Dockerfile . -t chcr.io/analogj/scrutiny:master-omnibus
+docker build -f docker/Dockerfile . -t ghcr.io/analogj/scrutiny:master-omnibus
 docker run -it --rm -p 8080:8080 \
 -v /run/udev:/run/udev:ro \
 --cap-add SYS_RAWIO \

README.md

@@ -26,7 +26,7 @@ If you run a server with more than a couple of hard drives, you're probably alre
 
 > smartd is a daemon that monitors the Self-Monitoring, Analysis and Reporting Technology (SMART) system built into many ATA, IDE and SCSI-3 hard drives. The purpose of SMART is to monitor the reliability of the hard drive and predict drive failures, and to carry out different types of drive self-tests.
 
-Theses S.M.A.R.T hard drive self-tests can help you detect and replace failing hard drives before they cause permanent data loss. However, there's a couple issues with `smartd`:
+These S.M.A.R.T hard drive self-tests can help you detect and replace failing hard drives before they cause permanent data loss. However, there's a couple issues with `smartd`:
 
 - There are more than a hundred S.M.A.R.T attributes, however `smartd` does not differentiate between critical and informational metrics
 - `smartd` does not record S.M.A.R.T attribute history, so it can be hard to determine if an attribute is degrading slowly over time.
@@ -69,10 +69,10 @@ See [docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](./docs/TROUBLESHOOTING_DEVICE_COL
 
 If you're using Docker, getting started is as simple as running the following command:
 
-> See [docker/example.omnibus.docker-compose.yml](./docker/example.omnibus.docker-compose.yml) for a docker-compose file.
+> See [docker/example.omnibus.docker-compose.yml](https://github.com/AnalogJ/scrutiny/blob/master/docker/example.omnibus.docker-compose.yml) for a docker-compose file.
 
 ```bash
-docker run -it --rm -p 8080:8080 -p 8086:8086 \
+docker run -p 8080:8080 -p 8086:8086 --restart unless-stopped \
   -v `pwd`/scrutiny:/opt/scrutiny/config \
   -v `pwd`/influxdb2:/opt/scrutiny/influxdb \
   -v /run/udev:/run/udev:ro \
@@ -100,20 +100,20 @@ other Docker images:
 - `influxdb:2.2` - InfluxDB image, used by the Web container to persist SMART data. Only one container necessary
   See [docs/TROUBLESHOOTING_INFLUXDB.md](./docs/TROUBLESHOOTING_INFLUXDB.md)
 
-> See [docker/example.hubspoke.docker-compose.yml](./docker/example.hubspoke.docker-compose.yml) for a docker-compose file.
+> See [docker/example.hubspoke.docker-compose.yml](https://github.com/AnalogJ/scrutiny/blob/master/docker/example.hubspoke.docker-compose.yml) for a docker-compose file.
 
 ```bash
-docker run --rm -p 8086:8086 \
+docker run -p 8086:8086 --restart unless-stopped \
   -v `pwd`/influxdb2:/var/lib/influxdb2 \
   --name scrutiny-influxdb \
   influxdb:2.2
 
-docker run --rm -p 8080:8080 \
+docker run -p 8080:8080 --restart unless-stopped \
   -v `pwd`/scrutiny:/opt/scrutiny/config \
   --name scrutiny-web \
   ghcr.io/analogj/scrutiny:master-web
 
-docker run --rm \
+docker run --restart unless-stopped \
   -v /run/udev:/run/udev:ro \
   --cap-add SYS_RAWIO \
   --device=/dev/sda \
@@ -265,7 +265,8 @@ We use SemVer for versioning. For the versions available, see the tags on this r
 
 # Authors
 
-Jason Kulatunga - Initial Development - @AnalogJ
+* Jason Kulatunga - Initial Development - [@AnalogJ](https://github.com/AnalogJ/)
+* Aram Akhavan - Maintenence - [@kaysond](https://github.com/kaysond/)
 
 # Licenses
 

collector/pkg/collector/metrics.go

@@ -4,6 +4,12 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net/url"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
 	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/detect"
@@ -11,10 +17,6 @@ import (
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/samber/lo"
 	"github.com/sirupsen/logrus"
-	"net/url"
-	"os"
-	"os/exec"
-	"strings"
 )
 
 type MetricsCollector struct {
@@ -90,8 +92,9 @@ func (mc *MetricsCollector) Run() error {
 			//go mc.Collect(&wg, device.WWN, device.DeviceName, device.DeviceType)
 			mc.Collect(device.WWN, device.DeviceName, device.DeviceType)
 
-			// TODO: we may need to sleep for between each call to smartctl -a
-			//time.Sleep(30 * time.Millisecond)
+			if mc.config.GetInt("commands.metrics_smartctl_wait") > 0 {
+				time.Sleep(time.Duration(mc.config.GetInt("commands.metrics_smartctl_wait")) * time.Second)
+			}
 		}
 
 		//mc.logger.Infoln("Main: Waiting for workers to finish")
@@ -113,7 +116,7 @@ func (mc *MetricsCollector) Validate() error {
 	return nil
 }
 
-//func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
+// func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
 func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceType string) {
 	//defer wg.Done()
 	if len(deviceWWN) == 0 {

collector/pkg/config/config.go

@@ -20,7 +20,7 @@ import (
 type configuration struct {
 	*viper.Viper
 
-	deviceOverrides []models.ScanOverride
+	deviceOverrides    []models.ScanOverride
 }
 
 //Viper uses the following precedence order. Each item takes precedence over the item below it:
@@ -47,9 +47,17 @@ func (c *configuration) Init() error {
 	c.SetDefault("commands.metrics_scan_args", "--scan --json")
 	c.SetDefault("commands.metrics_info_args", "--info --json")
 	c.SetDefault("commands.metrics_smart_args", "--xall --json")
+	c.SetDefault("commands.metrics_smartctl_wait", 0)
 
+	//configure env variable parsing.
+	c.SetEnvPrefix("COLLECTOR")
+	c.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
+	c.AutomaticEnv()
+	
 	//c.SetDefault("collect.short.command", "-a -o on -S on")
 
+	c.SetDefault("allow_listed_devices", []string{})
+
 	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
 	c.SetConfigType("yaml")
 	//c.SetConfigName("drawbridge")
@@ -186,3 +194,18 @@ func (c *configuration) GetCommandMetricsSmartArgs(deviceName string) string {
 	}
 	return c.GetString("commands.metrics_smart_args")
 }
+
+func (c *configuration) IsAllowlistedDevice(deviceName string) bool {
+	allowList := c.GetStringSlice("allow_listed_devices")
+	if len(allowList) == 0 {
+		return true
+	}
+
+	for _, item := range allowList {
+		if item == deviceName {
+			return true
+		}
+	}
+
+	return false
+}

collector/pkg/config/config_test.go

@@ -144,3 +144,29 @@ func TestConfiguration_OverrideDeviceCommands_MetricsInfoArgs(t *testing.T) {
 	require.Equal(t, "--info --json", testConfig.GetCommandMetricsInfoArgs("/dev/sdb"))
 	//require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Commands: {MetricsInfoArgs: "--info --json -T "}}}, scanOverrides)
 }
+
+func TestConfiguration_DeviceAllowList(t *testing.T) {
+	t.Parallel()
+
+	t.Run("present", func(t *testing.T) {
+		testConfig, err := config.Create()
+		require.NoError(t, err)
+
+		require.NoError(t, testConfig.ReadConfig(path.Join("testdata", "allow_listed_devices_present.yaml")))
+
+		require.True(t, testConfig.IsAllowlistedDevice("/dev/sda"), "/dev/sda should be allow listed")
+		require.False(t, testConfig.IsAllowlistedDevice("/dev/sdc"), "/dev/sda should not be allow listed")
+	})
+
+	t.Run("missing", func(t *testing.T) {
+		testConfig, err := config.Create()
+		require.NoError(t, err)
+
+		// Really just any other config where the key is full missing
+		require.NoError(t, testConfig.ReadConfig(path.Join("testdata", "override_device_commands.yaml")))
+
+		// Anything should be allow listed if the key isnt there
+		require.True(t, testConfig.IsAllowlistedDevice("/dev/sda"), "/dev/sda should be allow listed")
+		require.True(t, testConfig.IsAllowlistedDevice("/dev/sdc"), "/dev/sda should be allow listed")
+	})
+}

collector/pkg/config/interface.go

@@ -25,4 +25,6 @@ type Interface interface {
 	GetDeviceOverrides() []models.ScanOverride
 	GetCommandMetricsInfoArgs(deviceName string) string
 	GetCommandMetricsSmartArgs(deviceName string) string
+
+	IsAllowlistedDevice(deviceName string) bool
 }

collector/pkg/config/mock/mock_config.go

@@ -175,6 +175,20 @@ func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
 }
 
+// IsAllowlistedDevice mocks base method.
+func (m *MockInterface) IsAllowlistedDevice(deviceName string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsAllowlistedDevice", deviceName)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsAllowlistedDevice indicates an expected call of IsAllowlistedDevice.
+func (mr *MockInterfaceMockRecorder) IsAllowlistedDevice(deviceName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAllowlistedDevice", reflect.TypeOf((*MockInterface)(nil).IsAllowlistedDevice), deviceName)
+}
+
 // IsSet mocks base method.
 func (m *MockInterface) IsSet(key string) bool {
 	m.ctrl.T.Helper()

collector/pkg/config/testdata/allow_listed_devices_present.yaml

@@ -0,0 +1,3 @@
+allow_listed_devices:
+- /dev/sda
+- /dev/sdb

collector/pkg/detect/detect.go

@@ -3,13 +3,14 @@ package detect
 import (
 	"encoding/json"
 	"fmt"
+	"os"
+	"strings"
+
 	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/sirupsen/logrus"
-	"os"
-	"strings"
 )
 
 type Detect struct {
@@ -47,7 +48,7 @@ func (d *Detect) SmartctlScan() ([]models.Device, error) {
 	return detectedDevices, nil
 }
 
-//updates a device model with information from smartctl --scan
+// updates a device model with information from smartctl --scan
 // It has a couple of issues however:
 // - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
 // - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
@@ -81,8 +82,9 @@ func (d *Detect) SmartCtlInfo(device *models.Device) error {
 	device.SerialNumber = availableDeviceInfo.SerialNumber
 	device.Firmware = availableDeviceInfo.FirmwareVersion
 	device.RotationSpeed = availableDeviceInfo.RotationRate
-	device.Capacity = availableDeviceInfo.UserCapacity.Bytes
+	device.Capacity = availableDeviceInfo.Capacity()
 	device.FormFactor = availableDeviceInfo.FormFactor.Name
+	device.DeviceType = availableDeviceInfo.Device.Type
 	device.DeviceProtocol = availableDeviceInfo.Device.Protocol
 	if len(availableDeviceInfo.Vendor) > 0 {
 		device.Manufacturer = availableDeviceInfo.Vendor
@@ -122,6 +124,11 @@ func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []mod
 
 		deviceFile := strings.ToLower(scannedDevice.Name)
 
+		// If the user has defined a device allow list, and this device isnt there, then ignore it
+		if !d.Config.IsAllowlistedDevice(deviceFile) {
+			continue
+		}
+
 		detectedDevice := models.Device{
 			HostId:     d.Config.GetString("host.id"),
 			DeviceType: scannedDevice.Type,

collector/pkg/detect/detect_test.go

@@ -1,19 +1,22 @@
 package detect_test
 
 import (
+	"os"
+	"strings"
+	"testing"
+
 	mock_shell "github.com/analogj/scrutiny/collector/pkg/common/shell/mock"
 	mock_config "github.com/analogj/scrutiny/collector/pkg/config/mock"
 	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/golang/mock/gomock"
 	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"io/ioutil"
-	"testing"
 )
 
 func TestDetect_SmartctlScan(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -21,9 +24,10 @@ func TestDetect_SmartctlScan(t *testing.T) {
 	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
 	fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
-	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_simple.json")
+	testScanResults, err := os.ReadFile("testdata/smartctl_scan_simple.json")
 	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
 
 	d := detect.Detect{
@@ -32,17 +36,17 @@ func TestDetect_SmartctlScan(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	scannedDevices, err := d.SmartctlScan()
 
-	//assert
+	// assert
 	require.NoError(t, err)
 	require.Equal(t, 7, len(scannedDevices))
 	require.Equal(t, "scsi", scannedDevices[0].DeviceType)
 }
 
 func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -50,9 +54,10 @@ func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
 	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
 	fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
-	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_megaraid.json")
+	testScanResults, err := os.ReadFile("testdata/smartctl_scan_megaraid.json")
 	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
 
 	d := detect.Detect{
@@ -61,20 +66,20 @@ func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	scannedDevices, err := d.SmartctlScan()
 
-	//assert
+	// assert
 	require.NoError(t, err)
 	require.Equal(t, 2, len(scannedDevices))
 	require.Equal(t, []models.Device{
-		models.Device{DeviceName: "bus/0", DeviceType: "megaraid,0"},
-		models.Device{DeviceName: "bus/0", DeviceType: "megaraid,1"},
+		{DeviceName: "bus/0", DeviceType: "megaraid,0"},
+		{DeviceName: "bus/0", DeviceType: "megaraid,1"},
 	}, scannedDevices)
 }
 
 func TestDetect_SmartctlScan_Nvme(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -82,9 +87,10 @@ func TestDetect_SmartctlScan_Nvme(t *testing.T) {
 	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
 	fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
-	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_nvme.json")
+	testScanResults, err := os.ReadFile("testdata/smartctl_scan_nvme.json")
 	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
 
 	d := detect.Detect{
@@ -93,19 +99,19 @@ func TestDetect_SmartctlScan_Nvme(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	scannedDevices, err := d.SmartctlScan()
 
-	//assert
+	// assert
 	require.NoError(t, err)
 	require.Equal(t, 1, len(scannedDevices))
 	require.Equal(t, []models.Device{
-		models.Device{DeviceName: "nvme0", DeviceType: "nvme"},
+		{DeviceName: "nvme0", DeviceType: "nvme"},
 	}, scannedDevices)
 }
 
 func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -113,6 +119,7 @@ func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
 	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
 	fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
 
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
@@ -129,16 +136,16 @@ func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, "sda", transformedDevices[0].DeviceName)
 	require.Equal(t, "scsi", transformedDevices[0].DeviceType)
 }
 
 func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -146,6 +153,7 @@ func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
 	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
 	fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
 
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
@@ -162,21 +170,22 @@ func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, []models.Device{}, transformedDevices)
 }
 
 func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
 	fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
 	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{
 		{
 			Device:     "/dev/bus/0",
@@ -187,7 +196,8 @@ func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
 			Device:     "/dev/twa0",
 			DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
 			Ignore:     false,
-		}})
+		},
+	})
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -203,15 +213,15 @@ func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, 12, len(transformedDevices))
 }
 
 func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -219,6 +229,7 @@ func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
 	fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
+	fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -234,17 +245,17 @@ func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, 1, len(transformedDevices))
 	require.Equal(t, "sat+megaraid", transformedDevices[0].DeviceType)
 }
 
 // test https://github.com/AnalogJ/scrutiny/issues/255#issuecomment-1164024126
 func TestDetect_TransformDetectedDevices_WithoutDeviceTypeOverride(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -252,6 +263,7 @@ func TestDetect_TransformDetectedDevices_WithoutDeviceTypeOverride(t *testing.T)
 	fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
 	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda"}})
+	fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -267,16 +279,16 @@ func TestDetect_TransformDetectedDevices_WithoutDeviceTypeOverride(t *testing.T)
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, 1, len(transformedDevices))
 	require.Equal(t, "scsi", transformedDevices[0].DeviceType)
 }
 
 func TestDetect_TransformDetectedDevices_WhenDeviceNotDetected(t *testing.T) {
-	//setup
+	// setup
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
@@ -290,10 +302,109 @@ func TestDetect_TransformDetectedDevices_WhenDeviceNotDetected(t *testing.T) {
 		Config: fakeConfig,
 	}
 
-	//test
+	// test
 	transformedDevices := d.TransformDetectedDevices(detectedDevices)
 
-	//assert
+	// assert
 	require.Equal(t, 1, len(transformedDevices))
 	require.Equal(t, "ata", transformedDevices[0].DeviceType)
 }
+
+func TestDetect_TransformDetectedDevices_AllowListFilters(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
+	fakeConfig.EXPECT().IsAllowlistedDevice("/dev/sda").Return(true)
+	fakeConfig.EXPECT().IsAllowlistedDevice("/dev/sdb").Return(false)
+	detectedDevices := models.Scan{
+		Devices: []models.ScanDevice{
+			{
+				Name:     "/dev/sda",
+				InfoName: "/dev/sda",
+				Protocol: "ata",
+				Type:     "ata",
+			},
+			{
+				Name:     "/dev/sdb",
+				InfoName: "/dev/sdb",
+				Protocol: "ata",
+				Type:     "ata",
+			},
+		},
+	}
+
+	d := detect.Detect{
+		Config: fakeConfig,
+	}
+
+	// test
+	transformedDevices := d.TransformDetectedDevices(detectedDevices)
+
+	// assert
+	require.Equal(t, 1, len(transformedDevices))
+	require.Equal(t, "sda", transformedDevices[0].DeviceName)
+}
+
+func TestDetect_SmartCtlInfo(t *testing.T) {
+	t.Run("should report nvme info", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+
+		const (
+			someArgs = "--info --json"
+
+			// device info
+			someDeviceName           = "some-device-name"
+			someModelName            = "KCD61LUL3T84"
+			someSerialNumber         = "61Q0A05UT7B8"
+			someFirmware             = "8002"
+			someDeviceProtocol       = "NVMe"
+			someDeviceType           = "nvme"
+			someCapacity       int64 = 3840755982336
+		)
+
+		fakeConfig := mock_config.NewMockInterface(ctrl)
+		fakeConfig.EXPECT().
+			GetCommandMetricsInfoArgs("/dev/" + someDeviceName).
+			Return(someArgs)
+		fakeConfig.EXPECT().
+			GetString("commands.metrics_smartctl_bin").
+			Return("smartctl")
+
+		someLogger := logrus.WithFields(logrus.Fields{})
+
+		smartctlInfoResults, err := os.ReadFile("testdata/smartctl_info_nvme.json")
+		require.NoError(t, err)
+
+		fakeShell := mock_shell.NewMockInterface(ctrl)
+		fakeShell.EXPECT().
+			Command(someLogger, "smartctl", append(strings.Split(someArgs, " "), "/dev/"+someDeviceName), "", gomock.Any()).
+			Return(string(smartctlInfoResults), err)
+
+		d := detect.Detect{
+			Logger: someLogger,
+			Shell:  fakeShell,
+			Config: fakeConfig,
+		}
+
+		someDevice := &models.Device{
+			WWN:        "some wwn",
+			DeviceName: someDeviceName,
+		}
+
+		require.NoError(t, d.SmartCtlInfo(someDevice))
+
+		assert.Equal(t, someDeviceName, someDevice.DeviceName)
+		assert.Equal(t, someModelName, someDevice.ModelName)
+		assert.Equal(t, someSerialNumber, someDevice.SerialNumber)
+		assert.Equal(t, someFirmware, someDevice.Firmware)
+		assert.Equal(t, someDeviceProtocol, someDevice.DeviceProtocol)
+		assert.Equal(t, someDeviceType, someDevice.DeviceType)
+		assert.Equal(t, someCapacity, someDevice.Capacity)
+	})
+}

collector/pkg/detect/testdata/smartctl_info_nvme.json

@@ -0,0 +1,48 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      2
+    ],
+    "svn_revision": "5155",
+    "platform_info": "x86_64-linux-6.1.69-talos",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "--info",
+      "--json",
+      "/dev/nvme4"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/nvme4",
+    "info_name": "/dev/nvme4",
+    "type": "nvme",
+    "protocol": "NVMe"
+  },
+  "model_name": "KCD61LUL3T84",
+  "serial_number": "61Q0A05UT7B8",
+  "firmware_version": "8002",
+  "nvme_pci_vendor": {
+    "id": 7695,
+    "subsystem_id": 7695
+  },
+  "nvme_ieee_oui_identifier": 9233294,
+  "nvme_total_capacity": 3840755982336,
+  "nvme_unallocated_capacity": 0,
+  "nvme_controller_id": 1,
+  "nvme_version": {
+    "string": "1.4",
+    "value": 66560
+  },
+  "nvme_number_of_namespaces": 16,
+  "local_time": {
+    "time_t": 1706045146,
+    "asctime": "Tue Jan 23 21:25:46 2024 UTC"
+  }
+}

docker/Dockerfile

@@ -12,22 +12,28 @@ RUN make binary-frontend
 
 
 ######## Build the backend
-FROM golang:1.20-bullseye as backendbuild
+FROM golang:1.20-bookworm as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 COPY --link . /go/src/github.com/analogj/scrutiny
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
+    apt-get install -y --no-install-recommends \
+    file \
+    && rm -rf /var/lib/apt/lists/*
 RUN make binary-clean binary-all WEB_BINARY_NAME=scrutiny
 
 
 ######## Combine build artifacts in runtime image
-FROM debian:bullseye-slim as runtime
+FROM debian:bookworm-slim as runtime
 ARG TARGETARCH
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 ENV INFLUXD_CONFIG_PATH=/opt/scrutiny/influxdb
-ENV S6VER="1.21.8.0"
+ENV S6VER="3.1.6.2"
 ENV INFLUXVER="2.2.0"
+ENV S6_SERVICES_READYTIME=1000
+SHELL ["/usr/bin/sh", "-c"]
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
     apt-get install -y --no-install-recommends \
@@ -36,17 +42,23 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
     curl \
     smartmontools \
     tzdata \
+    procps \
+    xz-utils \
+    && rm -rf /var/lib/apt/lists/* \
     && update-ca-certificates \
-    &&  case ${TARGETARCH} in \
-            "amd64")  S6_ARCH=amd64  ;; \
-            "arm64")  S6_ARCH=aarch64  ;; \
-        esac \
-    && curl https://github.com/just-containers/s6-overlay/releases/download/v${S6VER}/s6-overlay-${S6_ARCH}.tar.gz -L -s --output /tmp/s6-overlay-${S6_ARCH}.tar.gz \
-    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C / \
-    && rm -rf /tmp/s6-overlay-${S6_ARCH}.tar.gz \
-    && curl -L https://dl.influxdata.com/influxdb/releases/influxdb2-${INFLUXVER}-${TARGETARCH}.deb --output /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb \
+    && case ${TARGETARCH} in \
+           "amd64")  S6_ARCH=x86_64  ;; \
+           "arm64")  S6_ARCH=aarch64  ;; \
+       esac \
+    && curl https://github.com/just-containers/s6-overlay/releases/download/v${S6VER}/s6-overlay-noarch.tar.xz -L -s --output /tmp/s6-overlay-noarch.tar.xz \
+    && tar -Jxpf /tmp/s6-overlay-noarch.tar.xz -C / \
+    && rm -rf /tmp/s6-overlay-noarch.tar.xz \
+    && curl https://github.com/just-containers/s6-overlay/releases/download/v${S6VER}/s6-overlay-${S6_ARCH}.tar.xz -L -s --output /tmp/s6-overlay-${S6_ARCH}.tar.xz \
+    && tar -Jxpf /tmp/s6-overlay-${S6_ARCH}.tar.xz -C / \
+    && rm -rf /tmp/s6-overlay-${S6_ARCH}.tar.xz
+RUN curl -L https://dl.influxdata.com/influxdb/releases/influxdb2-${INFLUXVER}-${TARGETARCH}.deb --output /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb \
     && dpkg -i --force-all /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb \
-    && rm -rf /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
+    && rm -rf /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb
 
 COPY /rootfs /
 
@@ -57,6 +69,9 @@ RUN chmod 0644 /etc/cron.d/scrutiny && \
     rm -f /etc/cron.daily/* && \
     mkdir -p /opt/scrutiny/web && \
     mkdir -p /opt/scrutiny/config && \
-    chmod -R ugo+rwx /opt/scrutiny/config
+    chmod -R ugo+rwx /opt/scrutiny/config && \
+    chmod +x /etc/cont-init.d/* && \
+    chmod +x /etc/services.d/*/run && \
+    chmod +x /etc/services.d/*/finish
 
 CMD ["/init"]

docker/Dockerfile.collector

@@ -4,20 +4,21 @@
 
 
 ########
-FROM golang:1.20-bullseye as backendbuild
+FROM golang:1.20-bookworm as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
 COPY . /go/src/github.com/analogj/scrutiny
 
+RUN apt-get update && apt-get install -y file && rm -rf /var/lib/apt/lists/*
 RUN make binary-clean binary-collector
 
 ########
-FROM debian:bullseye-slim as runtime
+FROM debian:bookworm-slim as runtime
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && update-ca-certificates
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && rm -rf /var/lib/apt/lists/* && update-ca-certificates
 
 COPY /docker/entrypoint-collector.sh /entrypoint-collector.sh
 COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny

docker/Dockerfile.web

@@ -11,21 +11,22 @@ COPY --link . /go/src/github.com/analogj/scrutiny
 RUN make binary-frontend
 
 ######## Build the backend
-FROM golang:1.20-bullseye as backendbuild
+FROM golang:1.20-bookworm as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 COPY --link . /go/src/github.com/analogj/scrutiny
 
+RUN apt-get update && apt-get install -y file && rm -rf /var/lib/apt/lists/*
 RUN make binary-clean binary-all WEB_BINARY_NAME=scrutiny
 
 
 ######## Combine build artifacts in runtime image
-FROM debian:bullseye-slim as runtime
+FROM debian:bookworm-slim as runtime
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-RUN apt-get update && apt-get install -y ca-certificates curl tzdata && update-ca-certificates
+RUN apt-get update && apt-get install -y ca-certificates curl tzdata && rm -rf /var/lib/apt/lists/* && update-ca-certificates
 
 COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
 COPY --link --from=frontendbuild --chmod=644 /go/src/github.com/analogj/scrutiny/dist /opt/scrutiny/web

docker/entrypoint-collector.sh

@@ -25,4 +25,4 @@ fi
 
 # now that we have the env start cron in the foreground
 echo "starting cron"
-su -c "cron -f -L 15" root
+exec su -c "cron -f -L 15" root

docker/example.hubspoke.docker-compose.yml

@@ -2,6 +2,7 @@ version: '2.4'
 
 services:
   influxdb:
+    restart: unless-stopped
     image: influxdb:2.2
     ports:
       - '8086:8086'
@@ -15,6 +16,7 @@ services:
 
 
   web:
+    restart: unless-stopped
     image: 'ghcr.io/analogj/scrutiny:master-web'
     ports:
       - '8080:8080'
@@ -33,6 +35,7 @@ services:
       start_period: 10s
 
   collector:
+    restart: unless-stopped
     image: 'ghcr.io/analogj/scrutiny:master-collector'
     cap_add:
       - SYS_RAWIO
@@ -40,9 +43,13 @@ services:
       - '/run/udev:/run/udev:ro'
     environment:
       COLLECTOR_API_ENDPOINT: 'http://web:8080'
+      COLLECTOR_HOST_ID: 'scrutiny-collector-hostname'
+      # If true forces the collector to run on startup (cron will be started after the collector completes)
+      # see: https://github.com/AnalogJ/scrutiny/blob/master/docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md#collector-trigger-on-startup
+      COLLECTOR_RUN_STARTUP: false
     depends_on:
       web:
         condition: service_healthy
     devices:
       - "/dev/sda"
-      - "/dev/sdb"
+      - "/dev/sdb"

docker/example.omnibus.docker-compose.yml

@@ -2,6 +2,7 @@ version: '3.5'
 
 services:
   scrutiny:
+    restart: unless-stopped
     container_name: scrutiny
     image: ghcr.io/analogj/scrutiny:master-omnibus
     cap_add:

docs/INSTALL_HUB_SPOKE.md

@@ -59,6 +59,7 @@ networks:
 
 services:
   influxdb:
+    restart: unless-stopped
     container_name: influxdb
     image: influxdb:2.1-alpine
     ports:
@@ -73,11 +74,11 @@ services:
       - DOCKER_INFLUXDB_INIT_ORG=homelab
       - DOCKER_INFLUXDB_INIT_BUCKET=scrutiny
       - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=your-very-secret-token
-    restart: unless-stopped
     networks:
       - monitoring
 
   scrutiny:
+    restart: unless-stopped
     container_name: scrutiny
     image: ghcr.io/analogj/scrutiny:master-web
     ports:
@@ -91,10 +92,9 @@ services:
       - SCRUTINY_WEB_INFLUXDB_ORG=homelab
       - SCRUTINY_WEB_INFLUXDB_BUCKET=scrutiny
       # Optional but highly recommended to notify you in case of a problem
-      - SCRUTINY_WEB_NOTIFY_URLS=["http://gotify:80/message?token=a-gotify-token"]
+      - SCRUTINY_NOTIFY_URLS=["http://gotify:80/message?token=a-gotify-token"]
     depends_on:
       - influxdb
-    restart: unless-stopped
     networks:
       - notifications
       - monitoring
@@ -121,7 +121,7 @@ apt install smartmontools -y
 # 3. Make it exacutable
 # 4. List the contents of the library for confirmation
 mkdir -p /opt/scrutiny/bin && \
-curl -L https://github.com/AnalogJ/scrutiny/releases/download/v0.5.0/scrutiny-collector-metrics-linux-amd64 > /opt/scrutiny/bin/scrutiny-collector-metrics-linux-amd64 && \
+curl -L https://github.com/AnalogJ/scrutiny/releases/download/v0.8.1/scrutiny-collector-metrics-linux-amd64 > /opt/scrutiny/bin/scrutiny-collector-metrics-linux-amd64 && \
 chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics-linux-amd64 && \
 ls -lha /opt/scrutiny/bin
 ```
@@ -168,6 +168,7 @@ version: "3.4"
 services:
 
   collector:
+    restart: unless-stopped
     image: 'ghcr.io/analogj/scrutiny:master-collector'
     cap_add:
       - SYS_RAWIO

docs/SUPPORTED_NAS_OS.md

@@ -9,7 +9,8 @@ in `docs/guides/` or elsewhere) it will be linked here.
 - [ ] Proxmox
 - [x] Synology 
   - [Hub/Spoke Deployment - Collector](./INSTALL_SYNOLOGY_COLLECTOR.md)
-  - [Omnibus Deployment](https://drfrankenstein.co.uk/2022/07/28/scrutiny-in-docker-on-a-synology-nas)
+  - [Omnibus Deployment](https://drfrankenstein.co.uk/2022/07/28/scrutiny-in-docker-on-a-synology-nas) -  Docker Package
+  - [Omnibus Deployment](https://drfrankenstein.co.uk/scrutiny-in-container-manager-on-a-synology-nas/) -  Container Manager Package
 - [ ] OMV
 - [ ] Amahi
 - [ ] Running in a LXC container

docs/TESTERS.md

@@ -8,7 +8,7 @@ Thankfully the following users have been gracious enough to test/validate Scruti
 
 | Architecture Name | Binaries | Docker |
 | --- | --- | --- |
-| linux-amd64 | -- | @feroxy @rshxyz |
+| linux-amd64 | @TizzAmmazz | @feroxy @rshxyz |
 | linux-arm-5 | -- |  |
 | linux-arm-6 | -- |  |
 | linux-arm-7 | @Zorlin | @martini1992  |
@@ -17,4 +17,4 @@ Thankfully the following users have been gracious enough to test/validate Scruti
 | macos-amd64 | -- | -- |
 | macos-arm64 | -- | -- |
 | windows-amd64 | @gabrielv33 | -- |
-| windows-arm64 | -- | -- |
+| windows-arm64 | -- | -- |

docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md

@@ -250,8 +250,9 @@ UPDATE devices SET device_status = null;
 
 ### Seagate Drives Failing
 
-As thoroughly discussed in [#255](https://github.com/AnalogJ/scrutiny/issues/255), Seagate (Ironwolf & others) drives are almost always marked as failed by Scrutiny. 
+As thoroughly discussed in [#255](https://github.com/AnalogJ/scrutiny/issues/255) and [#522](https://github.com/AnalogJ/scrutiny/issues/522), Seagate (Ironwolf & others) drives are almost always marked as failed by Scrutiny. 
 
+#### Seek Error Rate & Read Error Rate (#255)
 > The `Seek Error Rate` & `Read Error Rate` attribute raw values are typically very high, and the 
 > normalised values (Current / Worst / Threshold) are usually quite low. Despite this, the numbers in most cases are perfectly OK
 > 
@@ -286,6 +287,15 @@ other drives, please read the following:
 - http://www.users.on.net/~fzabkar/HDD/Seagate_SER_RRER_HEC.html
 - https://www.truenas.com/community/threads/seagate-ironwolf-smart-test-raw_read_error_rate-seek_error_rate.68634/
 
+#### Seagate Raw Values are incorrect (#522)
+Basically Seagate drives are known to use a custom data format for a number of their SMART attributes. This causes issues with Scrutiny's threshold analysis. 
+
+- The workaround is to customize the smartctl command that Scrutiny uses for your drive by [creating a collector.yaml file](https://github.com/AnalogJ/scrutiny/issues/522#issuecomment-1766727988) and "fixing" each attribute
+- The **real "fix"** is to make sure your Seagate drive is correctly supported by smartmontools. See this [PR](https://github.com/smartmontools/smartmontools/pull/247)
+
+Sorry for the bad news, but this is a known issue and there's just not much we can do on the Scrutiny side. 
+
+
 ## Hub & Spoke model, with multiple Hosts.
 
 ![multiple-host-ids image](multiple-host-ids.png)

docs/TROUBLESHOOTING_DOCKER.md

@@ -17,3 +17,11 @@ So changing from `master-omnibus -> latest` will be the same thing for all inten
 > NOTE: Previously, there was a `automated cron build` that ran on the `master` and `beta` branches. 
 They used to trigger a `nightly` build, even if nothing has changed on the branch. This has a couple of benefits, but one is to 
 ensure that there's no broken external dependencies in our (unchanged) code. This `nightly` build no longer updates the `master-omnibus` tag. 
+
+# Running Docker `rootless`
+
+To avoid that the container(s) restart when you installed Docker as `rootless` you need to isssue the following commands to allow the session to stay alive even after you close your (SSH) sesssion:
+
+`sudo loginctl enable-linger $(whoami)`
+
+`systemctl --user enable docker`

docs/TROUBLESHOOTING_INFLUXDB.md

@@ -11,6 +11,32 @@ dependency. It's a dedicated timeseries database, as opposed to the general purp
 a bunch of testing and analysis before I made the change. With InfluxDB the memory footprint for Scrutiny (at idle) is ~
 100mb, which is still fairly reasonable.
 
+### Data Size
+
+It's surprisingly easy to reach extremely large database sizes, if you don't use downsampling, or you downsample incorrectly. 
+The growth rate is pretty unintuitive -- see https://github.com/AnalogJ/scrutiny/issues/650#issuecomment-2365174940 
+
+> Fasten stores the SMART metrics in a timeseries database (InfluxDB), and automatically downsamples the data on a schedule.
+>
+> The expectation was that cron would run daily, and there would be:
+>
+>    - 7 daily data points
+>    - 3 weekly data points
+>    - 11 monthly data points
+>    - and infinite yearly data points.
+>
+> These data points would be for each SMART metric, for each device.
+> eg. in one year, (7+3+11)*80ish SMART attributes = 1680 datapoints for one device
+>
+> If you're running cron every 15 minutes, your browser will instead be attempting to display:
+>
+>    - 96*7 daily data points
+>    - 3 weekly
+>    - 11 monthly
+>
+> so (96*7 + 3 + 11)*80 = 54,880 datapoints for each device 😭
+
+
 ## Installation
 
 InfluxDB is a required dependency for Scrutiny v0.4.0+.

docs/TROUBLESHOOTING_NOTIFICATIONS.md

@@ -24,8 +24,23 @@ SCRUTINY_MESSAGE - eg. "Scrutiny SMART error notification for device: %s\nFailur
 SCRUTINY_HOST_ID - (optional) eg. "my-custom-host-id"
 ```
 
+# Special Characters
+
+`Shoutrrr` supports special characters in the username and password fields, however you'll need to url-encode the
+username and the password separately.
+
+- if your username is: `myname@example.com`
+- if your password is `124@34$1`
+
+Then your `shoutrrr` url will look something like:
+
+- `smtp://myname%40example%2Ecom:124%4034%241@ms.my.domain.com:587`
+
 # Testing Notifications
-You can test that your notifications are configured correctly by posting an empty payload to the notifications health check API.
+
+You can test that your notifications are configured correctly by posting an empty payload to the notifications health
+check API.
+
 ```
 curl -X POST http://localhost:8080/api/health/notify
 ```

docs/dbdiagram.io.txt

@@ -2,6 +2,7 @@
 // SQLite Table(s)
 
 Table Device  {
+    Archived bool
 	//GORM attributes, see: http://gorm.io/docs/conventions.html
 	CreatedAt time
 	UpdatedAt time

example.collector.yaml

@@ -81,6 +81,7 @@ devices:
 #  metrics_scan_args: '--scan --json' # used to detect devices
 #  metrics_info_args: '--info --json' # used to determine device unique ID & register device with Scrutiny
 #  metrics_smart_args: '--xall --json' # used to retrieve smart data for each device.
+#  metrics_smartctl_wait: 0 # time to wait in seconds between each disk's check
 
 
 ########################################################################################################################

example.scrutiny.yaml

@@ -60,10 +60,14 @@ log:
 
 # Notification "urls" look like the following. For more information about service specific configuration see
 # Shoutrrr's documentation: https://containrrr.dev/shoutrrr/services/overview/
+#
+# note, usernames and passwords containing special characters will need to be urlencoded.
+# if your username is: "myname@example.com" and your password is "124@34$1"
+# your shoutrrr url will look like: "smtp://myname%40example%2Ecom:124%4034%241@ms.my.domain.com:587"
 
 #notify:
 #  urls:
-#    - "discord://token@channel"
+#    - "discord://token@webhookid"
 #    - "telegram://token@telegram?channels=channel-1[,channel-2,...]"
 #    - "pushover://shoutrrr:apiToken@userKey/?priority=1&devices=device1[,device2, ...]"
 #    - "slack://[botname@]token-a/token-b/token-c"

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
-	github.com/containrrr/shoutrrr v0.7.1
+	github.com/containrrr/shoutrrr v0.8.0
 	github.com/fatih/color v1.15.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/glebarez/sqlite v1.4.5
@@ -15,7 +15,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/samber/lo v1.25.0
 	github.com/sirupsen/logrus v1.6.0
-	github.com/spf13/viper v1.14.0
+	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.2.0
 	golang.org/x/sync v0.1.0
@@ -35,7 +35,7 @@ require (
 	github.com/go-playground/locales v0.13.0 // indirect
 	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/go-playground/validator/v10 v10.2.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
@@ -46,30 +46,29 @@ require (
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/kvz/logstreamer v0.0.0-20201023134116-02d20f4338f5 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/spf13/afero v1.9.2 // indirect
+	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/ugorji/go/codec v1.1.7 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
-	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/sys v0.7.0 // indirect
-	golang.org/x/term v0.1.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

rootfs/etc/cont-init.d/01-timezone

@@ -1,4 +1,4 @@
-#!/usr/bin/with-contenv bash
+#!/command/with-contenv bash
 
 if [ -n "${TZ}" ]
 then

rootfs/etc/cont-init.d/50-cron-config

@@ -1,4 +1,4 @@
-#!/usr/bin/with-contenv bash
+#!/command/with-contenv bash
 
 # Cron runs in its own isolated environment (usually using only /etc/environment )
 # So when the container starts up, we will do a dump of the runtime environment into a .env file that we
@@ -12,4 +12,4 @@ COLLECTOR_CRON_SCHEDULE=${COLLECTOR_CRON_SCHEDULE:-"0 0 * * *"}
 [[ "${COLLECTOR_CRON_SCHEDULE}" == \"*\" || "${COLLECTOR_CRON_SCHEDULE}" == \'*\' ]] && COLLECTOR_CRON_SCHEDULE="${COLLECTOR_CRON_SCHEDULE:1:-1}"
 
 # replace placeholder with correct value
-sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron.d/scrutiny
+sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron.d/scrutiny

rootfs/etc/services.d/collector-once/run

@@ -1,13 +1,25 @@
-#!/usr/bin/with-contenv bash
+#!/command/with-contenv bash
+
+# ensure not run (successfully) before
+if [ -f /tmp/custom-init-performed ]; then
+  echo 'INFO: custom init already performed'
+  s6-svc -D /run/service/collector-once     # prevent s6 from restarting service
+  exit 0
+fi
 
 echo "waiting for scrutiny service to start"
-s6-svwait -u /var/run/s6/services/scrutiny
-
-#tell s6 to only run this script once
-s6-svc -O /var/run/s6/services/collector-once
+s6-svwait -u /run/service/scrutiny
 
 # wait until scrutiny is "Ready"
 until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done
 
 echo "starting scrutiny collector (run-once mode. subsequent calls will be triggered via cron service)"
 /opt/scrutiny/bin/scrutiny-collector-metrics run
+
+# prevent script's core logic from running again
+touch /tmp/custom-init-performed
+
+# prevent s6 from restarting service
+s6-svc -D /run/service/collector-once
+
+exit 0

rootfs/etc/services.d/cron/finish

@@ -1,4 +1,4 @@
-#!/usr/bin/execlineb -S0
+#!/command/execlineb -S0
 
 echo "cron exiting"
 s6-svscanctl -t /var/run/s6/services

rootfs/etc/services.d/cron/run

@@ -1,4 +1,4 @@
-#!/usr/bin/with-contenv bash
+#!/command/with-contenv bash
 
 echo "starting cron"
 cron -f -L 15

rootfs/etc/services.d/influxdb/run

@@ -1,4 +1,4 @@
-#!/usr/bin/with-contenv bash
+#!/command/with-contenv bash
 
 mkdir -p /opt/scrutiny/influxdb/
 

rootfs/etc/services.d/scrutiny/run

@@ -1,4 +1,4 @@
-#!/usr/bin/with-contenv bash
+#!/command/with-contenv bash
 
 echo "waiting for influxdb"
 until $(curl --output /dev/null --silent --head --fail http://localhost:8086/health); do echo "influxdb not ready" && sleep 5; done

webapp/backend/pkg/database/helpers.go

@@ -1,12 +0,0 @@
-package database
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
-	"sort"
-)
-
-func sortSmartMeasurementsDesc(smartResults []measurements.Smart) {
-	sort.SliceStable(smartResults, func(i, j int) bool {
-		return smartResults[i].Date.After(smartResults[j].Date)
-	})
-}

webapp/backend/pkg/database/helpers_test.go

@@ -1,30 +0,0 @@
-package database
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
-	"github.com/stretchr/testify/require"
-	"testing"
-	"time"
-)
-
-func Test_sortSmartMeasurementsDesc_LatestFirst(t *testing.T) {
-	//setup
-	timeNow := time.Now()
-	smartResults := []measurements.Smart{
-		{
-			Date: timeNow.AddDate(0, 0, -2),
-		},
-		{
-			Date: timeNow,
-		},
-		{
-			Date: timeNow.AddDate(0, 0, -1),
-		},
-	}
-
-	//test
-	sortSmartMeasurementsDesc(smartResults)
-
-	//assert
-	require.Equal(t, smartResults[0].Date, timeNow)
-}

webapp/backend/pkg/database/interface.go

@@ -2,12 +2,15 @@ package database
 
 import (
 	"context"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 )
 
+// Create mock using:
+// mockgen -source=webapp/backend/pkg/database/interface.go -destination=webapp/backend/pkg/database/mock/mock_database.go
 type DeviceRepo interface {
 	Close() error
 	HealthCheck(ctx context.Context) error
@@ -17,12 +20,13 @@ type DeviceRepo interface {
 	UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error)
 	UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error)
 	GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error)
+	UpdateDeviceArchived(ctx context.Context, wwn string, archived bool) error
 	DeleteDevice(ctx context.Context, wwn string) error
 
 	SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error)
-	GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error)
+	GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error)
 
-	SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error
+	SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo, discardSCTTempHistory bool) error
 
 	GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
 	GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)

webapp/backend/pkg/database/migrations/m20220509170100/device.go

@@ -5,6 +5,7 @@ import (
 	"time"
 )
 
+// Deprecated: m20220509170100.Device is deprecated, only used by db migrations
 type Device struct {
 	//GORM attributes, see: http://gorm.io/docs/conventions.html
 	CreatedAt time.Time
@@ -14,9 +15,9 @@ type Device struct {
 	WWN string `json:"wwn" gorm:"primary_key"`
 
 	DeviceName     string `json:"device_name"`
-	DeviceUUID	   string `json:"device_uuid"`
-	DeviceSerialID	   string `json:"device_serial_id"`
-	DeviceLabel	   string `json:"device_label"`
+	DeviceUUID     string `json:"device_uuid"`
+	DeviceSerialID string `json:"device_serial_id"`
+	DeviceLabel    string `json:"device_label"`
 
 	Manufacturer   string `json:"manufacturer"`
 	ModelName      string `json:"model_name"`
@@ -38,4 +39,3 @@ type Device struct {
 	// Data set by Scrutiny
 	DeviceStatus pkg.DeviceStatus `json:"device_status"`
 }
-

webapp/backend/pkg/database/migrations/m20250221084400/device.go

@@ -0,0 +1,41 @@
+package m20250221084400
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"time"
+)
+
+type Device struct {
+	Archived bool `json:"archived"`
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN string `json:"wwn" gorm:"primary_key"`
+
+	DeviceName     string `json:"device_name"`
+	DeviceUUID     string `json:"device_uuid"`
+	DeviceSerialID string `json:"device_serial_id"`
+	DeviceLabel    string `json:"device_label"`
+
+	Manufacturer   string `json:"manufacturer"`
+	ModelName      string `json:"model_name"`
+	InterfaceType  string `json:"interface_type"`
+	InterfaceSpeed string `json:"interface_speed"`
+	SerialNumber   string `json:"serial_number"`
+	Firmware       string `json:"firmware"`
+	RotationSpeed  int    `json:"rotational_speed"`
+	Capacity       int64  `json:"capacity"`
+	FormFactor     string `json:"form_factor"`
+	SmartSupport   bool   `json:"smart_support"`
+	DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string `json:"label"`
+	HostId string `json:"host_id"`
+
+	// Data set by Scrutiny
+	DeviceStatus pkg.DeviceStatus `json:"device_status"`
+}

webapp/backend/pkg/database/mock/mock_database.go

@@ -0,0 +1,272 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: webapp/backend/pkg/database/interface.go
+
+// Package mock_database is a generated GoMock package.
+package mock_database
+
+import (
+	context "context"
+	reflect "reflect"
+
+	pkg "github.com/analogj/scrutiny/webapp/backend/pkg"
+	models "github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	collector "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	measurements "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockDeviceRepo is a mock of DeviceRepo interface.
+type MockDeviceRepo struct {
+	ctrl     *gomock.Controller
+	recorder *MockDeviceRepoMockRecorder
+}
+
+// MockDeviceRepoMockRecorder is the mock recorder for MockDeviceRepo.
+type MockDeviceRepoMockRecorder struct {
+	mock *MockDeviceRepo
+}
+
+// NewMockDeviceRepo creates a new mock instance.
+func NewMockDeviceRepo(ctrl *gomock.Controller) *MockDeviceRepo {
+	mock := &MockDeviceRepo{ctrl: ctrl}
+	mock.recorder = &MockDeviceRepoMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockDeviceRepo) EXPECT() *MockDeviceRepoMockRecorder {
+	return m.recorder
+}
+
+// Close mocks base method.
+func (m *MockDeviceRepo) Close() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Close")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Close indicates an expected call of Close.
+func (mr *MockDeviceRepoMockRecorder) Close() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockDeviceRepo)(nil).Close))
+}
+
+// UpdateDeviceArchived mocks base method.
+func (m *MockDeviceRepo) UpdateDeviceArchived(ctx context.Context, wwn string, archived bool) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateDeviceArchived", ctx, wwn)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateDeviceArchived indicates an expected call of UpdateDeviceArchived.
+func (mr *MockDeviceRepoMockRecorder) UpdateDeviceArchived(ctx, wwn, archived interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceArchived", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceArchived), ctx, wwn, archived)
+}
+
+// DeleteDevice mocks base method.
+func (m *MockDeviceRepo) DeleteDevice(ctx context.Context, wwn string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteDevice", ctx, wwn)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteDevice indicates an expected call of DeleteDevice.
+func (mr *MockDeviceRepoMockRecorder) DeleteDevice(ctx, wwn interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDevice", reflect.TypeOf((*MockDeviceRepo)(nil).DeleteDevice), ctx, wwn)
+}
+
+// GetDeviceDetails mocks base method.
+func (m *MockDeviceRepo) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDeviceDetails", ctx, wwn)
+	ret0, _ := ret[0].(models.Device)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDeviceDetails indicates an expected call of GetDeviceDetails.
+func (mr *MockDeviceRepoMockRecorder) GetDeviceDetails(ctx, wwn interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceDetails", reflect.TypeOf((*MockDeviceRepo)(nil).GetDeviceDetails), ctx, wwn)
+}
+
+// GetDevices mocks base method.
+func (m *MockDeviceRepo) GetDevices(ctx context.Context) ([]models.Device, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDevices", ctx)
+	ret0, _ := ret[0].([]models.Device)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDevices indicates an expected call of GetDevices.
+func (mr *MockDeviceRepoMockRecorder) GetDevices(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDevices", reflect.TypeOf((*MockDeviceRepo)(nil).GetDevices), ctx)
+}
+
+// GetSmartAttributeHistory mocks base method.
+func (m *MockDeviceRepo) GetSmartAttributeHistory(ctx context.Context, wwn, durationKey string, selectEntries, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetSmartAttributeHistory", ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes)
+	ret0, _ := ret[0].([]measurements.Smart)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetSmartAttributeHistory indicates an expected call of GetSmartAttributeHistory.
+func (mr *MockDeviceRepoMockRecorder) GetSmartAttributeHistory(ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSmartAttributeHistory", reflect.TypeOf((*MockDeviceRepo)(nil).GetSmartAttributeHistory), ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes)
+}
+
+// GetSmartTemperatureHistory mocks base method.
+func (m *MockDeviceRepo) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetSmartTemperatureHistory", ctx, durationKey)
+	ret0, _ := ret[0].(map[string][]measurements.SmartTemperature)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetSmartTemperatureHistory indicates an expected call of GetSmartTemperatureHistory.
+func (mr *MockDeviceRepoMockRecorder) GetSmartTemperatureHistory(ctx, durationKey interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSmartTemperatureHistory", reflect.TypeOf((*MockDeviceRepo)(nil).GetSmartTemperatureHistory), ctx, durationKey)
+}
+
+// GetSummary mocks base method.
+func (m *MockDeviceRepo) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetSummary", ctx)
+	ret0, _ := ret[0].(map[string]*models.DeviceSummary)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetSummary indicates an expected call of GetSummary.
+func (mr *MockDeviceRepoMockRecorder) GetSummary(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSummary", reflect.TypeOf((*MockDeviceRepo)(nil).GetSummary), ctx)
+}
+
+// HealthCheck mocks base method.
+func (m *MockDeviceRepo) HealthCheck(ctx context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "HealthCheck", ctx)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// HealthCheck indicates an expected call of HealthCheck.
+func (mr *MockDeviceRepoMockRecorder) HealthCheck(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HealthCheck", reflect.TypeOf((*MockDeviceRepo)(nil).HealthCheck), ctx)
+}
+
+// LoadSettings mocks base method.
+func (m *MockDeviceRepo) LoadSettings(ctx context.Context) (*models.Settings, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "LoadSettings", ctx)
+	ret0, _ := ret[0].(*models.Settings)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// LoadSettings indicates an expected call of LoadSettings.
+func (mr *MockDeviceRepoMockRecorder) LoadSettings(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadSettings", reflect.TypeOf((*MockDeviceRepo)(nil).LoadSettings), ctx)
+}
+
+// RegisterDevice mocks base method.
+func (m *MockDeviceRepo) RegisterDevice(ctx context.Context, dev models.Device) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RegisterDevice", ctx, dev)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RegisterDevice indicates an expected call of RegisterDevice.
+func (mr *MockDeviceRepoMockRecorder) RegisterDevice(ctx, dev interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterDevice", reflect.TypeOf((*MockDeviceRepo)(nil).RegisterDevice), ctx, dev)
+}
+
+// SaveSettings mocks base method.
+func (m *MockDeviceRepo) SaveSettings(ctx context.Context, settings models.Settings) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SaveSettings", ctx, settings)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SaveSettings indicates an expected call of SaveSettings.
+func (mr *MockDeviceRepoMockRecorder) SaveSettings(ctx, settings interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSettings", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSettings), ctx, settings)
+}
+
+// SaveSmartAttributes mocks base method.
+func (m *MockDeviceRepo) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SaveSmartAttributes", ctx, wwn, collectorSmartData)
+	ret0, _ := ret[0].(measurements.Smart)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// SaveSmartAttributes indicates an expected call of SaveSmartAttributes.
+func (mr *MockDeviceRepoMockRecorder) SaveSmartAttributes(ctx, wwn, collectorSmartData interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartAttributes", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartAttributes), ctx, wwn, collectorSmartData)
+}
+
+// SaveSmartTemperature mocks base method.
+func (m *MockDeviceRepo) SaveSmartTemperature(ctx context.Context, wwn, deviceProtocol string, collectorSmartData collector.SmartInfo, discardSCTTempHistory bool) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SaveSmartTemperature", ctx, wwn, deviceProtocol, collectorSmartData, discardSCTTempHistory)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SaveSmartTemperature indicates an expected call of SaveSmartTemperature.
+func (mr *MockDeviceRepoMockRecorder) SaveSmartTemperature(ctx, wwn, deviceProtocol, collectorSmartData, discardSCTTempHistory interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartTemperature", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartTemperature), ctx, wwn, deviceProtocol, collectorSmartData, discardSCTTempHistory)
+}
+
+// UpdateDevice mocks base method.
+func (m *MockDeviceRepo) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateDevice", ctx, wwn, collectorSmartData)
+	ret0, _ := ret[0].(models.Device)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// UpdateDevice indicates an expected call of UpdateDevice.
+func (mr *MockDeviceRepoMockRecorder) UpdateDevice(ctx, wwn, collectorSmartData interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDevice", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDevice), ctx, wwn, collectorSmartData)
+}
+
+// UpdateDeviceStatus mocks base method.
+func (m *MockDeviceRepo) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateDeviceStatus", ctx, wwn, status)
+	ret0, _ := ret[0].(models.Device)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// UpdateDeviceStatus indicates an expected call of UpdateDeviceStatus.
+func (mr *MockDeviceRepoMockRecorder) UpdateDeviceStatus(ctx, wwn, status interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceStatus", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceStatus), ctx, wwn, status)
+}

webapp/backend/pkg/database/scrutiny_repository_device.go

@@ -14,7 +14,7 @@ import (
 // Device
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-//insert device into DB (and update specified columns if device is already registered)
+// insert device into DB (and update specified columns if device is already registered)
 // update device fields that may change: (DeviceType, HostID)
 func (sr *scrutinyRepository) RegisterDevice(ctx context.Context, dev models.Device) error {
 	if err := sr.gormClient.WithContext(ctx).Clauses(clause.OnConflict{
@@ -51,7 +51,7 @@ func (sr *scrutinyRepository) UpdateDevice(ctx context.Context, wwn string, coll
 	return device, sr.gormClient.Model(&device).Updates(device).Error
 }
 
-//Update Device Status
+// Update Device Status
 func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) {
 	var device models.Device
 	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
@@ -74,6 +74,16 @@ func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, wwn string)
 	return device, nil
 }
 
+// Update Device Archived State
+func (sr *scrutinyRepository) UpdateDeviceArchived(ctx context.Context, wwn string, archived bool) error {
+	var device models.Device
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return fmt.Errorf("Could not get device from DB: %v", err)
+	}
+
+	return sr.gormClient.Model(&device).Where("wwn = ?", wwn).Update("archived", archived).Error
+}
+
 func (sr *scrutinyRepository) DeleteDevice(ctx context.Context, wwn string) error {
 	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).Delete(&models.Device{}).Error; err != nil {
 		return err

webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go

@@ -3,13 +3,14 @@ package database
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
 	"github.com/influxdata/influxdb-client-go/v2/api"
 	log "github.com/sirupsen/logrus"
-	"strings"
-	"time"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -30,14 +31,17 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin
 }
 
 // GetSmartAttributeHistory MUST return in sorted order, where newest entries are at the beginning of the list, and oldest are at the end.
-func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) {
+// When selectEntries is > 0, only the most recent selectEntries database entries are returned, starting from the selectEntriesOffset entry.
+// For example, with selectEntries = 5, selectEntries = 0, the most recent 5 are returned. With selectEntries = 3, selectEntries = 2, entries
+// 2 to 4 are returned (2 being the third newest, since it is zero-indexed)
+func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) {
 	// Get SMartResults from InfluxDB
 
 	//TODO: change the filter startrange to a real number.
 
 	// Get parser flux query result
 	//appConfig.GetString("web.influxdb.bucket")
-	queryStr := sr.aggregateSmartAttributesQuery(wwn, durationKey)
+	queryStr := sr.aggregateSmartAttributesQuery(wwn, durationKey, selectEntries, selectEntriesOffset, attributes)
 	log.Infoln(queryStr)
 
 	smartResults := []measurements.Smart{}
@@ -65,9 +69,6 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn
 		return nil, err
 	}
 
-	//we have to sort the smartResults again, because the `union` command will return multiple 'tables' and only sort the records in each table.
-	sortSmartMeasurementsDesc(smartResults)
-
 	return smartResults, nil
 
 	//if err := device.SquashHistory(); err != nil {
@@ -99,7 +100,7 @@ func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking,
 	return influxWriteApi.WritePoint(ctx, p)
 }
 
-func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string) string {
+func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string {
 
 	/*
 
@@ -108,28 +109,34 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration
 		|> range(start: -1w, stop: now())
 		|> filter(fn: (r) => r["_measurement"] == "smart" )
 		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> tail(n: 10, offset: 0)
 		|> schema.fieldsAsCols()
 
 		monthData = from(bucket: "metrics_weekly")
 		|> range(start: -1mo, stop: -1w)
 		|> filter(fn: (r) => r["_measurement"] == "smart" )
 		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> tail(n: 10, offset: 0)
 		|> schema.fieldsAsCols()
 
 		yearData = from(bucket: "metrics_monthly")
 		|> range(start: -1y, stop: -1mo)
 		|> filter(fn: (r) => r["_measurement"] == "smart" )
 		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> tail(n: 10, offset: 0)
 		|> schema.fieldsAsCols()
 
 		foreverData = from(bucket: "metrics_yearly")
 		|> range(start: -10y, stop: -1y)
 		|> filter(fn: (r) => r["_measurement"] == "smart" )
 		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> tail(n: 10, offset: 0)
 		|> schema.fieldsAsCols()
 
 		union(tables: [weekData, monthData, yearData, foreverData])
-		|> sort(columns: ["_time"], desc: false)
+		|> group()
+		|> sort(columns: ["_time"], desc: true)
+		|> tail(n: 6, offset: 4)
 		|> yield(name: "last")
 
 	*/
@@ -140,34 +147,60 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration
 
 	nestedDurationKeys := sr.lookupNestedDurationKeys(durationKey)
 
-	subQueryNames := []string{}
-	for _, nestedDurationKey := range nestedDurationKeys {
-		bucketName := sr.lookupBucketName(nestedDurationKey)
-		durationRange := sr.lookupDuration(nestedDurationKey)
-
-		subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey))
-		partialQueryStr = append(partialQueryStr, []string{
-			fmt.Sprintf(`%sData = from(bucket: "%s")`, nestedDurationKey, bucketName),
-			fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
-			`|> filter(fn: (r) => r["_measurement"] == "smart" )`,
-			fmt.Sprintf(`|> filter(fn: (r) => r["device_wwn"] == "%s" )`, wwn),
-			"|> schema.fieldsAsCols()",
-		}...)
-	}
-
-	if len(subQueryNames) == 1 {
+	if len(nestedDurationKeys) == 1 {
 		//there's only one bucket being queried, no need to union, just aggregate the dataset and return
 		partialQueryStr = append(partialQueryStr, []string{
-			subQueryNames[0],
+			sr.generateSmartAttributesSubquery(wwn, nestedDurationKeys[0], selectEntries, selectEntriesOffset, attributes),
+			fmt.Sprintf(`%sData`, nestedDurationKeys[0]),
+			`|> sort(columns: ["_time"], desc: true)`,
 			`|> yield()`,
 		}...)
-	} else {
-		partialQueryStr = append(partialQueryStr, []string{
-			fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")),
-			`|> sort(columns: ["_time"], desc: false)`,
-			`|> yield(name: "last")`,
-		}...)
+		return strings.Join(partialQueryStr, "\n")
 	}
 
+	subQueries := []string{}
+	subQueryNames := []string{}
+	for _, nestedDurationKey := range nestedDurationKeys {
+		subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey))
+		if selectEntries > 0 {
+			// We only need the last `n + offset` # of entries from each table to guarantee we can
+			// get the last `n` # of entries starting from `offset` of the union
+			subQueries = append(subQueries, sr.generateSmartAttributesSubquery(wwn, nestedDurationKey, selectEntries+selectEntriesOffset, 0, attributes))
+		} else {
+			subQueries = append(subQueries, sr.generateSmartAttributesSubquery(wwn, nestedDurationKey, 0, 0, attributes))
+		}
+	}
+	partialQueryStr = append(partialQueryStr, subQueries...)
+	partialQueryStr = append(partialQueryStr, []string{
+		fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")),
+		`|> group()`,
+		`|> sort(columns: ["_time"], desc: true)`,
+	}...)
+	if selectEntries > 0 {
+		partialQueryStr = append(partialQueryStr, fmt.Sprintf(`|> tail(n: %d, offset: %d)`, selectEntries, selectEntriesOffset))
+	}
+	partialQueryStr = append(partialQueryStr, `|> yield(name: "last")`)
+
+	return strings.Join(partialQueryStr, "\n")
+}
+
+func (sr *scrutinyRepository) generateSmartAttributesSubquery(wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string {
+	bucketName := sr.lookupBucketName(durationKey)
+	durationRange := sr.lookupDuration(durationKey)
+
+	partialQueryStr := []string{
+		fmt.Sprintf(`%sData = from(bucket: "%s")`, durationKey, bucketName),
+		fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
+		`|> filter(fn: (r) => r["_measurement"] == "smart" )`,
+		fmt.Sprintf(`|> filter(fn: (r) => r["device_wwn"] == "%s" )`, wwn),
+	}
+
+	partialQueryStr = append(partialQueryStr, `|> aggregateWindow(every: 1d, fn: last, createEmpty: false)`)
+	
+	if selectEntries > 0 {
+		partialQueryStr = append(partialQueryStr, fmt.Sprintf(`|> tail(n: %d, offset: %d)`, selectEntries, selectEntriesOffset))
+	}
+	partialQueryStr = append(partialQueryStr, "|> schema.fieldsAsCols()")
+
 	return strings.Join(partialQueryStr, "\n")
 }

webapp/backend/pkg/database/scrutiny_repository_migrations.go

@@ -4,11 +4,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strconv"
+	"time"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220716214900"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20250221084400"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
@@ -17,8 +21,6 @@ import (
 	"github.com/influxdata/influxdb-client-go/v2/api/http"
 	log "github.com/sirupsen/logrus"
 	"gorm.io/gorm"
-	"strconv"
-	"time"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -331,7 +333,6 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 						SettingDataType:       "string",
 						SettingValueString:    "smooth",
 					},
-
 					{
 						SettingKeyName:        "metrics.notify_level",
 						SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)",
@@ -369,6 +370,60 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 				return tx.Create(&defaultSettings).Error
 			},
 		},
+		{
+			ID: "m20231123123300", // add repeat_notifications setting.
+			Migrate: func(tx *gorm.DB) error {
+				//add repeat_notifications setting default.
+				var defaultSettings = []m20220716214900.Setting{
+					{
+						SettingKeyName:        "metrics.repeat_notifications",
+						SettingKeyDescription: "Whether to repeat all notifications or just when values change (true | false)",
+						SettingDataType:       "bool",
+						SettingValueBool:      true,
+					},
+				}
+				return tx.Create(&defaultSettings).Error
+			},
+		},
+		{
+			ID: "m20240722082740", // add powered_on_hours_unit setting.
+			Migrate: func(tx *gorm.DB) error {
+				//add powered_on_hours_unit setting default.
+				var defaultSettings = []m20220716214900.Setting{
+					{
+						SettingKeyName:        "powered_on_hours_unit",
+						SettingKeyDescription: "Presentation format for device powered on time ('humanize' | 'device_hours')",
+						SettingDataType:       "string",
+						SettingValueString:    "humanize",
+					},
+				}
+				return tx.Create(&defaultSettings).Error
+			},
+		},
+		{
+			ID: "m20250221084400", // add archived to device data
+			Migrate: func(tx *gorm.DB) error {
+
+				//migrate the device database.
+				// adding column (archived)
+				return tx.AutoMigrate(m20250221084400.Device{})
+			},
+		},
+		{
+			ID: "m20260105083200", // add discard_sct_temp_history setting.
+			Migrate: func(tx *gorm.DB) error {
+				//add discard_sct_temp_history setting default.
+				var defaultSettings = []m20220716214900.Setting{
+					{
+						SettingKeyName:        "collector.discard_sct_temp_history",
+						SettingKeyDescription: "Whether to discard SCT Temperature history (true | false)",
+						SettingDataType:       "bool",
+						SettingValueBool:      false,
+					},
+				}
+				return tx.Create(&defaultSettings).Error
+			},
+		},
 	})
 
 	if err := m.Migrate(); err != nil {
@@ -405,8 +460,8 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 
 // helpers
 
-//When adding data to influxdb, an error may be returned if the data point is outside the range of the retention policy.
-//This function will ignore retention policy errors, and allow the migration to continue.
+// When adding data to influxdb, an error may be returned if the data point is outside the range of the retention policy.
+// This function will ignore retention policy errors, and allow the migration to continue.
 func ignorePastRetentionPolicyError(err error) error {
 	var influxDbWriteError *http.Error
 	if errors.As(err, &influxDbWriteError) {

webapp/backend/pkg/database/scrutiny_repository_temperature.go

@@ -3,18 +3,19 @@ package database
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
-	"strings"
-	"time"
 )
 
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // Temperature Data
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error {
-	if len(collectorSmartData.AtaSctTemperatureHistory.Table) > 0 {
+// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo, discardSCTTempHistory bool) error {
+	if len(collectorSmartData.AtaSctTemperatureHistory.Table) > 0 && !discardSCTTempHistory {
 
 		for ndx, temp := range collectorSmartData.AtaSctTemperatureHistory.Table {
 			//temp value may be null, we must skip/ignore them. See #393
@@ -22,9 +23,11 @@ func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn stri
 				continue
 			}
 
-			minutesOffset := collectorSmartData.AtaSctTemperatureHistory.LoggingIntervalMinutes * int64(ndx) * 60
+			intervalSec := collectorSmartData.AtaSctTemperatureHistory.LoggingIntervalMinutes * 60
+			datapointTime := collectorSmartData.LocalTime.TimeT - int64(ndx) * intervalSec
+			alignedDatapointTime := datapointTime - datapointTime % intervalSec
 			smartTemp := measurements.SmartTemperature{
-				Date: time.Unix(collectorSmartData.LocalTime.TimeT-minutesOffset, 0),
+				Date: time.Unix(alignedDatapointTime, 0),
 				Temp: temp,
 			}
 
@@ -39,23 +42,22 @@ func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn stri
 				return err
 			}
 		}
-		// also add the current temperature.
-	} else {
-
-		smartTemp := measurements.SmartTemperature{
-			Date: time.Unix(collectorSmartData.LocalTime.TimeT, 0),
-			Temp: collectorSmartData.Temperature.Current,
-		}
-
-		tags, fields := smartTemp.Flatten()
-		tags["device_wwn"] = wwn
-		p := influxdb2.NewPoint("temp",
-			tags,
-			fields,
-			smartTemp.Date)
-		return sr.influxWriteApi.WritePoint(ctx, p)
 	}
-	return nil
+
+
+	// Even if ata_sct_temperature_history is present, also add current temperature. See #824
+	smartTemp := measurements.SmartTemperature{
+		Date: time.Unix(collectorSmartData.LocalTime.TimeT, 0),
+		Temp: collectorSmartData.Temperature.Current,
+	}
+
+	tags, fields := smartTemp.Flatten()
+	tags["device_wwn"] = wwn
+	p := influxdb2.NewPoint("temp",
+		tags,
+		fields,
+		smartTemp.Date)
+	return sr.influxWriteApi.WritePoint(ctx, p)
 }
 
 func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) {

webapp/backend/pkg/models/collector/smart.go

@@ -27,14 +27,11 @@ type SmartInfo struct {
 		Oui uint64 `json:"oui"`
 		ID  uint64 `json:"id"`
 	} `json:"wwn"`
-	FirmwareVersion string `json:"firmware_version"`
-	UserCapacity    struct {
-		Blocks int64 `json:"blocks"`
-		Bytes  int64 `json:"bytes"`
-	} `json:"user_capacity"`
-	LogicalBlockSize  int `json:"logical_block_size"`
-	PhysicalBlockSize int `json:"physical_block_size"`
-	RotationRate      int `json:"rotation_rate"`
+	FirmwareVersion   string       `json:"firmware_version"`
+	UserCapacity      UserCapacity `json:"user_capacity"`
+	LogicalBlockSize  int          `json:"logical_block_size"`
+	PhysicalBlockSize int          `json:"physical_block_size"`
+	RotationRate      int          `json:"rotation_rate"`
 	FormFactor        struct {
 		AtaValue int    `json:"ata_value"`
 		Name     string `json:"name"`
@@ -210,9 +207,10 @@ type SmartInfo struct {
 		ID          int `json:"id"`
 		SubsystemID int `json:"subsystem_id"`
 	} `json:"nvme_pci_vendor"`
-	NvmeIeeeOuiIdentifier  int `json:"nvme_ieee_oui_identifier"`
-	NvmeControllerID       int `json:"nvme_controller_id"`
-	NvmeNumberOfNamespaces int `json:"nvme_number_of_namespaces"`
+	NvmeIeeeOuiIdentifier  int   `json:"nvme_ieee_oui_identifier"`
+	NvmeTotalCapacity      int64 `json:"nvme_total_capacity"`
+	NvmeControllerID       int   `json:"nvme_controller_id"`
+	NvmeNumberOfNamespaces int   `json:"nvme_number_of_namespaces"`
 	NvmeNamespaces         []struct {
 		ID   int `json:"id"`
 		Size struct {
@@ -239,7 +237,23 @@ type SmartInfo struct {
 	ScsiErrorCounterLog ScsiErrorCounterLog `json:"scsi_error_counter_log"`
 }
 
-//Primary Attribute Structs
+// Capacity finds the total capacity of the device in bytes, or 0 if unknown.
+func (s *SmartInfo) Capacity() int64 {
+	switch {
+	case s.NvmeTotalCapacity > 0:
+		return s.NvmeTotalCapacity
+	case s.UserCapacity.Bytes > 0:
+		return s.UserCapacity.Bytes
+	}
+	return 0
+}
+
+type UserCapacity struct {
+	Blocks int64 `json:"blocks"`
+	Bytes  int64 `json:"bytes"`
+}
+
+// Primary Attribute Structs
 type AtaSmartAttributesTableItem struct {
 	ID         int    `json:"id"`
 	Name       string `json:"name"`

webapp/backend/pkg/models/collector/smart_test.go

@@ -0,0 +1,33 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSmartInfo_Capacity(t *testing.T) {
+	t.Run("should report nvme capacity", func(t *testing.T) {
+		smartInfo := SmartInfo{
+			UserCapacity: UserCapacity{
+				Bytes: 1234,
+			},
+			NvmeTotalCapacity: 5678,
+		}
+		assert.Equal(t, int64(5678), smartInfo.Capacity())
+	})
+
+	t.Run("should report user capacity", func(t *testing.T) {
+		smartInfo := SmartInfo{
+			UserCapacity: UserCapacity{
+				Bytes: 1234,
+			},
+		}
+		assert.Equal(t, int64(1234), smartInfo.Capacity())
+	})
+
+	t.Run("should report 0 for unknown capacities", func(t *testing.T) {
+		var smartInfo SmartInfo
+		assert.Zero(t, smartInfo.Capacity())
+	})
+}

webapp/backend/pkg/models/device.go

@@ -14,6 +14,7 @@ type DeviceWrapper struct {
 
 type Device struct {
 	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	Archived  bool `json:"archived"`
 	CreatedAt time.Time
 	UpdatedAt time.Time
 	DeletedAt *time.Time

webapp/backend/pkg/models/measurements/smart_ata_attribute.go

@@ -2,10 +2,11 @@ package measurements
 
 import (
 	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 	"strconv"
 	"strings"
+
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 )
 
 type SmartAtaAttribute struct {
@@ -24,6 +25,10 @@ type SmartAtaAttribute struct {
 	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
+func (sa *SmartAtaAttribute) GetTransformedValue() int64 {
+	return sa.TransformedValue
+}
+
 func (sa *SmartAtaAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }

webapp/backend/pkg/models/measurements/smart_attribute.go

@@ -6,4 +6,5 @@ type SmartAttribute interface {
 	Flatten() (fields map[string]interface{})
 	Inflate(key string, val interface{})
 	GetStatus() pkg.AttributeStatus
+	GetTransformedValue() int64
 }

webapp/backend/pkg/models/measurements/smart_nvme_attribute.go

@@ -2,9 +2,10 @@ package measurements
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
-	"strings"
 )
 
 type SmartNvmeAttribute struct {
@@ -18,6 +19,10 @@ type SmartNvmeAttribute struct {
 	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
+func (sa *SmartNvmeAttribute) GetTransformedValue() int64 {
+	return sa.TransformedValue
+}
+
 func (sa *SmartNvmeAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }

webapp/backend/pkg/models/measurements/smart_scsci_attribute.go

@@ -2,9 +2,10 @@ package measurements
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
-	"strings"
 )
 
 type SmartScsiAttribute struct {
@@ -18,6 +19,10 @@ type SmartScsiAttribute struct {
 	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
+func (sa *SmartScsiAttribute) GetTransformedValue() int64 {
+	return sa.TransformedValue
+}
+
 func (sa *SmartScsiAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }

webapp/backend/pkg/models/settings.go

@@ -8,17 +8,23 @@ package models
 //}
 
 type Settings struct {
-	Theme            string `json:"theme" mapstructure:"theme"`
-	Layout           string `json:"layout" mapstructure:"layout"`
-	DashboardDisplay string `json:"dashboard_display" mapstructure:"dashboard_display"`
-	DashboardSort    string `json:"dashboard_sort" mapstructure:"dashboard_sort"`
-	TemperatureUnit  string `json:"temperature_unit" mapstructure:"temperature_unit"`
-	FileSizeSIUnits  bool   `json:"file_size_si_units" mapstructure:"file_size_si_units"`
-	LineStroke       string `json:"line_stroke" mapstructure:"line_stroke"`
+	Theme              string `json:"theme" mapstructure:"theme"`
+	Layout             string `json:"layout" mapstructure:"layout"`
+	DashboardDisplay   string `json:"dashboard_display" mapstructure:"dashboard_display"`
+	DashboardSort      string `json:"dashboard_sort" mapstructure:"dashboard_sort"`
+	TemperatureUnit    string `json:"temperature_unit" mapstructure:"temperature_unit"`
+	FileSizeSIUnits    bool   `json:"file_size_si_units" mapstructure:"file_size_si_units"`
+	LineStroke         string `json:"line_stroke" mapstructure:"line_stroke"`
+	PoweredOnHoursUnit string `json:"powered_on_hours_unit" mapstructure:"powered_on_hours_unit"`
+
+	Collector struct {
+		DiscardSCTTempHistory bool `json:"discard_sct_temp_history" mapstructure:"discard_sct_temp_history"`
+	} `json:"collector" mapstructure:"collector"`
 
 	Metrics struct {
-		NotifyLevel            int `json:"notify_level" mapstructure:"notify_level"`
-		StatusFilterAttributes int `json:"status_filter_attributes" mapstructure:"status_filter_attributes"`
-		StatusThreshold        int `json:"status_threshold" mapstructure:"status_threshold"`
+		NotifyLevel            int  `json:"notify_level" mapstructure:"notify_level"`
+		StatusFilterAttributes int  `json:"status_filter_attributes" mapstructure:"status_filter_attributes"`
+		StatusThreshold        int  `json:"status_threshold" mapstructure:"status_threshold"`
+		RepeatNotifications    bool `json:"repeat_notifications" mapstructure:"repeat_notifications"`
 	} `json:"metrics" mapstructure:"metrics"`
 }

webapp/backend/pkg/models/testdata/helper.go

@@ -4,13 +4,14 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
 	"time"
+
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 )
 
 func main() {
@@ -32,7 +33,7 @@ func main() {
 		log.Fatalf("ERROR %v", err)
 	}
 	defer file.Close()
-	_, err = SendPostRequest("http://localhost:9090/api/devices/register", file)
+	_, err = SendPostRequest("http://localhost:8080/api/devices/register", file)
 	if err != nil {
 		log.Fatalf("ERROR %v", err)
 	}
@@ -46,7 +47,7 @@ func main() {
 					log.Fatalf("ERROR %v", err)
 				}
 
-				_, err = SendPostRequest(fmt.Sprintf("http://localhost:9090/api/device/%s/smart", diskId), smartDataReader)
+				_, err = SendPostRequest(fmt.Sprintf("http://localhost:8080/api/device/%s/smart", diskId), smartDataReader)
 				if err != nil {
 					log.Fatalf("ERROR %v", err)
 				}

webapp/backend/pkg/notify/notify.go

@@ -15,11 +15,13 @@ import (
 	"github.com/analogj/go-util/utils"
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 	"github.com/containrrr/shoutrrr"
 	shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types"
+	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 )
@@ -30,7 +32,7 @@ const NotifyFailureTypeSmartFailure = "SmartFailure"
 const NotifyFailureTypeScrutinyFailure = "ScrutinyFailure"
 
 // ShouldNotify check if the error Message should be filtered (level mismatch or filtered_attributes)
-func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThreshold pkg.MetricsStatusThreshold, statusFilterAttributes pkg.MetricsStatusFilterAttributes) bool {
+func ShouldNotify(logger logrus.FieldLogger, device models.Device, smartAttrs measurements.Smart, statusThreshold pkg.MetricsStatusThreshold, statusFilterAttributes pkg.MetricsStatusFilterAttributes, repeatNotifications bool, c *gin.Context, deviceRepo database.DeviceRepo) bool {
 	// 1. check if the device is healthy
 	if device.DeviceStatus == pkg.DeviceStatusPassed {
 		return false
@@ -54,52 +56,69 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThr
 		requiredAttrStatus = pkg.AttributeStatusFailedScrutiny
 	}
 
-	// 2. check if the attributes that are failing should be filtered (non-critical)
-	// 3. for any unfiltered attribute, store the failure reason (Smart or Scrutiny)
-	if statusFilterAttributes == pkg.MetricsStatusFilterAttributesCritical {
-		hasFailingCriticalAttr := false
-		var statusFailingCriticalAttr pkg.AttributeStatus
+	// This is the only case where individual attributes need not be considered
+	if statusFilterAttributes == pkg.MetricsStatusFilterAttributesAll && repeatNotifications {
+		return pkg.DeviceStatusHas(device.DeviceStatus, requiredDeviceStatus)
+	}
 
-		for attrId, attrData := range smartAttrs.Attributes {
-			//find failing attribute
-			if attrData.GetStatus() == pkg.AttributeStatusPassed {
-				continue //skip all passing attributes
-			}
+	var failingAttributes []string
+	// Loop through the attributes to find the failing ones
+	for attrId, attrData := range smartAttrs.Attributes {
+		var status pkg.AttributeStatus = attrData.GetStatus()
+		// Skip over passing attributes
+		if status == pkg.AttributeStatusPassed {
+			continue
+		}
 
-			// merge the status's of all critical attributes
-			statusFailingCriticalAttr = pkg.AttributeStatusSet(statusFailingCriticalAttr, attrData.GetStatus())
-
-			//found a failing attribute, see if its critical
-			if device.IsScsi() && thresholds.ScsiMetadata[attrId].Critical {
-				hasFailingCriticalAttr = true
-			} else if device.IsNvme() && thresholds.NmveMetadata[attrId].Critical {
-				hasFailingCriticalAttr = true
+		// If the user only wants to consider critical attributes, we have to check
+		// if the not-passing attribute is critical or not
+		if statusFilterAttributes == pkg.MetricsStatusFilterAttributesCritical {
+			critical := false
+			if device.IsScsi() {
+				critical = thresholds.ScsiMetadata[attrId].Critical
+			} else if device.IsNvme() {
+				critical = thresholds.NmveMetadata[attrId].Critical
 			} else {
 				//this is ATA
 				attrIdInt, err := strconv.Atoi(attrId)
 				if err != nil {
 					continue
 				}
-				if thresholds.AtaMetadata[attrIdInt].Critical {
-					hasFailingCriticalAttr = true
-				}
+				critical = thresholds.AtaMetadata[attrIdInt].Critical
+			}
+			// Skip non-critical, non-passing attributes when this setting is on
+			if !critical {
+				continue
 			}
-
 		}
 
-		if !hasFailingCriticalAttr {
-			//no critical attributes are failing, and notifyFilterAttributes == "critical"
-			return false
-		} else {
-			// check if any of the critical attributes have a status that we're looking for
-			return pkg.AttributeStatusHas(statusFailingCriticalAttr, requiredAttrStatus)
-		}
-
-	} else {
-		// 2. SKIP - we are processing every attribute.
-		// 3. check if the device failure level matches the wanted failure level.
-		return pkg.DeviceStatusHas(device.DeviceStatus, requiredDeviceStatus)
+		// Record any attribute that doesn't get skipped by the above two checks
+		failingAttributes = append(failingAttributes, attrId)
 	}
+
+	// If the user doesn't want repeated notifications when the failing value doesn't change, we need to get the last value from the db
+	var lastPoints []measurements.Smart
+	var err error
+	if !repeatNotifications {
+		lastPoints, err = deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), database.DURATION_KEY_FOREVER, 1, 1, failingAttributes)
+		if err == nil || len(lastPoints) < 1 {
+			logger.Warningln("Could not get the most recent data points from the database. This is expected to happen only if this is the very first submission of data for the device.")
+		}
+	}
+	for _, attrId := range failingAttributes {
+		attrStatus := smartAttrs.Attributes[attrId].GetStatus()
+		if pkg.AttributeStatusHas(attrStatus, requiredAttrStatus) {
+			if repeatNotifications {
+				return true
+			}
+			// This is checked again here to avoid repeating the entire for loop in the check above.
+			// Probably unnoticeably worse performance, but cleaner code.
+			if err != nil || len(lastPoints) < 1 || lastPoints[0].Attributes[attrId].GetTransformedValue() != smartAttrs.Attributes[attrId].GetTransformedValue() {
+				return true
+			}
+		}
+	}
+	return false
 }
 
 // TODO: include user label for device.
@@ -222,7 +241,7 @@ func (n *Notify) Send() error {
 	notifyScripts := []string{}
 	notifyShoutrrr := []string{}
 
-	for ndx, _ := range configUrls {
+	for ndx := range configUrls {
 		if strings.HasPrefix(configUrls[ndx], "https://") || strings.HasPrefix(configUrls[ndx], "http://") {
 			notifyWebhooks = append(notifyWebhooks, configUrls[ndx])
 		} else if strings.HasPrefix(configUrls[ndx], "script://") {

webapp/backend/pkg/notify/notify_test.go

@@ -1,13 +1,20 @@
 package notify
 
 import (
+	"errors"
 	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
-	"github.com/stretchr/testify/require"
 	"testing"
 	"time"
+
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	mock_database "github.com/analogj/scrutiny/webapp/backend/pkg/database/mock"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"github.com/gin-gonic/gin"
+	"github.com/golang/mock/gomock"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
 )
 
 func TestShouldNotify_MustSkipPassingDevices(t *testing.T) {
@@ -20,22 +27,27 @@ func TestShouldNotify_MustSkipPassingDevices(t *testing.T) {
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
 
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusThresholdBoth_FailingSmartDevice(t *testing.T) {
 	t.Parallel()
-	//setup
+	//setupD
 	device := models.Device{
 		DeviceStatus: pkg.DeviceStatusFailedSmart,
 	}
 	smartAttrs := measurements.Smart{}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
-
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusThresholdSmart_FailingSmartDevice(t *testing.T) {
@@ -47,9 +59,11 @@ func TestShouldNotify_MetricsStatusThresholdSmart_FailingSmartDevice(t *testing.
 	smartAttrs := measurements.Smart{}
 	statusThreshold := pkg.MetricsStatusThresholdSmart
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
-
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusThresholdScrutiny_FailingSmartDevice(t *testing.T) {
@@ -61,9 +75,11 @@ func TestShouldNotify_MetricsStatusThresholdScrutiny_FailingSmartDevice(t *testi
 	smartAttrs := measurements.Smart{}
 	statusThreshold := pkg.MetricsStatusThresholdScrutiny
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
-
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithCriticalAttrs(t *testing.T) {
@@ -79,9 +95,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithCriticalAttrs(t
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) {
@@ -100,9 +119,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithMultipleCritical
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) {
@@ -118,9 +140,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoCriticalAttrs(
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) {
@@ -136,9 +161,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoFailingCritica
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdBoth
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
 }
 
 func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresholdSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) {
@@ -157,9 +185,77 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresho
 	}}
 	statusThreshold := pkg.MetricsStatusThresholdSmart
 	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase))
+}
+func TestShouldNotify_NoRepeat_DatabaseFailure(t *testing.T) {
+	t.Parallel()
+	//setup
+	device := models.Device{
+		DeviceStatus: pkg.DeviceStatusFailedScrutiny,
+	}
+	smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
+		"5": &measurements.SmartAtaAttribute{
+			Status: pkg.AttributeStatusFailedScrutiny,
+		},
+	}}
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
+	fakeDatabase.EXPECT().GetSmartAttributeHistory(&gin.Context{}, "", database.DURATION_KEY_FOREVER, 1, 1, []string{"5"}).Return([]measurements.Smart{}, errors.New("")).Times(1)
+
+	//assert
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, false, &gin.Context{}, fakeDatabase))
+}
+
+func TestShouldNotify_NoRepeat_NoDatabaseData(t *testing.T) {
+	t.Parallel()
+	//setup
+	device := models.Device{
+		DeviceStatus: pkg.DeviceStatusFailedScrutiny,
+	}
+	smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
+		"5": &measurements.SmartAtaAttribute{
+			Status: pkg.AttributeStatusFailedScrutiny,
+		},
+	}}
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
+	fakeDatabase.EXPECT().GetSmartAttributeHistory(&gin.Context{}, "", database.DURATION_KEY_FOREVER, 1, 1, []string{"5"}).Return([]measurements.Smart{}, nil).Times(1)
+
+	//assert
+	require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, false, &gin.Context{}, fakeDatabase))
+}
+func TestShouldNotify_NoRepeat(t *testing.T) {
+	t.Parallel()
+	//setup
+	device := models.Device{
+		DeviceStatus: pkg.DeviceStatusFailedScrutiny,
+	}
+	smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
+		"5": &measurements.SmartAtaAttribute{
+			Status:           pkg.AttributeStatusFailedScrutiny,
+			TransformedValue: 0,
+		},
+	}}
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl)
+	fakeDatabase.EXPECT().GetSmartAttributeHistory(&gin.Context{}, "", database.DURATION_KEY_FOREVER, 1, 1, []string{"5"}).Return([]measurements.Smart{smartAttrs}, nil).Times(1)
+
+	//assert
+	require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, false, &gin.Context{}, fakeDatabase))
 }
 
 func TestNewPayload(t *testing.T) {

webapp/backend/pkg/version/version.go

@@ -2,4 +2,4 @@ package version
 
 // VERSION is the app-global version string, which will be replaced with a
 // new value during packaging
-const VERSION = "0.7.2"
+const VERSION = "0.8.1"

webapp/backend/pkg/web/handler/archive_device.go

@@ -0,0 +1,22 @@
+package handler
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"net/http"
+)
+
+func ArchiveDevice(c *gin.Context) {
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	err := deviceRepo.UpdateDeviceArchived(c, c.Param("wwn"), true)
+	if err != nil {
+		logger.Errorln("An error occurred while archiving device", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"success": true})
+}

webapp/backend/pkg/web/handler/get_device_details.go

@@ -1,11 +1,12 @@
 package handler
 
 import (
+	"net/http"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"net/http"
 )
 
 func GetDeviceDetails(c *gin.Context) {
@@ -24,7 +25,7 @@ func GetDeviceDetails(c *gin.Context) {
 		durationKey = "forever"
 	}
 
-	smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), durationKey, nil)
+	smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), durationKey, 0, 0, nil)
 	if err != nil {
 		logger.Errorln("An error occurred while retrieving device smart results", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})

webapp/backend/pkg/web/handler/unarchive_device.go

@@ -0,0 +1,22 @@
+package handler
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"net/http"
+)
+
+func UnarchiveDevice(c *gin.Context) {
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	err := deviceRepo.UpdateDeviceArchived(c, c.Param("wwn"), false)
+	if err != nil {
+		logger.Errorln("An error occurred while unarchiving device", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"success": true})
+}

webapp/backend/pkg/web/handler/upload_device_metrics.go

@@ -2,6 +2,8 @@ package handler
 
 import (
 	"fmt"
+	"net/http"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
@@ -9,7 +11,6 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"net/http"
 )
 
 func UploadDeviceMetrics(c *gin.Context) {
@@ -60,7 +61,7 @@ func UploadDeviceMetrics(c *gin.Context) {
 	}
 
 	// save smart temperature data (ignore failures)
-	err = deviceRepo.SaveSmartTemperature(c, c.Param("wwn"), updatedDevice.DeviceProtocol, collectorSmartData)
+	err = deviceRepo.SaveSmartTemperature(c, c.Param("wwn"), updatedDevice.DeviceProtocol, collectorSmartData, appConfig.GetBool(fmt.Sprintf("%s.collector.discard_sct_temp_history", config.DB_USER_SETTINGS_SUBKEY)))
 	if err != nil {
 		logger.Errorln("An error occurred while saving smartctl temp data", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
@@ -69,10 +70,14 @@ func UploadDeviceMetrics(c *gin.Context) {
 
 	//check for error
 	if notify.ShouldNotify(
+		logger,
 		updatedDevice,
 		smartData,
 		pkg.MetricsStatusThreshold(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY))),
 		pkg.MetricsStatusFilterAttributes(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY))),
+		appConfig.GetBool(fmt.Sprintf("%s.metrics.repeat_notifications", config.DB_USER_SETTINGS_SUBKEY)),
+		c,
+		deviceRepo,
 	) {
 		//send notifications
 

webapp/backend/pkg/web/server.go

@@ -42,8 +42,10 @@ func (ae *AppEngine) Setup(logger *logrus.Entry) *gin.Engine {
 			api.GET("/summary/temp", handler.GetDevicesSummaryTempHistory) //used by Dashboard (Temperature history dropdown)
 			api.POST("/device/:wwn/smart", handler.UploadDeviceMetrics)    //used by Collector to upload data
 			api.POST("/device/:wwn/selftest", handler.UploadDeviceSelfTests)
-			api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details
-			api.DELETE("/device/:wwn", handler.DeleteDevice)          //used by UI to delete device
+			api.GET("/device/:wwn/details", handler.GetDeviceDetails)   //used by Details
+			api.POST("/device/:wwn/archive", handler.ArchiveDevice)     //used by UI to archive device
+			api.POST("/device/:wwn/unarchive", handler.UnarchiveDevice) //used by UI to unarchive device
+			api.DELETE("/device/:wwn", handler.DeleteDevice)            //used by UI to delete device
 
 			api.GET("/settings", handler.GetSettings)   //used to get settings
 			api.POST("/settings", handler.SaveSettings) //used to save settings

webapp/backend/pkg/web/server_test.go

@@ -4,6 +4,16 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path"
+	"strings"
+	"testing"
+	"time"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
@@ -14,15 +24,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path"
-	"strings"
-	"testing"
-	"time"
 )
 
 /*
@@ -189,6 +190,8 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() {
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetBool("user.collector.discard_sct_temp_history").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
@@ -247,6 +250,8 @@ func (suite *ServerTestSuite) TestPopulateMultiple() {
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetBool("user.collector.discard_sct_temp_history").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
@@ -529,6 +534,8 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() {
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetBool("user.collector.discard_sct_temp_history").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{})

webapp/frontend/package.json

@@ -47,7 +47,7 @@
         "web-animations-js": "^2.3.2"
     },
     "devDependencies": {
-        "@angular-devkit/build-angular": "v13-lts",
+        "@angular-devkit/build-angular": "v21-lts",
         "@angular/cli": "v13-lts",
         "@angular/compiler-cli": "v13-lts",
         "@angular/language-service": "v13-lts",

webapp/frontend/src/app/core/config/app.config.ts

@@ -12,6 +12,8 @@ export type TemperatureUnit = 'celsius' | 'fahrenheit'
 
 export type LineStroke = 'smooth' | 'straight' | 'stepline'
 
+export type DevicePoweredOnUnit = 'humanize' | 'device_hours'
+
 
 export enum MetricsNotifyLevel {
     Warn = 1,
@@ -47,14 +49,21 @@ export interface AppConfig {
 
     file_size_si_units?: boolean;
 
+    powered_on_hours_unit?: DevicePoweredOnUnit;
+
     line_stroke?: LineStroke;
 
     // Settings from Scrutiny API
+    
+    collector?: {
+        discard_sct_temp_history?: boolean
+    }
 
     metrics?: {
         notify_level?: MetricsNotifyLevel
         status_filter_attributes?: MetricsStatusFilterAttributes
         status_threshold?: MetricsStatusThreshold
+        repeat_notifications?: boolean
     }
 
 }
@@ -76,13 +85,19 @@ export const appConfig: AppConfig = {
 
     temperature_unit: 'celsius',
     file_size_si_units: false,
+    powered_on_hours_unit: 'humanize',
 
     line_stroke: 'smooth',
+    
+    collector: {
+        discard_sct_temp_history : false,
+    },
 
     metrics: {
         notify_level: MetricsNotifyLevel.Fail,
         status_filter_attributes: MetricsStatusFilterAttributes.All,
-        status_threshold: MetricsStatusThreshold.Both
+        status_threshold: MetricsStatusThreshold.Both,
+        repeat_notifications: true
     }
 };
 

webapp/frontend/src/app/core/models/device-model.ts

@@ -1,5 +1,6 @@
 // maps to webapp/backend/pkg/models/device.go
 export interface DeviceModel {
+    archived?: boolean;
     wwn: string;
     device_name?: string;
     device_uuid?: string;

webapp/frontend/src/app/data/mock/device/details/sda.ts

@@ -20,7 +20,8 @@ export const sda = {
             'device_type': '',
             'label': '',
             'host_id': '',
-            'device_status': 0
+            'device_status': 0,
+            'archived': false
         },
         'smart_results': [{
             'date': '2021-10-24T23:20:44Z',

webapp/frontend/src/app/data/mock/summary/data.ts

@@ -28,7 +28,8 @@ export const summary = {
                     'device_type': '',
                     'label': '',
                     'host_id': '',
-                    'device_status': 0
+                    'device_status': 0,
+                    'archived': false
                 }
             },
             '0x5000cca252c859cc': {
@@ -55,7 +56,8 @@ export const summary = {
                     'device_type': '',
                     'label': '',
                     'host_id': '',
-                    'device_status': 0
+                    'device_status': 0,
+                    'archived': false
                 },
                 'smart': {
                     'collector_date': '2020-08-21T22:27:02Z',
@@ -91,7 +93,8 @@ export const summary = {
                     'device_type': '',
                     'label': '',
                     'host_id': '',
-                    'device_status': 0
+                    'device_status': 0,
+                    'archived': false
                 },
                 'smart': {
                     'collector_date': '2020-06-21T00:03:30Z',
@@ -127,7 +130,8 @@ export const summary = {
                     'device_type': '',
                     'label': '',
                     'host_id': '',
-                    'device_status': 0
+                    'device_status': 0,
+                    'archived': false
                 }
             },
             '0x5000cca264ec3183': {
@@ -154,7 +158,8 @@ export const summary = {
                     'device_type': '',
                     'label': '',
                     'host_id': 'custom host id',
-                    'device_status': 1
+                    'device_status': 1,
+                    'archived': false
                 },
                 'smart': {
                     'collector_date': '2020-09-13T16:29:23Z',
@@ -574,7 +579,8 @@ export const summary = {
                     'device_type': '',
                     'label': '',
                     'host_id': '',
-                    'device_status': 0
+                    'device_status': 0,
+                    'archived': false
                 }
             },
             '0x5002538e40a22954': {
@@ -601,7 +607,8 @@ export const summary = {
                     'device_type': '',
                     'label': '',
                     'host_id': '',
-                    'device_status': 0
+                    'device_status': 0,
+                    'archived': false
                 },
                 'smart': {
                     'collector_date': '2020-06-10T12:01:02Z',

webapp/frontend/src/app/layout/common/dashboard-device-archive-dialog/dashboard-device-archive-dialog.component.html

@@ -0,0 +1,11 @@
+<h2 mat-dialog-title>Archive {{data.title}}?</h2>
+<mat-dialog-content>This will remove the device from Scrutiny dashboard, unless you toggle show archived. <strong>Any data about the device
+    itself will remain untouched.</strong></mat-dialog-content>
+<mat-dialog-actions>
+    <button mat-button mat-dialog-close>Cancel</button>
+    <button class="yellow-600" mat-button (click)="onArchiveClick()">
+        <mat-icon class="icon-size-20 mr-3"
+                  [svgIcon]="'archive'"></mat-icon>
+        Archive
+    </button>
+</mat-dialog-actions>

webapp/frontend/src/app/layout/common/dashboard-device-archive-dialog/dashboard-device-archive-dialog.component.spec.ts

@@ -0,0 +1,64 @@
+import {async, ComponentFixture, TestBed} from '@angular/core/testing';
+
+import {DashboardDeviceArchiveDialogComponent} from './dashboard-device-archive-dialog.component';
+import {HttpClientModule} from '@angular/common/http';
+import {MAT_DIALOG_DATA, MatDialogModule, MatDialogRef} from '@angular/material/dialog';
+import {MatButtonModule} from '@angular/material/button';
+import {MatIconModule} from '@angular/material/icon';
+import {SharedModule} from '../../../shared/shared.module';
+import {DashboardDeviceArchiveDialogService} from './dashboard-device-archive-dialog.service';
+import {of} from 'rxjs';
+
+
+describe('DashboardDeviceArchiveDialogComponent', () => {
+    let component: DashboardDeviceArchiveDialogComponent;
+    let fixture: ComponentFixture<DashboardDeviceArchiveDialogComponent>;
+
+    const matDialogRefSpy = jasmine.createSpyObj('MatDialogRef', ['closeDialog', 'close']);
+    const dashboardDeviceArchiveDialogServiceSpy = jasmine.createSpyObj('DashboardDeviceArchiveDialogService', ['archiveDevice']);
+
+    beforeEach(async(() => {
+        TestBed.configureTestingModule({
+            imports: [
+                HttpClientModule,
+                MatDialogModule,
+                MatButtonModule,
+                MatIconModule,
+                SharedModule,
+            ],
+            providers: [
+                {provide: MatDialogRef, useValue: matDialogRefSpy},
+                {provide: MAT_DIALOG_DATA, useValue: {wwn: 'test-wwn', title: 'my-test-device-title'}},
+                {provide: DashboardDeviceArchiveDialogService, useValue: dashboardDeviceArchiveDialogServiceSpy}
+            ],
+            declarations: [DashboardDeviceArchiveDialogComponent]
+        })
+            .compileComponents()
+    }));
+
+    beforeEach(() => {
+        fixture = TestBed.createComponent(DashboardDeviceArchiveDialogComponent);
+        component = fixture.componentInstance;
+        fixture.detectChanges();
+    });
+
+    it('should create', () => {
+        expect(component).toBeTruthy();
+    });
+
+    it('should close the component if cancel is clicked', () => {
+        matDialogRefSpy.closeDialog.calls.reset();
+        matDialogRefSpy.closeDialog()
+        expect(matDialogRefSpy.closeDialog).toHaveBeenCalled();
+    });
+
+    it('should attempt to archive device if archive is clicked', () => {
+        dashboardDeviceArchiveDialogServiceSpy.archiveDevice.and.returnValue(of({'success': true}));
+
+        component.onArchiveClick()
+        expect(dashboardDeviceArchiveDialogServiceSpy.archiveDevice).toHaveBeenCalledWith('test-wwn');
+        expect(dashboardDeviceArchiveDialogServiceSpy.archiveDevice.calls.count())
+            .withContext('one call')
+            .toBe(1);
+    });
+});

webapp/frontend/src/app/layout/common/dashboard-device-archive-dialog/dashboard-device-archive-dialog.component.ts

@@ -0,0 +1,29 @@
+import {Component, Inject, OnInit} from '@angular/core';
+import {MAT_DIALOG_DATA, MatDialogRef} from '@angular/material/dialog';
+import {DashboardDeviceArchiveDialogService} from 'app/layout/common/dashboard-device-archive-dialog/dashboard-device-archive-dialog.service';
+
+@Component({
+  selector: 'app-dashboard-device-archive-dialog',
+  templateUrl: './dashboard-device-archive-dialog.component.html',
+  styleUrls: ['./dashboard-device-archive-dialog.component.scss'],
+})
+export class DashboardDeviceArchiveDialogComponent implements OnInit {
+
+    constructor(
+        public dialogRef: MatDialogRef<DashboardDeviceArchiveDialogComponent>,
+        @Inject(MAT_DIALOG_DATA) public data: {wwn: string, title: string},
+        private _archiveService: DashboardDeviceArchiveDialogService,
+    ) {
+    }
+
+  ngOnInit(): void {
+  }
+
+  onArchiveClick(): void {
+      this._archiveService.archiveDevice(this.data.wwn)
+          .subscribe((data) => {
+              this.dialogRef.close(data);
+          });
+
+  }
+}

webapp/frontend/src/app/layout/common/dashboard-device-archive-dialog/dashboard-device-archive-dialog.module.ts

@@ -0,0 +1,29 @@
+import {NgModule} from '@angular/core';
+import {RouterModule} from '@angular/router';
+import {MatButtonModule} from '@angular/material/button';
+import {MatIconModule} from '@angular/material/icon';
+import {SharedModule} from 'app/shared/shared.module';
+import {dashboardRoutes} from 'app/modules/dashboard/dashboard.routing';
+import {MatDialogModule} from '@angular/material/dialog';
+import {DashboardDeviceArchiveDialogComponent} from './dashboard-device-archive-dialog.component';
+
+@NgModule({
+    declarations: [
+        DashboardDeviceArchiveDialogComponent
+    ],
+    imports: [
+        RouterModule.forChild([]),
+        RouterModule.forChild(dashboardRoutes),
+        MatButtonModule,
+        MatIconModule,
+        SharedModule,
+        MatDialogModule
+    ],
+    exports     : [
+        DashboardDeviceArchiveDialogComponent,
+    ],
+    providers   : []
+})
+export class DashboardDeviceArchiveDialogModule
+{
+}

webapp/frontend/src/app/layout/common/dashboard-device-archive-dialog/dashboard-device-archive-dialog.service.ts

@@ -0,0 +1,38 @@
+import {Injectable} from '@angular/core';
+import {HttpClient} from '@angular/common/http';
+import {Observable} from 'rxjs';
+import {getBasePath} from 'app/app.routing';
+
+@Injectable({
+    providedIn: 'root'
+})
+export class DashboardDeviceArchiveDialogService
+{
+
+
+    /**
+     * Constructor
+     *
+     * @param {HttpClient} _httpClient
+     */
+    constructor(
+        private _httpClient: HttpClient
+    )
+    {
+    }
+
+    // -----------------------------------------------------------------------------------------------------
+    // @ Public methods
+    // -----------------------------------------------------------------------------------------------------
+
+
+    archiveDevice(wwn: string): Observable<any>
+    {
+        return this._httpClient.post( `${getBasePath()}/api/device/${wwn}/archive`, {});
+    }
+
+    unarchiveDevice(wwn: string): Observable<any>
+    {
+        return this._httpClient.post( `${getBasePath()}/api/device/${wwn}/unarchive`, {});
+    }
+}

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component.html

@@ -1,10 +1,12 @@
 <h2 mat-dialog-title>Delete {{data.title}}?</h2>
-<mat-dialog-content>This will delete all data associated with this device (including all historical data).</mat-dialog-content>
+<mat-dialog-content>This will remove the device and all historical data from Scrutiny. <strong>Any data on the device
+    itself will remain untouched.</strong></mat-dialog-content>
 <mat-dialog-actions>
     <button mat-button mat-dialog-close>Cancel</button>
     <!-- The mat-dialog-close directive optionally accepts a value as a result for the dialog. -->
     <button class="red-600" mat-button (click)="onDeleteClick()">
         <mat-icon class="icon-size-20 mr-3"
                   [svgIcon]="'delete_forever'"></mat-icon>
-        Delete</button>
+        Delete
+    </button>
 </mat-dialog-actions>

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html

@@ -1,5 +1,7 @@
-<div [ngClass]="{ 'border-green': deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) == 'passed',
-                                'border-red': deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) == 'failed' }"
+<div
+        [ngClass]="{ 'border-green': deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) == 'passed',
+                                'border-red': deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) == 'failed',
+                                 'text-disabled': deviceSummary.device.archived }"
      class="relative flex flex-col flex-auto p-6 pr-3 pb-3 bg-card rounded border-l-4 shadow-md overflow-hidden">
     <div class="absolute bottom-0 right-0 w-24 h-24 -m-6">
         <mat-icon class="icon-size-96 opacity-12 text-green"
@@ -21,6 +23,8 @@
             </div>
         </div>
         <div class="ml-auto" *ngIf="deviceSummary.device">
+            <mat-icon *ngIf="deviceSummary.device.archived"
+                      [svgIcon]="'archive'"></mat-icon>
             <button mat-icon-button
                     [matMenuTriggerFor]="previousStatementMenu">
                 <mat-icon [svgIcon]="'more_vert'"></mat-icon>
@@ -33,6 +37,14 @@
                                     <span>View Details</span>
                                 </span>
                 </a>
+                <a mat-menu-item
+                   (click)="openArchiveDialog()">
+                                <span class="flex items-center">
+                                    <mat-icon class="icon-size-20 mr-3"
+                                              [svgIcon]="deviceSummary.device.archived ? 'unarchive':'archive'"></mat-icon>
+                                    <span>{{deviceSummary.device.archived ? "Unarchive" : "Archive"}} Device</span>
+                                </span>
+                </a>
                 <a mat-menu-item (click)="openDeleteDialog()">
                                 <span class="flex items-center">
                                     <mat-icon class="icon-size-20 mr-3"
@@ -63,7 +75,7 @@
         </div>
         <div class="flex flex-col mx-6 my-3 xs:w-full">
             <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Powered On</div>
-            <div class="mt-2 font-medium text-3xl leading-none" *ngIf="deviceSummary.smart?.power_on_hours; else unknownPoweredOn">{{ humanizeDuration(deviceSummary.smart?.power_on_hours * 60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] }) }}</div>
+            <div class="mt-2 font-medium text-3xl leading-none" *ngIf="deviceSummary.smart?.power_on_hours; else unknownPoweredOn">{{ deviceSummary.smart?.power_on_hours | deviceHours:config.powered_on_hours_unit:{ round: true, largest: 1, units: ['y', 'd', 'h'] } }}</div>
             <ng-template #unknownPoweredOn><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
         </div>
     </div>

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.scss

@@ -0,0 +1,3 @@
+.text-disabled{
+    opacity: 0.8;
+}

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts

@@ -4,12 +4,13 @@ import {takeUntil} from 'rxjs/operators';
 import {AppConfig} from 'app/core/config/app.config';
 import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service';
 import {Subject} from 'rxjs';
-import humanizeDuration from 'humanize-duration'
 import {MatDialog} from '@angular/material/dialog';
 import {DashboardDeviceDeleteDialogComponent} from 'app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component';
 import {DeviceTitlePipe} from 'app/shared/device-title.pipe';
 import {DeviceSummaryModel} from 'app/core/models/device-summary-model';
 import {DeviceStatusPipe} from 'app/shared/device-status.pipe';
+import {DashboardDeviceArchiveDialogComponent} from '../dashboard-device-archive-dialog/dashboard-device-archive-dialog.component';
+import {DashboardDeviceArchiveDialogService} from '../dashboard-device-archive-dialog/dashboard-device-archive-dialog.service';
 
 @Component({
     selector: 'app-dashboard-device',
@@ -20,6 +21,7 @@ export class DashboardDeviceComponent implements OnInit {
 
     constructor(
         private _configService: ScrutinyConfigService,
+        private _archiveService: DashboardDeviceArchiveDialogService,
         public dialog: MatDialog,
     ) {
         // Set the private defaults
@@ -27,15 +29,14 @@ export class DashboardDeviceComponent implements OnInit {
     }
 
     @Input() deviceSummary: DeviceSummaryModel;
-    @Input() deviceWWN: string;
+    @Output() deviceArchived = new EventEmitter<string>();
+    @Output() deviceUnarchived = new EventEmitter<string>();
     @Output() deviceDeleted = new EventEmitter<string>();
 
     config: AppConfig;
 
     private _unsubscribeAll: Subject<void>;
 
-    readonly humanizeDuration = humanizeDuration;
-
     deviceStatusForModelWithThreshold = DeviceStatusPipe.deviceStatusForModelWithThreshold
 
     ngOnInit(): void {
@@ -72,11 +73,33 @@ export class DashboardDeviceComponent implements OnInit {
         }
     }
 
+    openArchiveDialog(): void {
+        if(this.deviceSummary.device.archived){
+            this._archiveService.unarchiveDevice(this.deviceSummary.device.wwn).subscribe((result) => {
+                if(result) {
+                    this.deviceUnarchived.emit(this.deviceSummary.device.wwn)
+                }
+            })
+            return;
+        }
+        const dialogRef = this.dialog.open(DashboardDeviceArchiveDialogComponent, {
+            data: {
+                wwn: this.deviceSummary.device.wwn,
+                title: DeviceTitlePipe.deviceTitleWithFallback(this.deviceSummary.device, this.config.dashboard_display)
+            }
+        });
+        dialogRef.afterClosed().subscribe(result => {
+            if(result) {
+                this.deviceArchived.emit(this.deviceSummary.device.wwn);
+            }
+        })
+    }
+
     openDeleteDialog(): void {
         const dialogRef = this.dialog.open(DashboardDeviceDeleteDialogComponent, {
             // width: '250px',
             data: {
-                wwn: this.deviceWWN,
+                wwn: this.deviceSummary.device.wwn,
                 title: DeviceTitlePipe.deviceTitleWithFallback(this.deviceSummary.device, this.config.dashboard_display)
             }
         });
@@ -84,7 +107,7 @@ export class DashboardDeviceComponent implements OnInit {
         dialogRef.afterClosed().subscribe(result => {
             console.log('The dialog was closed', result);
             if (result.success) {
-                this.deviceDeleted.emit(this.deviceWWN)
+                this.deviceDeleted.emit(this.deviceSummary.device.wwn)
             }
         });
     }

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.module.ts

@@ -7,6 +7,7 @@ import {DashboardDeviceComponent} from 'app/layout/common/dashboard-device/dashb
 import {dashboardRoutes} from '../../../modules/dashboard/dashboard.routing';
 import {MatMenuModule} from '@angular/material/menu';
 import {DashboardDeviceDeleteDialogModule} from 'app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.module';
+import {DashboardDeviceArchiveDialogModule} from '../dashboard-device-archive-dialog/dashboard-device-archive-dialog.module';
 
 @NgModule({
     declarations: [
@@ -19,7 +20,8 @@ import {DashboardDeviceDeleteDialogModule} from 'app/layout/common/dashboard-dev
         MatIconModule,
         MatMenuModule,
         SharedModule,
-        DashboardDeviceDeleteDialogModule
+        DashboardDeviceDeleteDialogModule,
+        DashboardDeviceArchiveDialogModule
     ],
     exports: [
         DashboardDeviceComponent,

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html

@@ -54,6 +54,14 @@
         </div>
 
         <div class="flex flex-col mt-5 gt-md:flex-row">
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Powered On Format</mat-label>
+                <mat-select [(ngModel)]="poweredOnHoursUnit">
+                    <mat-option value="humanize">Humanize</mat-option>
+                    <mat-option value="device_hours">Device Hours</mat-option>
+                </mat-select>
+            </mat-form-field>
+
             <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
                 <mat-label>Line stroke</mat-label>
                 <mat-select [(ngModel)]="lineStroke">
@@ -84,6 +92,33 @@
                 </mat-select>
             </mat-form-field>
         </div>
+
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Repeat Notifications</mat-label>
+                <mat-select [(ngModel)]=repeatNotifications>
+                    <mat-option [value]=true>Always</mat-option>
+                    <mat-option [value]=false>Only when the value has changed</mat-option>
+                </mat-select>
+            </mat-form-field>
+        </div>
+
+        <div class="mt-6 mb-2">
+            <h3 class="text-lg font-medium">Quirks</h3>
+            <div class="w-full border-b mt-1"></div>
+        </div>
+
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3"
+                matTooltip="Discard historical temperature data retrieved from the SMART Command Transport (SCT) Data Table, which may be inaccurate for some drives. The current temperature is always stored."
+                matTooltipPosition="right">
+                <mat-label>Discard SCT Temperature History</mat-label>
+                <mat-select [(ngModel)]=discardSCTTempHistory>
+                    <mat-option [value]=true>Enabled</mat-option>
+                    <mat-option [value]=false>Disabled</mat-option>
+                </mat-select>
+            </mat-form-field>
+        </div>
     </div>
 
 </mat-dialog-content>

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts

@@ -7,7 +7,8 @@ import {
     MetricsStatusThreshold,
     TemperatureUnit,
     LineStroke,
-    Theme
+    Theme,
+    DevicePoweredOnUnit
 } from 'app/core/config/app.config';
 import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service';
 import {Subject} from 'rxjs';
@@ -24,10 +25,13 @@ export class DashboardSettingsComponent implements OnInit {
     dashboardSort: string;
     temperatureUnit: string;
     fileSizeSIUnits: boolean;
+    poweredOnHoursUnit: string;
     lineStroke: string;
     theme: string;
+    discardSCTTempHistory: boolean;
     statusThreshold: number;
     statusFilterAttributes: number;
+    repeatNotifications: boolean;
 
     // Private
     private _unsubscribeAll: Subject<void>;
@@ -50,11 +54,15 @@ export class DashboardSettingsComponent implements OnInit {
                 this.dashboardSort = config.dashboard_sort;
                 this.temperatureUnit = config.temperature_unit;
                 this.fileSizeSIUnits = config.file_size_si_units;
+                this.poweredOnHoursUnit = config.powered_on_hours_unit;
                 this.lineStroke = config.line_stroke;
                 this.theme = config.theme;
 
+                this.discardSCTTempHistory = config.collector.discard_sct_temp_history;
+
                 this.statusFilterAttributes = config.metrics.status_filter_attributes;
                 this.statusThreshold = config.metrics.status_threshold;
+                this.repeatNotifications = config.metrics.repeat_notifications;
 
             });
 
@@ -66,11 +74,16 @@ export class DashboardSettingsComponent implements OnInit {
             dashboard_sort: this.dashboardSort as DashboardSort,
             temperature_unit: this.temperatureUnit as TemperatureUnit,
             file_size_si_units: this.fileSizeSIUnits,
+            powered_on_hours_unit: this.poweredOnHoursUnit as DevicePoweredOnUnit,
             line_stroke: this.lineStroke as LineStroke,
             theme: this.theme as Theme,
+            collector: {
+                discard_sct_temp_history: this.discardSCTTempHistory
+            },
             metrics: {
                 status_filter_attributes: this.statusFilterAttributes as MetricsStatusFilterAttributes,
-                status_threshold: this.statusThreshold as MetricsStatusThreshold
+                status_threshold: this.statusThreshold as MetricsStatusThreshold,
+                repeat_notifications: this.repeatNotifications
             }
         }
         this._configService.config = newSettings

webapp/frontend/src/app/modules/dashboard/dashboard.component.html

@@ -1,4 +1,3 @@
-
 <div *ngIf="summaryData; else emptyDashboard">
     <div class="flex flex-col flex-auto w-full p-8 xs:p-2">
 
@@ -11,7 +10,15 @@
                 </div>
                 <!-- Action buttons -->
                 <div class="flex items-center">
-                    <button matTooltip="not yet implemented" class="xs:hidden" mat-stroked-button>
+                    <button class="xs:hidden" mat-stroked-button
+                            [color]="showArchived ? 'primary' : null"
+                            (click)="showArchived=!showArchived">
+                        <mat-icon class="icon-size-20"
+                                  [color]="showArchived ? 'primary' : null"
+                                  [svgIcon]="'archive'"></mat-icon>
+                                <span class="ml-2">Archived</span>
+                    </button>
+                    <button matTooltip="not yet implemented" class="ml-2 xs:hidden" mat-stroked-button>
                         <mat-icon class="icon-size-20"
                                   [svgIcon]="'save'"></mat-icon>
                         <span class="ml-2">Export</span>
@@ -31,6 +38,12 @@
                             <mat-icon [svgIcon]="'more_vert'"></mat-icon>
                         </button>
                         <mat-menu #actionsMenu="matMenu">
+                            <button mat-menu-item (click)="showArchived=!showArchived">
+                                <mat-icon class="icon-size-20"
+                                          [color]="showArchived ? 'primary' : null"
+                                          [svgIcon]="'archive'"></mat-icon>
+                                <span class="ml-2">Archived</span>
+                            </button>
                             <button mat-menu-item
                                     matTooltip="not yet implemented">
                                 <mat-icon class="icon-size-20"
@@ -49,13 +62,16 @@
 
 
             <div class="flex flex-wrap w-full" *ngFor="let hostId of hostGroups | keyvalue">
-                <h3 class="ml-4" *ngIf="hostId.key">{{hostId.key}}</h3>
+                <h3 class="ml-4" *ngIf="hostId.key">{{ hostId.key }}</h3>
                 <div class="flex flex-wrap w-full">
-                    <app-dashboard-device (deviceDeleted)="onDeviceDeleted($event)"
-                                          class="flex gt-sm:w-1/2 min-w-80 p-4"
-                                          *ngFor="let deviceSummary of (deviceSummariesForHostGroup(hostId.value) | deviceSort:config.dashboard_sort:config.dashboard_display )"
-                                          [deviceWWN]="deviceSummary.device.wwn"
-                                          [deviceSummary]="deviceSummary"></app-dashboard-device>
+                    <ng-container *ngFor="let deviceSummary of (deviceSummariesForHostGroup(hostId.value) | deviceSort:config.dashboard_sort:config.dashboard_display )">
+                        <app-dashboard-device *ngIf="showArchived || !deviceSummary.device.archived"
+                                              (deviceArchived)="onDeviceArchived($event)"
+                                              (deviceUnarchived)="onDeviceUnarchived($event)"
+                                              (deviceDeleted)="onDeviceDeleted($event)"
+                                              class="flex gt-sm:w-1/2 min-w-80 p-4"
+                                              [deviceSummary]="deviceSummary"></app-dashboard-device>
+                    </ng-container>
                 </div>
             </div>
 
@@ -67,13 +83,13 @@
                         <div class="flex items-center justify-between">
                             <div class="flex flex-col">
                                 <div class="font-bold text-md text-secondary uppercase tracking-wider mr-4">Temperature</div>
-                                <div class="text-sm text-hint font-medium">Temperature history for each device </div>
+                                <div class="text-sm text-hint font-medium">Temperature history for each device</div>
                             </div>
                             <div>
                                 <button class="h-8 min-h-8 px-2"
                                         mat-button
                                         [matMenuTriggerFor]="tempRangeMenu">
-                                    <span class="font-medium text-sm text-hint">{{tempDurationKey}}</span>
+                                    <span class="font-medium text-sm text-hint">{{ tempDurationKey }}</span>
                                 </button>
                                 <mat-menu #tempRangeMenu="matMenu">
                                     <button (click)="changeSummaryTempDuration('forever')" mat-menu-item>forever</button>
@@ -109,7 +125,8 @@
              src="assets/images/dashboard-placeholder.png">
 
         <h1>No Devices Detected!</h1>
-        <p style="max-width:700px;">Scrutiny includes a Collector agent that you must run on all of your systems. The Collector is responsible for detecting connected storage devices and collecting S.M.A.R.T data on a configurable schedule.</p>
+        <p style="max-width:700px;">Scrutiny includes a Collector agent that you must run on all of your systems. The Collector is responsible for detecting connected storage
+            devices and collecting S.M.A.R.T data on a configurable schedule.</p>
 
         <p><br/>You can trigger the Collector manually by running the following command, then refreshing this page:</p>
         <code>scrutiny-collector-metrics run</code>

webapp/frontend/src/app/modules/dashboard/dashboard.component.ts

@@ -34,6 +34,7 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
     temperatureOptions: ApexOptions;
     tempDurationKey = 'forever'
     config: AppConfig;
+    showArchived: boolean;
 
     // Private
     private _unsubscribeAll: Subject<void>;
@@ -159,9 +160,18 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
 
             for(const tempHistory of deviceSummary.temp_history){
                 const newDate = new Date(tempHistory.date);
+                let temperature;
+                switch (this.config.temperature_unit) {
+                    case 'celsius':
+                        temperature = tempHistory.temp;
+                        break
+                    case 'fahrenheit':
+                        temperature = TemperaturePipe.celsiusToFahrenheit(tempHistory.temp)
+                        break
+                }
                 deviceSeriesMetadata.data.push({
                     x: newDate,
-                    y: TemperaturePipe.formatTemperature(tempHistory.temp, this.config.temperature_unit, false)
+                    y: temperature
                 })
             }
             deviceTemperatureSeries.push(deviceSeriesMetadata)
@@ -206,6 +216,8 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
             },
             tooltip: {
                 theme: 'dark',
+                shared: true,
+                intersect: false,
                 x    : {
                     format: 'MMM dd, yyyy HH:mm:ss'
                 },
@@ -248,6 +260,14 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
         delete this.summaryData[wwn] // remove the device from the summary list.
     }
 
+    onDeviceArchived(wwn: string): void {
+        this.summaryData[wwn].device.archived = true;
+    }
+
+    onDeviceUnarchived(wwn: string): void {
+        this.summaryData[wwn].device.archived = false;
+    }
+
     /*
 
     DURATION_KEY_WEEK    = "week"

webapp/frontend/src/app/modules/detail/detail.component.html

@@ -123,7 +123,7 @@
                         <div class="text-secondary text-md">Power Cycle Count</div>
                     </div>
                     <div *ngIf="smart_results[0]?.power_on_hours"  class="my-2 col-span-2 lt-md:col-span-1">
-                        <div matTooltip="{{humanizeDuration(smart_results[0]?.power_on_hours * 60 * 60 * 1000, { conjunction: ' and ', serialComma: false })}}">{{humanizeDuration(smart_results[0]?.power_on_hours *60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] })}}</div>
+                        <div matTooltip="{{humanizeDuration(smart_results[0]?.power_on_hours * 60 * 60 * 1000, { conjunction: ' and ', serialComma: false })}}">{{ smart_results[0]?.power_on_hours | deviceHours:config.powered_on_hours_unit:{ round: true, largest: 1, units: ['y', 'd', 'h'] } }}</div>
                         <div class="text-secondary text-md">Powered On</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">

webapp/frontend/src/app/shared/device-hours.pipe.spec.ts

@@ -0,0 +1,52 @@
+import { DeviceHoursPipe } from "./device-hours.pipe";
+
+describe("DeviceHoursPipe", () => {
+  it("create an instance", () => {
+    const pipe = new DeviceHoursPipe();
+    expect(pipe).toBeTruthy();
+  });
+
+  describe("#transform", () => {
+    const testCases = [
+      {
+        input: 12345,
+        configuration: "device_hours",
+        result: "12345 hours",
+      },
+      {
+        input: 15273,
+        configuration: "humanize",
+        result: "1 year, 8 months, 3 weeks, 6 days, 15 hours",
+      },
+      {
+        input: 48,
+        configuration: null,
+        result: "2 days",
+      },
+      {
+        input: 168,
+        configuration: "scrutiny",
+        result: "1 week",
+      },
+      {
+        input: null,
+        configuration: "device_hours",
+        result: "Unknown",
+      },
+      {
+        input: null,
+        configuration: "humanize",
+        result: "Unknown",
+      },
+    ];
+
+    testCases.forEach((test, index) => {
+      it(`format input '${test.input}' with configuration '${test.configuration}', should be '${test.result}' (testcase: ${index + 1})`, () => {
+        // test
+        const pipe = new DeviceHoursPipe();
+        const formatted = pipe.transform(test.input, test.configuration);
+        expect(formatted).toEqual(test.result);
+      });
+    });
+  });
+});

webapp/frontend/src/app/shared/device-hours.pipe.ts

@@ -0,0 +1,19 @@
+import { Pipe, PipeTransform } from '@angular/core';
+import humanizeDuration from 'humanize-duration';
+
+@Pipe({ name: 'deviceHours' })
+export class DeviceHoursPipe implements PipeTransform {
+    static format(hoursOfRunTime: number, unit: string, humanizeConfig: object): string {
+      if (hoursOfRunTime === null) {
+        return 'Unknown';
+      }
+      if (unit === 'device_hours') {
+          return `${hoursOfRunTime} hours`;
+        }
+        return humanizeDuration(hoursOfRunTime * 60 * 60 * 1000, humanizeConfig);
+    }
+
+  transform(hoursOfRunTime: number, unit = 'humanize', humanizeConfig: any = {}): string {
+        return DeviceHoursPipe.format(hoursOfRunTime, unit, humanizeConfig)
+  }
+}

webapp/frontend/src/app/shared/shared.module.ts

@@ -6,6 +6,7 @@ import { DeviceSortPipe } from './device-sort.pipe';
 import { TemperaturePipe } from './temperature.pipe';
 import { DeviceTitlePipe } from './device-title.pipe';
 import { DeviceStatusPipe } from './device-status.pipe';
+import { DeviceHoursPipe } from './device-hours.pipe';
 
 @NgModule({
     declarations: [
@@ -13,7 +14,8 @@ import { DeviceStatusPipe } from './device-status.pipe';
         DeviceSortPipe,
         TemperaturePipe,
         DeviceTitlePipe,
-        DeviceStatusPipe
+        DeviceStatusPipe,
+        DeviceHoursPipe
     ],
     imports: [
         CommonModule,
@@ -28,7 +30,8 @@ import { DeviceStatusPipe } from './device-status.pipe';
         DeviceSortPipe,
         DeviceTitlePipe,
         DeviceStatusPipe,
-        TemperaturePipe
+        TemperaturePipe,
+        DeviceHoursPipe
     ]
 })
 export class SharedModule

webapp/frontend/src/app/shared/temperature.pipe.spec.ts

@@ -11,16 +11,16 @@ describe('TemperaturePipe', () => {
         const testCases = [
             {
                 'c': -273.15,
-                'f': -460,
+                'f': -459.66999999999996,
             },{
                 'c': -34.44,
-                'f': -30,
+                'f': -29.991999999999997,
             },{
                 'c': -23.33,
-                'f': -10,
+                'f': -9.993999999999993,
             },{
                 'c': -17.78,
-                'f': -0,
+                'f': -0.0040000000000048885,
             },{
                 'c': 0,
                 'f': 32,
@@ -29,10 +29,10 @@ describe('TemperaturePipe', () => {
                 'f': 50,
             },{
                 'c': 26.67,
-                'f': 80,
+                'f': 80.006,
             },{
                 'c': 37,
-                'f': 99,
+                'f': 98.6,
             },{
                 'c': 60,
                 'f': 140,
@@ -42,8 +42,7 @@ describe('TemperaturePipe', () => {
             it(`should correctly convert ${test.c}, Celsius to Fahrenheit (testcase: ${index + 1})`, () => {
                 // test
                 const numb = TemperaturePipe.celsiusToFahrenheit(test.c)
-                const roundNumb = Math.round(numb);
-                expect(roundNumb).toEqual(test.f);
+                expect(numb).toEqual(test.f);
             });
         })
     });
@@ -55,6 +54,11 @@ describe('TemperaturePipe', () => {
                 'unit': 'celsius',
                 'includeUnits': true,
                 'result': '26.67°C'
+            },{
+                'c': 26.6767,
+                'unit': 'celsius',
+                'includeUnits': true,
+                'result': '26.677°C'
             },{
                 'c': 26.67,
                 'unit': 'celsius',
@@ -64,12 +68,17 @@ describe('TemperaturePipe', () => {
                 'c': 26.67,
                 'unit': 'fahrenheit',
                 'includeUnits': true,
-                'result': '80.006°F',
+                'result': '26.67°F',
+            },{
+                'c': 26.6767,
+                'unit': 'fahrenheit',
+                'includeUnits': true,
+                'result': '26.677°F',
             },{
                 'c': 26.67,
                 'unit': 'fahrenheit',
                 'includeUnits': false,
-                'result': '80.006',
+                'result': '26.67',
             }
         ]
         testCases.forEach((test, index) => {

webapp/frontend/src/app/shared/temperature.pipe.ts

@@ -6,29 +6,35 @@ import {formatNumber} from '@angular/common';
 })
 export class TemperaturePipe implements PipeTransform {
     static celsiusToFahrenheit(celsiusTemp: number): number {
-        return celsiusTemp * 9.0 / 5.0 + 32;
+        return celsiusTemp * 9/5 + 32;
     }
-    static formatTemperature(celsiusTemp: number, unit: string, includeUnits: boolean): number|string {
-        let convertedTemp
-        let convertedUnitSuffix
+    static formatTemperature(temp: number, unit: string, includeUnits: boolean): number|string {
+        let unitSuffix
         switch (unit) {
             case 'celsius':
-                convertedTemp = celsiusTemp
-                convertedUnitSuffix = '°C'
+                unitSuffix = '°C'
                 break
             case 'fahrenheit':
-                convertedTemp = TemperaturePipe.celsiusToFahrenheit(celsiusTemp)
-                convertedUnitSuffix = '°F'
+                unitSuffix = '°F'
                 break
         }
         if(includeUnits){
-            return formatNumber(convertedTemp, 'en-US') + convertedUnitSuffix
+            return formatNumber(temp, 'en-US') + unitSuffix
         } else {
-            return formatNumber(convertedTemp, 'en-US',)
+            return formatNumber(temp, 'en-US',)
         }
     }
 
   transform(celsiusTemp: number, unit = 'celsius', includeUnits = false): number|string {
+        let temperature;
+        switch (unit) {
+            case 'celsius':
+                temperature = celsiusTemp;
+                break
+            case 'fahrenheit':
+                temperature = TemperaturePipe.celsiusToFahrenheit(celsiusTemp)
+                break
+        }
         return TemperaturePipe.formatTemperature(celsiusTemp, unit, includeUnits)
   }