(v0.4.9) Automated packaging of release by Packagr

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

trying to fix docker image builds (take 1h+ right now).

.dockerignore

@@ -1,4 +1,3 @@
-/dist
 /vendor
 /.idea
 /.github

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,20 +22,19 @@ See [/docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](docs/TROUBLESHOOTING_DEVICE_COLL
 
 ```
 docker run -it --rm -p 8080:8080 \
+-v `pwd`/config:/opt/scrutiny/config \
 -v /run/udev:/run/udev:ro \
 --cap-add SYS_RAWIO \
 --device=/dev/sda \
 --device=/dev/sdb \
 -e DEBUG=true \
--e COLLECTOR_LOG_FILE=/tmp/collector.log \
--e SCRUTINY_LOG_FILE=/tmp/web.log \
+-e COLLECTOR_LOG_FILE=/opt/scrutiny/config/collector.log \
+-e SCRUTINY_LOG_FILE=/opt/scrutiny/config/web.log \
 --name scrutiny \
 ghcr.io/analogj/scrutiny:master-omnibus
 
 # in another terminal trigger the collector
 docker exec scrutiny scrutiny-collector-metrics run
-
-# then use docker cp to copy the log files out of the container.
-docker cp scrutiny:/tmp/collector.log collector.log
-docker cp scrutiny:/tmp/web.log web.log
 ```
+
+The log files will be available on your host in the `config` directory. Please attach them to this issue. 

.github/workflows/build.yaml

@@ -1,12 +1,12 @@
 name: CI
 # This workflow is triggered on pushes & pull requests
-on: [push, pull_request]
+on: [pull_request]
 
 jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container: techknowlogick/xgo:go-1.13.x
+    container: techknowlogick/xgo:go-1.17.x
 
     # Service containers to run with `build` (Required for end-to-end testing)
     services:

.github/workflows/docker-build.yaml

@@ -20,16 +20,20 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -38,8 +42,10 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v4
         with:
+          flavor: |
+            latest=false
           tags: |
             type=ref,enable=true,event=branch,suffix=-collector
             type=ref,enable=true,event=tag,suffix=-collector
@@ -48,14 +54,16 @@ jobs:
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v3
         with:
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
           context: .
           file: docker/Dockerfile.collector
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+#          cache-from: type=gha
+#          cache-to: type=gha,mode=max
 
   web:
     runs-on: ubuntu-latest
@@ -66,15 +74,29 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: "Populate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
+      - name: "Generate frontend"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: node:lts
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            cd /work
+            make frontend
+            ls -alt /work
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -83,25 +105,27 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v4
         with:
+          flavor: |
+            latest=false
           tags: |
             type=ref,enable=true,event=branch,suffix=-web
             type=ref,enable=true,event=tag,suffix=-web
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v3
         with:
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
           context: .
           file: docker/Dockerfile.web
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-
+#          cache-from: type=gha
+#          cache-to: type=gha,mode=max
   omnibus:
     runs-on: ubuntu-latest
     permissions:
@@ -111,15 +135,29 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: "Populate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
+      - name: "Generate frontend & version information"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: node:lts
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            cd /work
+            make frontend
+            ls -alt /work
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -128,17 +166,16 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v4
         with:
           tags: |
             type=ref,enable=true,event=branch,suffix=-omnibus
             type=ref,enable=true,event=tag,suffix=-omnibus
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v3
         with:
           platforms: linux/amd64,linux/arm64
           context: .
@@ -146,3 +183,5 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+#          cache-from: type=gha
+#          cache-to: type=gha,mode=max

.github/workflows/release-freebsd.yaml

@@ -59,6 +59,16 @@ jobs:
             file "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}" || true
             ldd "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}" || true
 
+      - name: Release Asset - Collector - freebsd-amd64
+        id: upload-release-asset2
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: './dist/scrutiny-collector-metrics-freebsd-amd64'
+          asset_name: scrutiny-collector-metrics-freebsd-amd64
+          asset_content_type: application/octet-stream
 
       - name: Release Asset - Web - freebsd-amd64
         id: upload-release-asset1
@@ -71,13 +81,3 @@ jobs:
           asset_name: scrutiny-web-freebsd-amd64
           asset_content_type: application/octet-stream
 
-      - name: Release Asset - Collector - freebsd-amd64
-        id: upload-release-asset2
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: './dist/scrutiny-collector-metrics-freebsd-amd64'
-          asset_name: scrutiny-collector-metrics-freebsd-amd64
-          asset_content_type: application/octet-stream

.github/workflows/release-frontend.yaml

@@ -15,13 +15,15 @@ jobs:
         uses: actions/checkout@v2
         with:
           ref: ${{github.event.release.tag_name}}
+      - name: "Generate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
       - name: Build Frontend
         run: |
           cd webapp/frontend
           npm install -g @angular/cli@9.1.4
           npm install
           mkdir -p dist
-          ng build --output-path=dist --deploy-url="/web/" --base-href="/web/" --prod
+          npm run build:prod -- --output-path=dist
           tar -czf scrutiny-web-frontend.tar.gz dist
       - name: Upload Frontend Asset
         id: upload-release-asset3

.github/workflows/release.yaml

@@ -16,7 +16,7 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container: techknowlogick/xgo:go-1.13.x
+    container: techknowlogick/xgo:go-1.17.x
     # Service containers to run with `build` (Required for end-to-end testing)
     services:
       influxdb:

.github/workflows/sponsors.yaml

@@ -8,6 +8,7 @@ jobs:
   build:
     name: is-sponsor-label
     runs-on: ubuntu-latest
+    if: ${{ false }}
     steps:
       - uses: JasonEtco/is-sponsor-label-action@v1.2.0
         env:

.gitignore

@@ -64,3 +64,5 @@ scrutiny-*.db
 scrutiny_test.db
 scrutiny.yaml
 coverage.txt
+/config
+/influxdb

CONTRIBUTING.md

@@ -1,75 +1,109 @@
 # Contributing
 
-There are multiple ways to develop on the scrutiny codebase locally. The two most popular are:
-- Docker Development Container - only requires docker
-- Run Components Locally - requires smartmontools, golang & nodejs installed locally
+The Scrutiny repository is a [monorepo](https://en.wikipedia.org/wiki/Monorepo) containing source code for:
+- Scrutiny Backend Server (API)
+- Scrutiny Frontend Angular SPA
+- S.M.A.R.T Collector
 
-## Docker Development
-```
-docker build -f docker/Dockerfile . -t chcr.io/analogj/scrutiny:master-omnibus
-docker run -it --rm -p 8080:8080 \
--v /run/udev:/run/udev:ro \
---cap-add SYS_RAWIO \
---device=/dev/sda \
---device=/dev/sdb \
-ghcr.io/analogj/scrutiny:master-omnibus
-/opt/scrutiny/bin/scrutiny-collector-metrics run
-```
+Depending on the functionality you are adding, you may need to setup a development environment for 1 or more projects. 
 
+# Modifying the Scrutiny Backend Server (API)
 
-## Local Development
+1. install the [Go runtime](https://go.dev/doc/install) (v1.17+)
+2. download the `scrutiny-web-frontend.tar.gz` for the [latest release](https://github.com/AnalogJ/scrutiny/releases/latest). Extract to a folder named `dist`
+3. create a `scrutiny.yaml` config file
+    ```yaml
+    # config file for local development. store as scrutiny.yaml
+    version: 1
+    
+    web:
+      listen:
+        port: 8080
+        host: 0.0.0.0
+      database:
+        # can also set absolute path here
+        location: ./scrutiny.db
+      src:
+        frontend:
+          path: ./dist
+      influxdb:
+        retention_policy: false
+    
+    log:
+      file: 'web.log' #absolute or relative paths allowed, eg. web.log
+      level: DEBUG
 
-### Frontend
-The frontend is written in Angular.
-If you're working on the frontend and can use mocked data rather than a real backend, you can use
-```
-cd webapp/frontend
-npm install
-ng serve --deploy-url="/web/" --base-href="/web/"
-```
+    ```
+4. start a InfluxDB docker container. 
+    ```bash
+    docker run -p 8086:8086 --rm influxdb:2.2
+    ```
+5. start the scrutiny web server
+    ```bash
+    go mod vendor
+    go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
+    ```
+6. open your browser to [http://localhost:8080/web](http://localhost:8080/web)
 
-However, if you need to also run the backend, and use real data, you'll need to run the following command:
-```
-cd webapp/frontend && ng build --watch --output-path=../../dist --prod
-```
+# Modifying the Scrutiny Frontend Angular SPA
 
-> Note: if you do not add `--prod` flag, app will display mocked data for api calls.
+The frontend is written in Angular. If you're working on the frontend and can use mocked data rather than a real backend, you can follow the instructions below:
 
-### Backend
+1. install [NodeJS](https://nodejs.org/en/download/)
+2. start the Angular Frontend Application
+    ```bash
+    cd webapp/frontend
+    npm install
+    npm run start -- --deploy-url="/web/" --base-href="/web/" --port 4200
+    ```
+3. open your browser and visit [http://localhost:4200/web](http://localhost:4200/web)
 
-If you're using the `ng build` command above to generate your frontend, you'll need to create a custom config file and
-override the `web.src.frontend.path` value.
+# Modifying both Scrutiny Backend and Frontend Applications
+If you're developing a feature that requires changes to the backend and the frontend, or a frontend feature that requires real data, 
+you'll need to follow the steps below:
 
-```
-# config file for local development. store as scrutiny.yaml
-version: 1
+1. install the [Go runtime](https://go.dev/doc/install) (v1.17+)
+2. install [NodeJS](https://nodejs.org/en/download/)
+3. create a `scrutiny.yaml` config file
+    ```yaml
+    # config file for local development. store as scrutiny.yaml
+    version: 1
+    
+    web:
+      listen:
+        port: 8080
+        host: 0.0.0.0
+      database:
+        # can also set absolute path here
+        location: ./scrutiny.db
+      src:
+        frontend:
+          path: ./dist
+      influxdb:
+        retention_policy: false
+    
+    log:
+      file: 'web.log' #absolute or relative paths allowed, eg. web.log
+      level: DEBUG
 
-web:
-  listen:
-    port: 8080
-    host: 0.0.0.0
-  database:
-    # can also set absolute path here
-    location: ./scrutiny.db
-  src:
-    frontend:
-      path: ./dist
-  influxdb:
-    retention_policy: false
-
-log:
-  file: 'web.log' #absolute or relative paths allowed, eg. web.log
-  level: DEBUG
-
-```
-
-Once you've created a config file, you can pass it to the scrutiny binary during startup.
-
-```
-go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
-```
-
-Now visit http://localhost:8080
+    ```
+4. start a InfluxDB docker container.
+    ```bash
+    docker run -p 8086:8086 --rm influxdb:2.2
+    ```
+5. build the Angular Frontend Application
+    ```bash
+    cd webapp/frontend
+    npm install
+    npm run build:prod -- --watch --output-path=../../dist
+    # Note: if you do not add `--prod` flag, app will display mocked data for api calls.
+    ```
+6. start the scrutiny web server
+    ```bash
+    go mod vendor
+    go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
+    ```
+7. open your browser to [http://localhost:8080/web](http://localhost:8080/web)
 
 
 If you'd like to populate the database with some test data,  you can run the following commands:
@@ -82,15 +116,6 @@ If you'd like to populate the database with some test data,  you can run the fol
 docker run -p 8086:8086 --rm influxdb:2.2
 
 
-docker run --rm -p 8086:8086 \
-      -e DOCKER_INFLUXDB_INIT_MODE=setup \
-      -e DOCKER_INFLUXDB_INIT_USERNAME=admin \
-      -e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
-      -e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
-      -e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
-      influxdb:2.2
-
-
 # curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/web/testdata/register-devices-req.json localhost:8080/api/devices/register
 # curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata.json localhost:8080/api/device/0x5000cca264eb01d7/smart
 # curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata-date.json localhost:8080/api/device/0x5000cca264eb01d7/smart
@@ -105,14 +130,14 @@ curl localhost:8080/api/summary
 
 ```
 
-### Collector
+# Modifying the Collector
 ```
 brew install smartmontools
 go run collector/cmd/collector-metrics/collector-metrics.go run --debug
 ```
 
 
-## Debugging
+# Debugging
 
 If you need more verbose logs for debugging, you can use the following environmental variables:
 
@@ -131,3 +156,32 @@ Finally, you can copy the files from the scrutiny container to your host using t
 docker cp scrutiny:/tmp/collector.log collector.log
 docker cp scrutiny:/tmp/web.log web.log
 ```
+
+# Docker Development
+
+```
+docker build -f docker/Dockerfile . -t chcr.io/analogj/scrutiny:master-omnibus
+docker run -it --rm -p 8080:8080 \
+-v /run/udev:/run/udev:ro \
+--cap-add SYS_RAWIO \
+--device=/dev/sda \
+--device=/dev/sdb \
+ghcr.io/analogj/scrutiny:master-omnibus
+/opt/scrutiny/bin/scrutiny-collector-metrics run
+```
+
+
+# Running Tests
+
+```bash
+docker run -p 8086:8086 -d --rm \
+-e DOCKER_INFLUXDB_INIT_MODE=setup \
+-e DOCKER_INFLUXDB_INIT_USERNAME=admin \
+-e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
+-e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
+-e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
+-e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \
+influxdb:2.2
+go test ./...
+
+```

Makefile

@@ -9,6 +9,7 @@ BINARY=\
   linux/arm-7   \
   linux/arm64   \
 
+.ONESHELL: # Applies to every targets in the file! .ONESHELL instructs make to invoke a single instance of the shell and provide it with the entire recipe, regardless of how many lines it contains.
 .PHONY: all $(BINARY)
 all: $(BINARY) windows/amd64
 
@@ -38,5 +39,28 @@ windows/amd64:
 	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
 	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
 
+
+docker-collector:
+	@echo "building collector docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.collector -t analogj/scrutiny-dev:collector .
+
+docker-web:
+	@echo "building web docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.web -t analogj/scrutiny-dev:web .
+
+docker-omnibus:
+	@echo "building omnibus docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile -t analogj/scrutiny-dev:omnibus .
+
+# reduce logging, disable angular-cli analytics for ci environment
+frontend: export NPM_CONFIG_LOGLEVEL = warn
+frontend: export NG_CLI_ANALYTICS = false
+frontend:
+	cd webapp/frontend
+	npm install -g @angular/cli@9.1.4
+	mkdir -p $(CURDIR)/dist
+	npm install
+	npm run build:prod -- --output-path=$(CURDIR)/dist
+
 # clean:
 # 	rm scrutiny-collector-metrics-* scrutiny-web-*

README.md

@@ -17,8 +17,6 @@
 WebUI for smartd S.M.A.R.T monitoring
 
 > NOTE: Scrutiny is a Work-in-Progress and still has some rough edges.
->
-> WARNING: Once the [InfluxDB](https://github.com/AnalogJ/scrutiny/tree/influxdb) branch is merged, Scrutiny will use both sqlite and InfluxDB for data storage. Unfortunately, this may not be backwards compatible with the database structures in the master (sqlite only) branch. 
 
 [![](docs/dashboard.png)](https://imgur.com/a/5k8qMzS)
 
@@ -60,18 +58,21 @@ Scrutiny uses `smartctl --scan` to detect devices/drives.
 - All RAID controllers supported by `smartctl` are automatically supported by Scrutiny.
     - While some RAID controllers support passing through the underlying SMART data to `smartctl` others do not.
     - In some cases `--scan` does not correctly detect the device type, returning [incomplete SMART data](https://github.com/AnalogJ/scrutiny/issues/45).
-    Scrutiny will eventually support overriding detected device type via the config file.
+    Scrutiny supports overriding detected device type via the config file: see [example.collector.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.collector.yaml)
 - If you use docker, you **must** pass though the RAID virtual disk to the container using `--device` (see below)
     - This device may be in `/dev/*` or `/dev/bus/*`.
     - If you're unsure, run `smartctl --scan` on your host, and pass all listed devices to the container.
 
+See [docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](./docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md) for help
 
 ## Docker
 
 If you're using Docker, getting started is as simple as running the following command:
 
+> See [docker/example.omnibus.docker-compose.yml](./docker/example.omnibus.docker-compose.yml) for a docker-compose file. 
+
 ```bash
-docker run -it --rm -p 8080:8080 \
+docker run -it --rm -p 8080:8080 -p 8086:8086 \
   -v `pwd`/scrutiny:/opt/scrutiny/config \
   -v `pwd`/influxdb2:/opt/scrutiny/influxdb \
   -v /run/udev:/run/udev:ro \
@@ -95,6 +96,8 @@ In addition to the Omnibus image (available under the `latest` tag) there are 2
 - `ghcr.io/analogj/scrutiny:master-collector` - Contains the Scrutiny data collector, `smartctl` binary and cron-like scheduler. You can run one collector on each server.
 - `ghcr.io/analogj/scrutiny:master-web` - Contains the Web UI, API and Database. Only one container necessary
 
+> See [docker/example.hubspoke.docker-compose.yml](./docker/example.hubspoke.docker-compose.yml) for a docker-compose file.
+
 ```bash
 docker run --rm -p 8086:8086 \
   -v `pwd`/influxdb2:/var/lib/influxdb2 \
@@ -174,7 +177,9 @@ Scrutiny supports sending SMART device failure notifications via the following s
 - Telegram
 - Tulip
 
-Check the `notify.urls` section of [example.scrutiny.yml](example.scrutiny.yaml) for more information and documentation for service specific setup.
+Check the `notify.urls` section of [example.scrutiny.yml](example.scrutiny.yaml) for examples.
+
+For more information and troubleshooting, see the [TROUBLESHOOTING_NOTIFICATIONS.md](./docs/TROUBLESHOOTING_NOTIFICATIONS.md) file
 
 ### Testing Notifications
 
@@ -233,9 +238,9 @@ scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
 | amd64 | :white_check_mark: | :white_check_mark: |
 | arm-5 | :white_check_mark: |  |
 | arm-6 | :white_check_mark: |  |
-| arm-7 | :white_check_mark: |  |
+| arm-7 | :white_check_mark: | web/collector only. see [#236](https://github.com/AnalogJ/scrutiny/issues/236)  |
 | arm64 | :white_check_mark: | :white_check_mark: |
-| freebsd | :white_check_mark: |  | 
+| freebsd | collector only. see [#238](https://github.com/AnalogJ/scrutiny/issues/238) |  | 
 | macos-amd64 |  | :white_check_mark: |
 | macos-arm64 |  | :white_check_mark: |
 | windows-amd64 | :white_check_mark: |  |

collector/pkg/collector/metrics.go

@@ -116,12 +116,13 @@ func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceT
 	}
 	mc.logger.Infof("Collecting smartctl results for %s\n", deviceName)
 
-	args := []string{"-x", "-j"}
+	fullDeviceName := fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName)
+	args := strings.Split(mc.config.GetCommandMetricsSmartArgs(fullDeviceName), " ")
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
 	if len(deviceType) > 0 && deviceType != "scsi" && deviceType != "ata" {
-		args = append(args, "-d", deviceType)
+		args = append(args, "--device", deviceType)
 	}
-	args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName))
+	args = append(args, fullDeviceName)
 
 	result, err := mc.shell.Command(mc.logger, "smartctl", args, "", os.Environ())
 	resultBytes := []byte(result)

collector/pkg/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"fmt"
 	"github.com/analogj/go-util/utils"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/collector/pkg/models"
@@ -8,6 +9,8 @@ import (
 	"github.com/spf13/viper"
 	"log"
 	"os"
+	"sort"
+	"strings"
 )
 
 // When initializing this class the following methods must be called:
@@ -16,6 +19,8 @@ import (
 // This is done automatically when created via the Factory.
 type configuration struct {
 	*viper.Viper
+
+	deviceOverrides []models.ScanOverride
 }
 
 //Viper uses the following precedence order. Each item takes precedence over the item below it:
@@ -38,6 +43,10 @@ func (c *configuration) Init() error {
 
 	c.SetDefault("api.endpoint", "http://localhost:8080")
 
+	c.SetDefault("commands.metrics_scan_args", "--scan --json")
+	c.SetDefault("commands.metrics_info_args", "--info --json")
+	c.SetDefault("commands.metrics_smart_args", "--xall --json")
+
 	//c.SetDefault("collect.short.command", "-a -o on -S on")
 
 	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
@@ -90,16 +99,89 @@ func (c *configuration) ValidateConfig() error {
 	// check that device prefix matches OS
 	// check that schema of config file is valid
 
-	return nil
+	// check that the collector commands are valid
+	commandArgStrings := map[string]string{
+		"commands.metrics_scan_args":  c.GetString("commands.metrics_scan_args"),
+		"commands.metrics_info_args":  c.GetString("commands.metrics_info_args"),
+		"commands.metrics_smart_args": c.GetString("commands.metrics_smart_args"),
+	}
+
+	errorStrings := []string{}
+	for configKey, commandArgString := range commandArgStrings {
+		args := strings.Split(commandArgString, " ")
+		//ensure that the args string contains `--json` or `-j` flag
+		containsJsonFlag := false
+		containsDeviceFlag := false
+		for _, flag := range args {
+			if strings.HasPrefix(flag, "--json") || strings.HasPrefix(flag, "-j") {
+				containsJsonFlag = true
+			}
+			if strings.HasPrefix(flag, "--device") || strings.HasPrefix(flag, "-d") {
+				containsDeviceFlag = true
+			}
+		}
+
+		if !containsJsonFlag {
+			errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' is missing '--json' flag", configKey))
+		}
+
+		if containsDeviceFlag {
+			errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' must not contain '--device' or '-d' flag", configKey))
+		}
+	}
+	//sort(errorStrings)
+	sort.Strings(errorStrings)
+
+	if len(errorStrings) == 0 {
+		return nil
+	} else {
+		return errors.ConfigValidationError(strings.Join(errorStrings, ", "))
+	}
 }
 
-func (c *configuration) GetScanOverrides() []models.ScanOverride {
+func (c *configuration) GetDeviceOverrides() []models.ScanOverride {
 	// we have to support 2 types of device types.
 	// - simple device type (device_type: 'sat')
 	// and list of device types (type: \n- 3ware,0 \n- 3ware,1 \n- 3ware,2)
 	// GetString will return "" if this is a list of device types.
 
-	overrides := []models.ScanOverride{}
-	c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
-	return overrides
+	if c.deviceOverrides == nil {
+		overrides := []models.ScanOverride{}
+		c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
+		c.deviceOverrides = overrides
+	}
+
+	return c.deviceOverrides
+}
+
+func (c *configuration) GetCommandMetricsInfoArgs(deviceName string) string {
+	overrides := c.GetDeviceOverrides()
+
+	for _, deviceOverrides := range overrides {
+		if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) {
+			//found matching device
+			if len(deviceOverrides.Commands.MetricsInfoArgs) > 0 {
+				return deviceOverrides.Commands.MetricsInfoArgs
+			} else {
+				return c.GetString("commands.metrics_info_args")
+			}
+		}
+	}
+	return c.GetString("commands.metrics_info_args")
+}
+
+func (c *configuration) GetCommandMetricsSmartArgs(deviceName string) string {
+	overrides := c.GetDeviceOverrides()
+
+	for _, deviceOverrides := range overrides {
+		if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) {
+			//found matching device
+			if len(deviceOverrides.Commands.MetricsSmartArgs) > 0 {
+				return deviceOverrides.Commands.MetricsSmartArgs
+			} else {
+				return c.GetString("commands.metrics_smart_args")
+			}
+		}
+	}
+	return c.GetString("commands.metrics_smart_args")
 }

collector/pkg/config/config_test.go

@@ -30,7 +30,7 @@ func TestConfiguration_GetScanOverrides_Simple(t *testing.T) {
 	//test
 	err := testConfig.ReadConfig(path.Join("testdata", "simple_device.yaml"))
 	require.NoError(t, err, "should correctly load simple device config")
-	scanOverrides := testConfig.GetScanOverrides()
+	scanOverrides := testConfig.GetDeviceOverrides()
 
 	//assert
 	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat"}, Ignore: false}}, scanOverrides)
@@ -45,7 +45,7 @@ func TestConfiguration_GetScanOverrides_Ignore(t *testing.T) {
 	//test
 	err := testConfig.ReadConfig(path.Join("testdata", "ignore_device.yaml"))
 	require.NoError(t, err, "should correctly load ignore device config")
-	scanOverrides := testConfig.GetScanOverrides()
+	scanOverrides := testConfig.GetDeviceOverrides()
 
 	//assert
 	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}, scanOverrides)
@@ -60,7 +60,7 @@ func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
 	//test
 	err := testConfig.ReadConfig(path.Join("testdata", "raid_device.yaml"))
 	require.NoError(t, err, "should correctly load ignore device config")
-	scanOverrides := testConfig.GetScanOverrides()
+	scanOverrides := testConfig.GetDeviceOverrides()
 
 	//assert
 	require.Equal(t, []models.ScanOverride{
@@ -75,3 +75,53 @@ func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
 			Ignore:     false,
 		}}, scanOverrides)
 }
+
+func TestConfiguration_InvalidCommands_MissingJson(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_missing_json.yaml"))
+	require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_scan_args' is missing '--json' flag"`, "should throw an error because json flag is missing")
+}
+
+func TestConfiguration_InvalidCommands_IncludesDevice(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_includes_device.yaml"))
+	require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_info_args' must not contain '--device' or '-d' flag, configuration key 'commands.metrics_smart_args' must not contain '--device' or '-d' flag"`, "should throw an error because device flags detected")
+}
+
+func TestConfiguration_OverrideCommands(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "override_commands.yaml"))
+	require.NoError(t, err, "should not throw an error")
+	require.Equal(t, "--xall --json -T permissive", testConfig.GetString("commands.metrics_smart_args"))
+}
+
+func TestConfiguration_OverrideDeviceCommands_MetricsInfoArgs(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "override_device_commands.yaml"))
+	require.NoError(t, err, "should correctly override device command")
+
+	//assert
+	require.Equal(t, "--info --json -T permissive", testConfig.GetCommandMetricsInfoArgs("/dev/sda"))
+	require.Equal(t, "--info --json", testConfig.GetCommandMetricsInfoArgs("/dev/sdb"))
+	//require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Commands: {MetricsInfoArgs: "--info --json -T "}}}, scanOverrides)
+}

collector/pkg/config/interface.go

@@ -22,5 +22,7 @@ type Interface interface {
 	GetStringSlice(key string) []string
 	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
 
-	GetScanOverrides() []models.ScanOverride
+	GetDeviceOverrides() []models.ScanOverride
+	GetCommandMetricsInfoArgs(deviceName string) string
+	GetCommandMetricsSmartArgs(deviceName string) string
 }

collector/pkg/config/mock/mock_config.go

@@ -5,88 +5,37 @@
 package mock_config
 
 import (
+	reflect "reflect"
+
 	models "github.com/analogj/scrutiny/collector/pkg/models"
 	gomock "github.com/golang/mock/gomock"
 	viper "github.com/spf13/viper"
-	reflect "reflect"
 )
 
-// MockInterface is a mock of Interface interface
+// MockInterface is a mock of Interface interface.
 type MockInterface struct {
 	ctrl     *gomock.Controller
 	recorder *MockInterfaceMockRecorder
 }
 
-// MockInterfaceMockRecorder is the mock recorder for MockInterface
+// MockInterfaceMockRecorder is the mock recorder for MockInterface.
 type MockInterfaceMockRecorder struct {
 	mock *MockInterface
 }
 
-// NewMockInterface creates a new mock instance
+// NewMockInterface creates a new mock instance.
 func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
 	mock := &MockInterface{ctrl: ctrl}
 	mock.recorder = &MockInterfaceMockRecorder{mock}
 	return mock
 }
 
-// EXPECT returns an object that allows the caller to indicate expected use
+// EXPECT returns an object that allows the caller to indicate expected use.
 func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
-// Init mocks base method
-func (m *MockInterface) Init() error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Init")
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Init indicates an expected call of Init
-func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
-}
-
-// ReadConfig mocks base method
-func (m *MockInterface) ReadConfig(configFilePath string) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// ReadConfig indicates an expected call of ReadConfig
-func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
-}
-
-// Set mocks base method
-func (m *MockInterface) Set(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Set", key, value)
-}
-
-// Set indicates an expected call of Set
-func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
-}
-
-// SetDefault mocks base method
-func (m *MockInterface) SetDefault(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "SetDefault", key, value)
-}
-
-// SetDefault indicates an expected call of SetDefault
-func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
-}
-
-// AllSettings mocks base method
+// AllSettings mocks base method.
 func (m *MockInterface) AllSettings() map[string]interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "AllSettings")
@@ -94,27 +43,13 @@ func (m *MockInterface) AllSettings() map[string]interface{} {
 	return ret0
 }
 
-// AllSettings indicates an expected call of AllSettings
+// AllSettings indicates an expected call of AllSettings.
 func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings))
 }
 
-// IsSet mocks base method
-func (m *MockInterface) IsSet(key string) bool {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "IsSet", key)
-	ret0, _ := ret[0].(bool)
-	return ret0
-}
-
-// IsSet indicates an expected call of IsSet
-func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
-}
-
-// Get mocks base method
+// Get mocks base method.
 func (m *MockInterface) Get(key string) interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Get", key)
@@ -122,13 +57,13 @@ func (m *MockInterface) Get(key string) interface{} {
 	return ret0
 }
 
-// Get indicates an expected call of Get
+// Get indicates an expected call of Get.
 func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key)
 }
 
-// GetBool mocks base method
+// GetBool mocks base method.
 func (m *MockInterface) GetBool(key string) bool {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetBool", key)
@@ -136,13 +71,55 @@ func (m *MockInterface) GetBool(key string) bool {
 	return ret0
 }
 
-// GetBool indicates an expected call of GetBool
+// GetBool indicates an expected call of GetBool.
 func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key)
 }
 
-// GetInt mocks base method
+// GetCommandMetricsInfoArgs mocks base method.
+func (m *MockInterface) GetCommandMetricsInfoArgs(deviceName string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCommandMetricsInfoArgs", deviceName)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetCommandMetricsInfoArgs indicates an expected call of GetCommandMetricsInfoArgs.
+func (mr *MockInterfaceMockRecorder) GetCommandMetricsInfoArgs(deviceName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsInfoArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsInfoArgs), deviceName)
+}
+
+// GetCommandMetricsSmartArgs mocks base method.
+func (m *MockInterface) GetCommandMetricsSmartArgs(deviceName string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCommandMetricsSmartArgs", deviceName)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetCommandMetricsSmartArgs indicates an expected call of GetCommandMetricsSmartArgs.
+func (mr *MockInterfaceMockRecorder) GetCommandMetricsSmartArgs(deviceName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsSmartArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsSmartArgs), deviceName)
+}
+
+// GetDeviceOverrides mocks base method.
+func (m *MockInterface) GetDeviceOverrides() []models.ScanOverride {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDeviceOverrides")
+	ret0, _ := ret[0].([]models.ScanOverride)
+	return ret0
+}
+
+// GetDeviceOverrides indicates an expected call of GetDeviceOverrides.
+func (mr *MockInterfaceMockRecorder) GetDeviceOverrides() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceOverrides", reflect.TypeOf((*MockInterface)(nil).GetDeviceOverrides))
+}
+
+// GetInt mocks base method.
 func (m *MockInterface) GetInt(key string) int {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetInt", key)
@@ -150,13 +127,13 @@ func (m *MockInterface) GetInt(key string) int {
 	return ret0
 }
 
-// GetInt indicates an expected call of GetInt
+// GetInt indicates an expected call of GetInt.
 func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
 }
 
-// GetString mocks base method
+// GetString mocks base method.
 func (m *MockInterface) GetString(key string) string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetString", key)
@@ -164,13 +141,13 @@ func (m *MockInterface) GetString(key string) string {
 	return ret0
 }
 
-// GetString indicates an expected call of GetString
+// GetString indicates an expected call of GetString.
 func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key)
 }
 
-// GetStringSlice mocks base method
+// GetStringSlice mocks base method.
 func (m *MockInterface) GetStringSlice(key string) []string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetStringSlice", key)
@@ -178,13 +155,79 @@ func (m *MockInterface) GetStringSlice(key string) []string {
 	return ret0
 }
 
-// GetStringSlice indicates an expected call of GetStringSlice
+// GetStringSlice indicates an expected call of GetStringSlice.
 func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key)
 }
 
-// UnmarshalKey mocks base method
+// Init mocks base method.
+func (m *MockInterface) Init() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Init")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Init indicates an expected call of Init.
+func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
+}
+
+// IsSet mocks base method.
+func (m *MockInterface) IsSet(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsSet", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsSet indicates an expected call of IsSet.
+func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
+}
+
+// ReadConfig mocks base method.
+func (m *MockInterface) ReadConfig(configFilePath string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReadConfig indicates an expected call of ReadConfig.
+func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
+}
+
+// Set mocks base method.
+func (m *MockInterface) Set(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Set", key, value)
+}
+
+// Set indicates an expected call of Set.
+func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
+}
+
+// SetDefault mocks base method.
+func (m *MockInterface) SetDefault(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetDefault", key, value)
+}
+
+// SetDefault indicates an expected call of SetDefault.
+func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
+}
+
+// UnmarshalKey mocks base method.
 func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{key, rawVal}
@@ -196,23 +239,9 @@ func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts
 	return ret0
 }
 
-// UnmarshalKey indicates an expected call of UnmarshalKey
+// UnmarshalKey indicates an expected call of UnmarshalKey.
 func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{key, rawVal}, decoderOpts...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...)
 }
-
-// GetScanOverrides mocks base method
-func (m *MockInterface) GetScanOverrides() []models.ScanOverride {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetScanOverrides")
-	ret0, _ := ret[0].([]models.ScanOverride)
-	return ret0
-}
-
-// GetScanOverrides indicates an expected call of GetScanOverrides
-func (mr *MockInterfaceMockRecorder) GetScanOverrides() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScanOverrides", reflect.TypeOf((*MockInterface)(nil).GetScanOverrides))
-}

collector/pkg/config/testdata/invalid_commands_includes_device.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan --json' # used to detect devices
+  metrics_info_args: '--info --json --device=sat' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json -d sat' # used to retrieve smart data for each device.

collector/pkg/config/testdata/invalid_commands_missing_json.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan' # used to detect devices
+  metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json' # used to retrieve smart data for each device.

collector/pkg/config/testdata/override_commands.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan --json' # used to detect devices
+  metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device.

collector/pkg/config/testdata/override_device_commands.yaml

@@ -0,0 +1,5 @@
+version: 1
+devices:
+  - device: /dev/sda
+    commands:
+      metrics_info_args: "--info --json -T permissive"

collector/pkg/detect/detect.go

@@ -28,7 +28,8 @@ type Detect struct {
 // models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
 func (d *Detect) SmartctlScan() ([]models.Device, error) {
 	//we use smartctl to detect all the drives available.
-	detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
+	args := strings.Split(d.Config.GetString("commands.metrics_scan_args"), " ")
+	detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ())
 	if err != nil {
 		d.Logger.Errorf("Error scanning for devices: %v", err)
 		return nil, err
@@ -51,13 +52,13 @@ func (d *Detect) SmartctlScan() ([]models.Device, error) {
 // - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
 // - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
 func (d *Detect) SmartCtlInfo(device *models.Device) error {
-
-	args := []string{"--info", "-j"}
+	fullDeviceName := fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName)
+	args := strings.Split(d.Config.GetCommandMetricsInfoArgs(fullDeviceName), " ")
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
 	if len(device.DeviceType) > 0 && device.DeviceType != "scsi" && device.DeviceType != "ata" {
-		args = append(args, "-d", device.DeviceType)
+		args = append(args, "--device", device.DeviceType)
 	}
-	args = append(args, fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName))
+	args = append(args, fullDeviceName)
 
 	availableDeviceInfoJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ())
 	if err != nil {
@@ -138,7 +139,7 @@ func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []mod
 
 	//now tha we've "grouped" all the devices, lets override any groups specified in the config file.
 
-	for _, overrideDevice := range d.Config.GetScanOverrides() {
+	for _, overrideDevice := range d.Config.GetDeviceOverrides() {
 		overrideDeviceFile := strings.ToLower(overrideDevice.Device)
 
 		if overrideDevice.Ignore {

collector/pkg/detect/detect_test.go

@@ -18,7 +18,8 @@ func TestDetect_SmartctlScan(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
 	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_simple.json")
@@ -45,7 +46,8 @@ func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
 	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_megaraid.json")
@@ -75,7 +77,8 @@ func TestDetect_SmartctlScan_Nvme(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
 
 	fakeShell := mock_shell.NewMockInterface(mockCtrl)
 	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_nvme.json")
@@ -104,7 +107,9 @@ func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -134,7 +139,9 @@ func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{
@@ -163,7 +170,8 @@ func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{
 		{
 			Device:     "/dev/bus/0",
 			DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
@@ -202,7 +210,8 @@ func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
 	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
-	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
 	detectedDevices := models.Scan{
 		Devices: []models.ScanDevice{
 			{

collector/pkg/detect/devices_darwin.go

@@ -13,7 +13,7 @@ func DevicePrefix() string {
 
 func (d *Detect) Start() ([]models.Device, error) {
 	d.Shell = shell.Create()
-	// call the base/common functionality to get a list of devicess
+	// call the base/common functionality to get a list of devices
 	detectedDevices, err := d.SmartctlScan()
 	if err != nil {
 		return nil, err

collector/pkg/detect/devices_linux.go

@@ -1,9 +1,12 @@
 package detect
 
 import (
+	"fmt"
 	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/jaypipes/ghw"
+	"io/ioutil"
+	"path/filepath"
 	"strings"
 )
 
@@ -22,6 +25,7 @@ func (d *Detect) Start() ([]models.Device, error) {
 	//inflate device info for detected devices.
 	for ndx, _ := range detectedDevices {
 		d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+		populateUdevInfo(&detectedDevices[ndx]) //ignore errors.
 	}
 
 	return detectedDevices, nil
@@ -49,3 +53,51 @@ func (d *Detect) wwnFallback(detectedDevice *models.Device) {
 	//wwn must always be lowercase.
 	detectedDevice.WWN = strings.ToLower(detectedDevice.WWN)
 }
+
+// as discussed in
+// - https://github.com/AnalogJ/scrutiny/issues/225
+// - https://github.com/jaypipes/ghw/issues/59#issue-361915216
+// udev exposes its data in a standardized way under /run/udev/data/....
+func populateUdevInfo(detectedDevice *models.Device) error {
+	// Get device major:minor numbers
+	// `cat /sys/class/block/sda/dev`
+	devNo, err := ioutil.ReadFile(filepath.Join("/sys/class/block/", detectedDevice.DeviceName, "dev"))
+	if err != nil {
+		return err
+	}
+
+	// Look up block device in udev runtime database
+	// `cat /run/udev/data/b8:0`
+	udevID := "b" + strings.TrimSpace(string(devNo))
+	udevBytes, err := ioutil.ReadFile(filepath.Join("/run/udev/data/", udevID))
+	if err != nil {
+		return err
+	}
+
+	deviceMountPaths := []string{}
+	udevInfo := make(map[string]string)
+	for _, udevLine := range strings.Split(string(udevBytes), "\n") {
+		if strings.HasPrefix(udevLine, "E:") {
+			if s := strings.SplitN(udevLine[2:], "=", 2); len(s) == 2 {
+				udevInfo[s[0]] = s[1]
+			}
+		} else if strings.HasPrefix(udevLine, "S:") {
+			deviceMountPaths = append(deviceMountPaths, udevLine[2:])
+		}
+	}
+
+	//Set additional device information.
+	if deviceLabel, exists := udevInfo["ID_FS_LABEL"]; exists {
+		detectedDevice.DeviceLabel = deviceLabel
+	}
+	if deviceUUID, exists := udevInfo["ID_FS_UUID"]; exists {
+		detectedDevice.DeviceUUID = deviceUUID
+	}
+	if deviceSerialID, exists := udevInfo["ID_SERIAL"]; exists {
+		detectedDevice.DeviceSerialID = fmt.Sprintf("%s-%s", udevInfo["ID_BUS"], deviceSerialID)
+	}
+
+
+	return nil
+}
+

collector/pkg/models/device.go

@@ -1,10 +1,13 @@
 package models
 
 type Device struct {
-	WWN    string `json:"wwn"`
-	HostId string `json:"host_id"`
+	WWN string `json:"wwn"`
 
 	DeviceName     string `json:"device_name"`
+	DeviceUUID	   string `json:"device_uuid"`
+	DeviceSerialID	   string `json:"device_serial_id"`
+	DeviceLabel	   string `json:"device_label"`
+
 	Manufacturer   string `json:"manufacturer"`
 	ModelName      string `json:"model_name"`
 	InterfaceType  string `json:"interface_type"`
@@ -17,6 +20,10 @@ type Device struct {
 	SmartSupport   bool   `json:"smart_support"`
 	DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
 	DeviceType     string `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string `json:"label"`
+	HostId string `json:"host_id"`
 }
 
 type DeviceWrapper struct {

collector/pkg/models/scan_override.go

@@ -4,4 +4,8 @@ type ScanOverride struct {
 	Device     string   `mapstructure:"device"`
 	DeviceType []string `mapstructure:"type"`
 	Ignore     bool     `mapstructure:"ignore"`
+	Commands   struct {
+		MetricsInfoArgs  string `mapstructure:"metrics_info_args"`
+		MetricsSmartArgs string `mapstructure:"metrics_smart_args"`
+	} `mapstructure:"commands"`
 }

docker/Dockerfile

@@ -1,5 +1,5 @@
 ########
-FROM golang:1.14.4-buster as backendbuild
+FROM golang:1.17.10-buster as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
@@ -11,39 +11,25 @@ RUN go mod vendor && \
     go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
 
 ########
-FROM node:lts-slim as frontendbuild
-
-#reduce logging, disable angular-cli analytics for ci environment
-ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
-
-WORKDIR /opt/scrutiny/src
-COPY webapp/frontend /opt/scrutiny/src
-
-RUN npm install -g @angular/cli@9.1.4 && \
-    mkdir -p /scrutiny/dist && \
-    npm install && \
-    ng build --output-path=/opt/scrutiny/dist --prod
-
-
-########
-FROM ubuntu:bionic as runtime
+FROM ubuntu:latest as runtime
 ARG TARGETARCH
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 ENV INFLUXD_CONFIG_PATH=/opt/scrutiny/influxdb
 
-RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates curl tzdata \
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates curl tzdata \
     && update-ca-certificates \
     &&  case ${TARGETARCH} in \
             "amd64")  S6_ARCH=amd64  ;; \
             "arm64")  S6_ARCH=aarch64  ;; \
         esac \
     && curl https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-${S6_ARCH}.tar.gz -L -s --output /tmp/s6-overlay-${S6_ARCH}.tar.gz \
-    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C /
-
-ADD https://dl.influxdata.com/influxdb/releases/influxdb2-2.2.0-${TARGETARCH}.deb /tmp/
-RUN dpkg -i /tmp/influxdb2-2.2.0-${TARGETARCH}.deb && rm -rf /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
+    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C / --exclude="./bin" \
+    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C /usr ./bin \
+    && rm -rf /tmp/s6-overlay-${S6_ARCH}.tar.gz \
+    && curl -L https://dl.influxdata.com/influxdb/releases/influxdb2-2.2.0-${TARGETARCH}.deb --output /tmp/influxdb2-2.2.0-${TARGETARCH}.deb \
+    && dpkg -i --force-all /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
 
 COPY /rootfs /
 
@@ -51,7 +37,7 @@ COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /opt/scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
-COPY --from=frontendbuild /opt/scrutiny/dist /opt/scrutiny/web
+COPY dist /opt/scrutiny/web
 RUN chmod +x /opt/scrutiny/bin/scrutiny && \
     chmod +x /opt/scrutiny/bin/scrutiny-collector-selftest && \
     chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \

docker/Dockerfile.collector

@@ -1,5 +1,5 @@
 ########
-FROM golang:1.14.4-buster as backendbuild
+FROM golang:1.17.10-buster as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
@@ -10,11 +10,11 @@ RUN go mod vendor && \
     go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
 
 ########
-FROM ubuntu:bionic as runtime
+FROM ubuntu:latest as runtime
 WORKDIR /scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates tzdata && update-ca-certificates
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && update-ca-certificates
 
 COPY /docker/entrypoint-collector.sh /entrypoint-collector.sh
 COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny

docker/Dockerfile.web

@@ -1,5 +1,5 @@
 ########
-FROM golang:1.14.4-buster as backendbuild
+FROM golang:1.17.10-buster as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
@@ -9,22 +9,7 @@ RUN go mod vendor && \
     go build -ldflags '-w -extldflags "-static"' -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go
 
 ########
-FROM node:lts-slim as frontendbuild
-
-#reduce logging, disable angular-cli analytics for ci environment
-ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
-
-WORKDIR /opt/scrutiny/src
-COPY webapp/frontend /opt/scrutiny/src
-
-RUN npm install -g @angular/cli@9.1.4 && \
-    mkdir -p /opt/scrutiny/dist && \
-    npm install && \
-    ng build --output-path=/opt/scrutiny/dist --prod
-
-
-########
-FROM ubuntu:bionic as runtime
+FROM ubuntu:latest as runtime
 EXPOSE 8080
 WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
@@ -32,7 +17,7 @@ ENV PATH="/opt/scrutiny/bin:${PATH}"
 RUN apt-get update && apt-get install -y ca-certificates curl tzdata && update-ca-certificates
 
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
-COPY --from=frontendbuild /opt/scrutiny/dist /opt/scrutiny/web
+COPY dist /opt/scrutiny/web
 RUN chmod +x /opt/scrutiny/bin/scrutiny && \
     mkdir -p /opt/scrutiny/web && \
     mkdir -p /opt/scrutiny/config && \

docker/Dockerfile.xgo

@@ -1,4 +1,4 @@
-FROM techknowlogick/xgo:go-1.13.x
+FROM techknowlogick/xgo:go-1.17.x
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 

docker/entrypoint-collector.sh

@@ -1,14 +1,19 @@
 #!/bin/bash
+
 # Cron runs in its own isolated environment (usually using only /etc/environment )
 # So when the container starts up, we will do a dump of the runtime environment into a .env file that we
-# will then source into the crontab file (/etc/cron.d/scrutiny.sh)
-
-printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
+# will then source into the crontab file (/etc/cron.d/scrutiny)
+(set -o posix; export -p) > /env.sh
 
 # adding ability to customize the cron schedule.
 COLLECTOR_CRON_SCHEDULE=${COLLECTOR_CRON_SCHEDULE:-"0 0 * * *"}
+
+# if the cron schedule has been overridden via env variable (eg docker-compose) we should make sure to strip quotes
+[[ "${COLLECTOR_CRON_SCHEDULE}" == \"*\" || "${COLLECTOR_CRON_SCHEDULE}" == \'*\' ]] && COLLECTOR_CRON_SCHEDULE="${COLLECTOR_CRON_SCHEDULE:1:-1}"
+
+# replace placeholder with correct value
 sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron.d/scrutiny
 
 # now that we have the env start cron in the foreground
 echo "starting cron"
-su -c "cron -l 8 -f" root
+su -c "cron -f -L 15" root

docker/example.hubspoke.docker-compose.yml

@@ -0,0 +1,48 @@
+version: '2.4'
+
+services:
+  influxdb:
+    image: influxdb:2.2
+    ports:
+      - '8086:8086'
+    volumes:
+      - './influxdb:/var/lib/influxdb2'
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8086/health"]
+      interval: 5s
+      timeout: 10s
+      retries: 20
+
+
+  web:
+    image: 'ghcr.io/analogj/scrutiny:master-web'
+    ports:
+      - '8080:8080'
+    volumes:
+      - './config:/opt/scrutiny/config'
+    environment:
+      SCRUTINY_WEB_INFLUXDB_HOST: 'influxdb'
+    depends_on:
+      influxdb:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/api/health"]
+      interval: 5s
+      timeout: 10s
+      retries: 20
+      start_period: 10s
+
+  collector:
+    image: 'ghcr.io/analogj/scrutiny:master-collector'
+    cap_add:
+      - SYS_RAWIO
+    volumes:
+      - '/run/udev:/run/udev:ro'
+    environment:
+      COLLECTOR_API_ENDPOINT: 'http://web:8080'
+    depends_on:
+      web:
+        condition: service_healthy
+    devices:
+      - "/dev/sda"
+      - "/dev/sdb"

docker/example.omnibus.docker-compose.yml

@@ -7,11 +7,12 @@ services:
     cap_add:
       - SYS_RAWIO
     ports:
-      - "8080:8080"
+      - "8080:8080" # webapp
+      - "8086:8086" # influxDB admin
     volumes:
       - /run/udev:/run/udev:ro
       - ./config:/opt/scrutiny/config
-      - ./influxdb:/var/lib/influxdb2
+      - ./influxdb:/opt/scrutiny/influxdb
     devices:
       - "/dev/sda"
       - "/dev/sdb"

docs/INSTALL_ANSIBLE.md

@@ -0,0 +1,17 @@
+# Ansible Install
+
+[Zorlin](https://github.com/Zorlin) has developed and now maintains [an Ansible playbook](https://github.com/Zorlin/scrutiny-playbook) which automates the steps involved in manually setting up Scrutiny.
+
+Using it is simple:
+
+* Grab a copy of the playbook
+* Follow the directions in the playbook repository
+* Run `ansible-playbook site.yml`
+* Visit http://your-machine:8080 to see your new Scrutiny installation.
+
+It will automatically pull metrics from machines once a day, at 1am.
+
+You can see it in action below.
+
+[![asciicast](https://asciinema.org/a/493531.svg)](https://asciinema.org/a/493531)
+

docs/INSTALL_HUB_SPOKE.md

@@ -0,0 +1 @@
+> See [docker/example.hubspoke.docker-compose.yml](https://github.com/AnalogJ/scrutiny/blob/master/docker/example.hubspoke.docker-compose.yml) for a docker-compose file.

docs/INSTALL_MANUAL.md

@@ -2,12 +2,18 @@
 
 While the easiest way to get started with [Scrutiny is using Docker](https://github.com/AnalogJ/scrutiny#docker),
 it is possible to run it manually without much work. You can even mix and match, using Docker for one component and
-a manual installation for the other.
+a manual installation for the other. There's also [an installer](INSTALL_ANSIBLE.md) which automates this manual installation procedure.
 
-Scrutiny is made up of two components: a collector and a webapp/api. Here's how each component can be deployed manually.
+Scrutiny is made up of three components: an influxdb Database, a collector and a webapp/api. Here's how each component can be deployed manually.
 
 > Note: the `/opt/scrutiny` directory is not hardcoded, you can use any directory name/path.
 
+## InfluxDB
+
+Please follow the official InfluxDB installation guide. Note, you'll need to install v2.2.0+. 
+
+https://docs.influxdata.com/influxdb/v2.2/install/
+
 ## Webapp/API
 
 ### Dependencies
@@ -45,6 +51,17 @@ web:
       # The path to the Scrutiny frontend files (js, css, images) must be specified.
       # We'll populate it with files in the next section
       path: /opt/scrutiny/web
+  
+  # if you're runnning influxdb on a different host (or using a cloud-provider) you'll need to update the host & port below. 
+  # token, org, bucket are unnecessary for a new InfluxDB installation, as Scrutiny will automatically run the InfluxDB setup, 
+  # and store the information in the config file. If you 're re-using an existing influxdb installation, you'll need to provide
+  # the `token`
+  influxdb:
+    host: localhost
+    port: 8086
+#    token: 'my-token'
+#    org: 'my-org'
+#    bucket: 'bucket'
 ```
 
 > Note: for a full list of available configuration options, please check the [example.scrutiny.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.scrutiny.yaml) file.
@@ -66,9 +83,11 @@ Now that we have downloaded the required files, let's prepare the filesystem.
 chmod +x /opt/scrutiny/bin/scrutiny-web-linux-amd64
 
 # Next, lets extract the frontend files.
+# NOTE: after extraction, there **should not** be a `dist` subdirectory in `/opt/scrutiny/web` directory.
 cd /opt/scrutiny/web
 tar xvzf scrutiny-web-frontend.tar.gz --strip-components 1 -C .
 
+
 # Cleanup
 rm -rf scrutiny-web-frontend.tar.gz
 ```
@@ -96,7 +115,8 @@ Unlike the webapp, the collector does have some dependencies:
 Unfortunately the version of `smartmontools` (which contains `smartctl`) available in some of the base OS repositories is ancient.
 So you'll need to install the v7+ version using one of the following commands:
 
-- **Ubuntu:** `apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1`
+- **Ubuntu (22.04/Jammy/LTS):** `apt-get install -y smartmontools`
+- **Ubuntu (18.04/Bionic):** `apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1`
 - **Centos8:**
     - `dnf install https://extras.getpagespeed.com/release-el8-latest.rpm`
     - `dnf install smartmontools`

docs/INSTALL_PFSENSE.md

@@ -0,0 +1,72 @@
+# pfsense Install
+
+This bascially follows the [Manual collector instructions](https://github.com/AnalogJ/scrutiny/blob/master/docs/INSTALL_MANUAL.md#collector) and assumes you are running a hub and spoke deployment and already have the web app setup.
+
+
+### Dependencies
+
+SSH into pfsense, hit `8` for the shell and install the required dependencies.
+
+```
+pkg install smartmontools
+```
+
+Ensure smartmontools is v7+. This won't be a problem in pfsense 2.6.0+
+
+
+### Directory Structure
+
+Now let's create a directory structure to contain the Scrutiny collector binary.
+
+```
+mkdir -p /opt/scrutiny/bin
+```
+
+
+### Download Files
+
+Next, we'll download the Scrutiny collector binary from the [latest Github release](https://github.com/analogj/scrutiny/releases).
+
+> NOTE: Ensure you have the latest version in the below command
+
+```
+fetch -o /opt/scrutiny/bin https://github.com/AnalogJ/scrutiny/releases/download/vX.X.X/scrutiny-collector-metrics-freebsd-amd64
+```
+
+
+### Prepare Scrutiny
+
+Now that we have downloaded the required files, let's prepare the filesystem.
+
+```
+chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64
+```
+
+
+### Start Scrutiny Collector, Populate Webapp
+
+Next, we will manually trigger the collector, to populate the Scrutiny dashboard:
+
+> NOTE: if you need to pass a config file to the scrutiny collector, you can provide it using the `--config` flag.
+
+```
+/opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64 run --api-endpoint "http://localhost:8080"
+```
+> NOTE: change the IP address to that of your web app
+
+### Schedule Collector with Cron
+
+Finally you need to schedule the collector to run periodically.
+
+Login to the pfsense webGUI and head to `Services/Cron` add an entry with the following details:
+
+```
+Minute: */15
+Hour: *
+Day of the Month: *
+Month of the Year: *
+Day of the Week: *
+User: root
+Command: /opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64 run --api-endpoint "http://localhost:8080" >/dev/null 2>&1
+```
+> NOTE: `>/dev/null 2>&1` is used to stop cron confirmation emails being sent.

docs/SUPPORTED_NAS_OS.md

@@ -4,11 +4,13 @@ These are the officially supported NAS OS's (with documentation and setup guides
 Once a guide is created (in `docs/guides/`) it will be linked here.
 
 - [ ] freenas/truenas
-- [x] [unraid](https://github.com/AnalogJ/scrutiny/blob/master/docs/INSTALL_UNRAID.md)
+- [x] [unraid](./INSTALL_UNRAID.md)
 - [ ] ESXI
 - [ ] Proxmox
 - [ ] Synology
 - [ ] OMV
 - [ ] Amahi
 - [ ] Running in a LXC container
+- [x] [PFSense](./INSTALL_UNRAID.md)
+- [ ] QNAP
 

docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md

@@ -52,6 +52,8 @@ If the output is the same, your devices will be processed by Scrutiny.
 In some cases `--scan` does not correctly detect the device type, returning [incomplete SMART data](https://github.com/AnalogJ/scrutiny/issues/45).
 Scrutiny will supports overriding the detected device type via the config file.
 
+[example.collector.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.collector.yaml)
+
 ### RAID Controllers (Megaraid/3ware/HBA/Adaptec/HPE/etc)
 Smartctl has support for a large number of [RAID controllers](https://www.smartmontools.org/wiki/Supported_RAID-Controllers), however this 
 support is not automatic, and may require some additional device type hinting. You can provide this information to the Scrutiny collector
@@ -111,12 +113,73 @@ instead of the block device (`/dev/nvme0n1`). See [#209](https://github.com/Anal
 
 ### ATA
 
-### Standby/Sleeping Disks
+### Exit Codes
+
+If you see an error message similar to `smartctl returned an error code (2) while processing /dev/sda`, this means that
+`smartctl` (not Scrutiny) exited with an error code. Scrutiny will attempt to print a helpful error message to help you debug,
+but you can look at the table (and associated links) below to debug `smartctl`. 
+
+> smartctl Return Values
+> The return values of smartctl are defined by a bitmask. If all is well with the disk, the return value (exit status) of 
+> smartctl is 0 (all bits turned off). If a problem occurs, or an error, potential error, or fault is detected, then 
+> a non-zero status is returned. In this case, the eight different bits in the return value have the following meanings 
+> for ATA disks; some of these values may also be returned for SCSI disks.
+> 
+> source: http://www.linuxguide.it/command_line/linux-manpage/do.php?file=smartctl#sect7
+
+
+| Exit Code (Isolated) | Binary | Problem Message |
+| --- | --- | --- |
+| 1 | Bit 0 | Command line did not parse. |
+| 2 | Bit 1 | Device open failed, or device did not return an IDENTIFY DEVICE structure. |
+| 4 | Bit 2 | Some SMART command to the disk failed, or there was a checksum error in a SMART data structure (see В´-bВ´ option above). |
+| 8 | Bit 3 | SMART status check returned “DISK FAILING". |
+| 16 | Bit 4 | We found prefail Attributes <= threshold. |
+| 32 | Bit 5 | SMART status check returned “DISK OK” but we found that some (usage or prefail) Attributes have been <= threshold at some time in the past. |
+| 64 | Bit 6 | The device error log contains records of errors. |
+| 128 | Bit 7 | The device self-test log contains records of errors. |
+
+#### Standby/Sleeping Disks
+
+Disks in Standby/Sleep can also cause `smartctl` to exit abnormally, usually with `exit code: 2`. 
+
 - https://github.com/AnalogJ/scrutiny/issues/221
 - https://github.com/AnalogJ/scrutiny/issues/157
 
 ### Volume Mount All Devices (`/dev`) - Privileged
 
+> WARNING: This is an insecure/dangerous workaround. Running Scrutiny (or any Docker image) with `--privileged` is equivalent to running it with root access. 
+
+If you have exhausted all other mechanisms to get your disks working with `smartctl` running within a container, you can try running the docker image with the following additional flags:
+
+- `--privileged` (instead of `--cap-add`) - this gives the docker container full access to your system. Scrutiny does not require this permission, however it can be helpful for `smartctl`
+- `-v /dev:/dev:ro` (instead of `--device`) - this mounts the `/dev` folder (containing all your device files) into the container, allowing `smartctl` to see your disks, exactly as if it were running on your host directly. 
+
+With this workaround your `docker run` command would look similar to the following:
+
+```bash
+docker run -it --rm -p 8080:8080 -p 8086:8086 \
+  -v `pwd`/scrutiny:/opt/scrutiny/config \
+  -v `pwd`/influxdb2:/opt/scrutiny/influxdb \
+  -v /run/udev:/run/udev:ro \
+  --privileged \
+  -v /dev:/dev \
+  --name scrutiny \
+  ghcr.io/analogj/scrutiny:master-omnibus
+```
+
+## Scrutiny detects Failure but SMART Passed?
+
+There's 2 different mechanisms that Scrutiny uses to detect failures.
+
+The first is simple SMART failures. If SMART thinks an attribute is in a failed state, Scrutiny will display it as failed as well.
+
+The second is using BackBlaze failure data: [https://backblaze.com/blog-smart-stats-2014-8.html](https://backblaze.com/blog-smart-stats-2014-8.html) 
+If Scrutiny detects that an attribute corresponds with a high rate of failure using BackBlaze's data, it will also mark that attribute (and disk) as failed (even though SMART may think the device is still healthy).
+
+This can cause some confusion when comparing Scrutiny's dashboard against other SMART analysis tools. 
+If you hover over the "failed" label beside an attribute, Scrutiny will tell you if the failure was due to SMART or Scrutiny/BackBlaze data. 
+
 
 ## Hub & Spoke model, with multiple Hosts.
 
@@ -125,3 +188,17 @@ Thankfully the collector has a special `--host-id` flag (or `COLLECTOR_HOST_ID`
 
 See the [docs/INSTALL_HUB_SPOKE.md](/docs/INSTALL_HUB_SPOKE.md) guide for more information. 
 
+## Collector DEBUG mode
+
+You can use environmental variables to enable debug logging and/or log files for the collector:
+
+```bash
+DEBUG=true
+COLLECTOR_LOG_FILE=/tmp/collector.log
+```
+
+Or if you're not using docker, you can pass CLI arguments to the collector during startup:
+
+```bash
+scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
+```

docs/TROUBLESHOOTING_INFLUXDB.md

@@ -0,0 +1,68 @@
+# InfluxDB Troubleshooting
+
+## Installation 
+InfluxDB is a required dependency for Scrutiny v0.4.0+. 
+
+https://docs.influxdata.com/influxdb/v2.2/install/
+
+## Persistence
+
+To ensure that all data is correctly stored, you must also persist the InfluxDB database directory
+
+- If you're using the Official Scrutiny Omnibus image (`ghcr.io/analogj/scrutiny:master-omnibus`), the path is `/opt/scrutiny/influxdb`
+- If you're deploying in Hub/Spoke mode with the InfluxDB maintained image (`influxdb:2.2`), the path is `/var/lib/influxdb2`
+
+If you attempt to restart Scrutiny but you forgot to persist the InfluxDB directory, you will get an error message like follows:
+
+```
+scrutiny    | time="2022-05-12T22:54:12Z" level=info msg="Trying to connect to scrutiny sqlite db: /opt/scrutiny/config/scrutiny.db\n"
+scrutiny    | time="2022-05-12T22:54:12Z" level=info msg="Successfully connected to scrutiny sqlite db: /opt/scrutiny/config/scrutiny.db\n"
+scrutiny    | ts=2022-05-12T22:54:12.240791Z lvl=info msg=Unauthorized log_id=0aQcVlOW000 error="authorization not found"
+scrutiny    | panic: unauthorized: unauthorized access
+```
+
+Unfortunately this may mean that your database is lost, and the previous Scrutiny data is unavailable. 
+You should fix the docker-compose/docker run command that you're using to ensure that your database folder is persisted correctly, 
+then delete the `web.influxdb.token` field in your `scrutiny.yaml` file, and then restart Scrutiny.
+
+
+## First Start
+The web/api service will trigger an InfluxDB onboarding process automatically when it first starts. After that, it will store the newly generated influxdb api token in the Scrutiny config file. 
+
+If this Credential is not correctly stored in the scrutiny config file, Scrutiny will fail to start (with an authentication error)
+
+```
+scrutiny    | time="2022-05-12T22:52:55Z" level=info msg="Successfully connected to scrutiny sqlite db: /opt/scrutiny/config/scrutiny.db\n"
+scrutiny    | ts=2022-05-12T22:52:55.235753Z lvl=error msg="failed to onboard user admin" log_id=0aQcRnc0000 handler=onboard error="onboarding has already been completed" took=0.038ms
+scrutiny    | ts=2022-05-12T22:52:55.235816Z lvl=error msg="api error encountered" log_id=0aQcRnc0000 error="onboarding has already been completed"
+scrutiny    | panic: conflict: onboarding has already been completed
+```
+
+You can fix this issue by authenticating to the InfluxDB admin portal (the default credentials are username: `admin`, password: `password12345`),
+then retrieving the API token, and writing it to your `scrutiny.yaml` config file under the `web.influxdb.token` field:
+
+![influx db admin token](./influxdb-admin-token.png)
+
+## Upgrading from v0.3.x to v0.4.x
+
+When upgrading from v0.3.x to v0.4.x, some users have noticed problems such as:
+
+```
+2022/05/13 14:38:05 Loading configuration file: /opt/scrutiny/config/scrutiny.yaml
+time="2022-05-13T14:38:05Z" level=info msg="Trying to connect to scrutiny sqlite db:"
+time="2022-05-13T14:38:05Z" level=info msg="Successfully connected to scrutiny sqlite db:"
+panic: a username and password is required for a setup
+```
+
+As discussed in [#248](https://github.com/AnalogJ/scrutiny/issues/248) and [#234](https://github.com/AnalogJ/scrutiny/issues/234),
+this usually related to either:
+
+- Upgrading from the LSIO Scrutiny image to the Official Scrutiny image, without removing LSIO specific environmental variables
+  - remove the `SCRUTINY_WEB=true` and `SCRUTINY_COLLECTOR=true` environmental variables. They were used by the LSIO image, but are unnecessary and cause issues with the official Scrutiny image.
+- Updated versions of the [LSIO Scrutiny images are broken](https://github.com/linuxserver/docker-scrutiny/issues/22), as they have not installed InfluxDB which is a required dependency of Scrutiny v0.4.x
+  - You can revert to an earlier version of the LSIO image (`lscr.io/linuxserver/scrutiny:060ac7b8-ls34`), or just change to the official Scrutiny image (`ghcr.io/analogj/scrutiny:master-omnibus`)
+
+Here's a couple of confirmed working docker-compose files that you may want to look at:
+
+- https://github.com/AnalogJ/scrutiny/blob/master/docker/example.hubspoke.docker-compose.yml
+- https://github.com/AnalogJ/scrutiny/blob/master/docker/example.omnibus.docker-compose.yml

docs/TROUBLESHOOTING_NOTIFICATIONS.md

@@ -0,0 +1,25 @@
+# Notifications
+
+As documented in [example.scrutiny.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.scrutiny.yaml#L59-L75)
+there are multiple ways to configure notifications for Scrutiny.
+
+Under the hood we use a library called [Shoutrrr](https://github.com/containrrr/shoutrrr) to send our notifications, and you should use their documentation if you run into
+any issues: https://containrrr.dev/shoutrrr/services/overview/
+
+
+# Script Notifications
+
+While the Shoutrrr library supports many popular providers for sending notifications Scrutiny also supports a "script" based
+notification system, allowing you to execute a custom script whenever a notification needs to be sent. 
+Data is provided to this script using the following environmental variables:
+
+```
+SCRUTINY_SUBJECT - 	eg. "Scrutiny SMART error (%s) detected on device: %s"
+SCRUTINY_DATE 
+SCRUTINY_FAILURE_TYPE - EmailTest, SmartFail, ScrutinyFail
+SCRUTINY_DEVICE_NAME - eg. /dev/sda
+SCRUTINY_DEVICE_TYPE - ATA/SCSI/NVMe
+SCRUTINY_DEVICE_SERIAL - eg. WDDJ324KSO
+SCRUTINY_MESSAGE - eg. "Scrutiny SMART error notification for device: %s\nFailure Type: %s\nDevice Name: %s\nDevice Serial: %s\nDevice Type: %s\nDate: %s"
+```
+

docs/TROUBLESHOOTING_REVERSE_PROXY.md

@@ -55,4 +55,52 @@ api:
 You may also configure these values using the following environmental variables (both are required).
 
 - `COLLECTOR_API_ENDPOINT=http://localhost:8080/custombasepath`
-- `SCRUTINY_WEB_LISTEN_BASEPATH=/custombasepath`
+- `SCRUTINY_WEB_LISTEN_BASEPATH=/custombasepath`
+
+# Real Examples
+
+## Caddy
+
+1. Create a Caddyfile
+    ```yaml
+    # Caddyfile
+    :9090
+    
+    # The `scrutiny` text in this file must match the service name in the docker-compose file below. 
+    # The `/custom/` text is the custom base path scrutiny will be availble on. 
+    reverse_proxy /custom/* scrutiny:8080
+
+    ```
+2. Create a `docker-compose.yml` file
+
+    ```yaml
+    # docker-compose.yml
+    version: '3.5'
+    
+    services:
+      scrutiny:
+        container_name: scrutiny
+        image: ghcr.io/analogj/scrutiny:master-omnibus
+        cap_add:
+          - SYS_RAWIO
+        ports:
+          - "8086:8086" # influxDB admin
+        volumes:
+          - /run/udev:/run/udev:ro
+          - ./config:/opt/scrutiny/config
+          - ./influxdb:/opt/scrutiny/influxdb
+        devices:
+          - "/dev/sda"
+          - "/dev/sdb"
+        environment:
+          - SCRUTINY_WEB_LISTEN_BASEPATH=/custom
+          - COLLECTOR_API_ENDPOINT=http://localhost:8080/custom
+      caddy:
+        image: caddy
+        volumes:
+          - ./Caddyfile:/etc/caddy/Caddyfile
+        ports:
+          - "9090:9090"
+    ```
+3. run `docker-compose up`
+4. visit [http://localhost:9090/custom/web](http://localhost:9090/custom/web) - access the scrutiny container via caddy reverse proxy

example.collector.yaml

@@ -53,6 +53,13 @@ devices:
 #      - 3ware,3
 #      - 3ware,4
 #      - 3ware,5
+#
+#  # example to show how to override the smartctl command args (per device), see below for how to override these globally.
+#  - device: /dev/sda
+#    commands:
+#      metrics_info_args: '--info --json -T permissive' # used to determine device unique ID & register device with Scrutiny
+#      metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device.
+
 
 #log:
 #  file: '' #absolute or relative paths allowed, eg. web.log
@@ -64,6 +71,12 @@ devices:
 # if you need to use a custom base path (for a reverse proxy), you can add a suffix to the endpoint.
 #  See docs/TROUBLESHOOTING_REVERSE_PROXY.md for more info,
 
+# example to show how to override the smartctl command args globally
+#commands:
+#  metrics_scan_args: '--scan --json' # used to detect devices
+#  metrics_info_args: '--info --json' # used to determine device unique ID & register device with Scrutiny
+#  metrics_smart_args: '--xall --json' # used to retrieve smart data for each device.
+
 
 ########################################################################################################################
 # FEATURES COMING SOON
@@ -72,3 +85,13 @@ devices:
 #
 ########################################################################################################################
 
+#collect:
+#  metric:
+#    enable: true
+#    command: '-a -o on -S on'
+#  long:
+#    enable: false
+#    command: ''
+#  short:
+#    enable: false
+#    command: ''

example.scrutiny.yaml

@@ -34,7 +34,13 @@ web:
     # the location on the filesystem where scrutiny javascript + css is located
     frontend:
       path: /opt/scrutiny/web
+
+  # if you're running influxdb on a different host (or using a cloud-provider) you'll need to update the host & port below.
+  # token, org, bucket are unnecessary for a new InfluxDB installation, as Scrutiny will automatically run the InfluxDB setup,
+  # and store the information in the config file. If you 're re-using an existing influxdb installation, you'll need to provide
+  # the `token`
   influxdb:
+#    scheme: 'http'
     host: 0.0.0.0
     port: 8086
 #    token: 'my-token'
@@ -75,12 +81,6 @@ log:
 #
 ########################################################################################################################
 
-#disks:
-#  include:
-#    # - /dev/sda
-#  exclude:
-#    # - /dev/sdb
-
 #limits:
 #  ata:
 #    critical:
@@ -95,14 +95,3 @@ log:
 #    critical: true
 #    standard: true
 
-
-#collect:
-#  metric:
-#    enable: true
-#    command: '-a -o on -S on'
-#  long:
-#    enable: false
-#    command: ''
-#  short:
-#    enable: false
-#    command: ''

go.mod

@@ -1,31 +1,83 @@
 module github.com/analogj/scrutiny
 
-go 1.13
+go 1.17
 
 require (
 	github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
-	github.com/citilinkru/libudev v1.0.0 // indirect
 	github.com/containrrr/shoutrrr v0.4.4
 	github.com/fatih/color v1.10.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-gormigrate/gormigrate/v2 v2.0.0
 	github.com/golang/mock v1.4.3
-	github.com/google/uuid v1.2.0 // indirect
-	github.com/influxdata/influxdb-client-go/v2 v2.2.3
+	github.com/influxdata/influxdb-client-go/v2 v2.9.0
 	github.com/jaypipes/ghw v0.6.1
 	github.com/jinzhu/gorm v1.9.16
-	github.com/klauspost/compress v1.12.1 // indirect
-	github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect
-	github.com/mattn/go-sqlite3 v1.14.4 // indirect
 	github.com/mitchellh/mapstructure v1.2.2
-	github.com/onsi/ginkgo v1.16.1 // indirect
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/viper v1.7.0
 	github.com/stretchr/testify v1.5.1
 	github.com/urfave/cli/v2 v2.2.0
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
-	golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect
 	gorm.io/driver/sqlite v1.1.3
 	gorm.io/gorm v1.20.2
+)
+
+require (
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/citilinkru/libudev v1.0.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deepmap/oapi-codegen v1.8.2 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/go-playground/locales v0.13.0 // indirect
+	github.com/go-playground/universal-translator v0.17.0 // indirect
+	github.com/go-playground/validator/v10 v10.2.0 // indirect
+	github.com/golang/protobuf v1.4.2 // indirect
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
+	github.com/jaypipes/pcidb v0.5.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.1 // indirect
+	github.com/json-iterator/go v1.1.9 // indirect
+	github.com/klauspost/compress v1.12.1 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect
+	github.com/leodido/go-urn v1.2.0 // indirect
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-sqlite3 v1.14.4 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo v1.16.1 // indirect
+	github.com/pelletier/go-toml v1.7.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/ugorji/go/codec v1.1.7 // indirect
+	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
+	golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
+	golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect
+	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/protobuf v1.23.0 // indirect
+	gopkg.in/ini.v1 v1.55.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gosrc.io/xmpp v0.5.1 // indirect
+	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
 	nhooyr.io/websocket v1.8.7 // indirect
 )

go.sum

@@ -11,6 +11,7 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -60,9 +61,10 @@ github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deepmap/oapi-codegen v1.3.13 h1:9HKGCsdJqE4dnrQ8VerFS0/1ZOJPmAhN+g8xgp8y3K4=
-github.com/deepmap/oapi-codegen v1.3.13/go.mod h1:WAmG5dWY8/PYHt4vKxlt90NsbHMAOCiteYKZMiIRfOo=
+github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU=
+github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
 github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg=
 github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
@@ -75,14 +77,14 @@ github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/getkin/kin-openapi v0.13.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw=
+github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
-github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gormigrate/gormigrate/v2 v2.0.0 h1:e2A3Uznk4viUC4UuemuVgsNnvYZyOA8B3awlYk3UioU=
 github.com/go-gormigrate/gormigrate/v2 v2.0.0/go.mod h1:YuVJ+D/dNt4HWrThTBnjgZuRbt7AuwINeg4q52ZE3Jw=
@@ -93,6 +95,9 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -100,15 +105,20 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -136,6 +146,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -149,9 +160,12 @@ github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -179,12 +193,14 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/influxdata/influxdb-client-go/v2 v2.2.3 h1:082jdJ5t1CFeo0rpGQvKAK1mONVSbFhL4finWA5bRM8=
-github.com/influxdata/influxdb-client-go/v2 v2.2.3/go.mod h1:fa/d1lAdUHxuc1jedx30ZfNG573oQTQmUni3N6pcW+0=
+github.com/influxdata/influxdb-client-go/v2 v2.9.0 h1:1Ejxpt+cpWkadefxd5xvVx7pFgFaafdNp1ItfHzKRW4=
+github.com/influxdata/influxdb-client-go/v2 v2.9.0/go.mod h1:x7Jo5UHHl+w8wu8UnGiNobDDHygojXwJX4mx7rXGKMk=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
+github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
 github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
 github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
 github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
@@ -192,18 +208,24 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU
 github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk=
 github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
 github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
+github.com/jackc/pgconn v1.6.4 h1:S7T6cx5o2OqmxdHaXLH1ZeD1SbI8jBznyYE9Ec0RCQ8=
 github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78=
+github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
 github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
 github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
 github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
 github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
 github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
 github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
 github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.0.2 h1:q1Hsy66zh4vuNsajBUF2PNqfAMMfxU5mk594lPE9vjY=
 github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
 github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
 github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
 github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
 github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
@@ -211,6 +233,7 @@ github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrU
 github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0=
 github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
 github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
+github.com/jackc/pgtype v1.4.2 h1:t+6LWm5eWPLX1H5Se702JSBcirq6uWa4jiG4wV1rAWY=
 github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
 github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
 github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
@@ -218,11 +241,13 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ
 github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA=
 github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
+github.com/jackc/pgx/v4 v4.8.1 h1:SUbCLP2pXvf/Sr/25KsuI4aTxiFYIvpfk4l6aTSdyCw=
 github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
 github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jaypipes/ghw v0.6.1 h1:Ewt3mdpiyhWotGyzg1ursV/6SnToGcG4215X6rR2af8=
 github.com/jaypipes/ghw v0.6.1/go.mod h1:QOXppNRCLGYR1H+hu09FxZPqjNt09bqUZUnOL3Rcero=
@@ -236,12 +261,14 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
 github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
 github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -255,13 +282,15 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 h1:3tLzEnUizyN9YLWFTT9loC30lSBvh2y70LTDcZOTs1s=
 github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0/go.mod h1:8/LTPeDLaklcUjgSQBHbhBF1ibKAFxzS5o+H7USfMSA=
-github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
+github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
@@ -282,8 +311,8 @@ github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIG
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -291,7 +320,6 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
-github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
@@ -331,6 +359,7 @@ github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54=
 github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -371,7 +400,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -416,7 +447,7 @@ github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
-github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -441,15 +472,15 @@ golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -488,12 +519,13 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -531,27 +563,35 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 h1:iGu644GcxtEcrInvDsQRCwJjtCIOlT2V7IRt6ah2Whw=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -616,6 +656,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@@ -634,11 +675,14 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gorm.io/driver/mysql v1.0.1 h1:omJoilUzyrAp0xNoio88lGJCroGdIOen9hq2A/+3ifw=
 gorm.io/driver/mysql v1.0.1/go.mod h1:KtqSthtg55lFp3S5kUXqlGaelnWpKitn4k1xZTnoiPw=
+gorm.io/driver/postgres v1.0.0 h1:Yh4jyFQ0a7F+JPU0Gtiam/eKmpT/XFc1FKxotGqc6FM=
 gorm.io/driver/postgres v1.0.0/go.mod h1:wtMFcOzmuA5QigNsgEIb7O5lhvH1tHAF1RbWmLWV4to=
 gorm.io/driver/sqlite v1.1.1/go.mod h1:hm2olEcl8Tmsc6eZyxYSeznnsDaMqamBvEXLNtBg4cI=
 gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc=
 gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c=
+gorm.io/driver/sqlserver v1.0.2 h1:FzxAlw0/7hntMzSiNfotpYCo9Lz8dqWQGdmCGqIiFGo=
 gorm.io/driver/sqlserver v1.0.2/go.mod h1:gb0Y9QePGgqjzrVyTQUZeh9zkd5v0iz71cM1B4ZycEY=
 gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
 gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=

rootfs/etc/cont-init.d/50-config

@@ -1,4 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-COLLECTOR_CRON_SCHEDULE=${COLLECTOR_CRON_SCHEDULE:-"0 0 * * *"}
-sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron.d/scrutiny

rootfs/etc/cont-init.d/50-cron-config

@@ -0,0 +1,15 @@
+#!/usr/bin/with-contenv bash
+
+# Cron runs in its own isolated environment (usually using only /etc/environment )
+# So when the container starts up, we will do a dump of the runtime environment into a .env file that we
+# will then source into the crontab file (/etc/cron.d/scrutiny)
+(set -o posix; export -p) > /env.sh
+
+# adding ability to customize the cron schedule.
+COLLECTOR_CRON_SCHEDULE=${COLLECTOR_CRON_SCHEDULE:-"0 0 * * *"}
+
+# if the cron schedule has been overridden via env variable (eg docker-compose) we should make sure to strip quotes
+[[ "${COLLECTOR_CRON_SCHEDULE}" == \"*\" || "${COLLECTOR_CRON_SCHEDULE}" == \'*\' ]] && COLLECTOR_CRON_SCHEDULE="${COLLECTOR_CRON_SCHEDULE:1:-1}"
+
+# replace placeholder with correct value
+sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron.d/scrutiny

rootfs/etc/services.d/collector-once/run

@@ -9,5 +9,5 @@ s6-svc -O /var/run/s6/services/collector-once
 # wait until scrutiny is "Ready"
 until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done
 
-echo "starting scrutiny collector"
+echo "starting scrutiny collector (run-once mode. subsequent calls will be triggered via cron service)"
 /opt/scrutiny/bin/scrutiny-collector-metrics run

rootfs/etc/services.d/cron/run

@@ -1,10 +1,4 @@
 #!/usr/bin/with-contenv bash
 
-# Cron runs in its own isolated environment (usually using only /etc/environment )
-# So when the container starts up, we will do a dump of the runtime environment into a .env file that we
-# will then source into the crontab file (/etc/cron.d/scrutiny.sh)
-
-printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
-
 echo "starting cron"
-cron -f
+cron -f -L 15

webapp/backend/pkg/config/config.go

@@ -39,12 +39,14 @@ func (c *configuration) Init() error {
 
 	c.SetDefault("notify.urls", []string{})
 
-	c.SetDefault("web.influxdb.host", "0.0.0.0")
+	c.SetDefault("web.influxdb.scheme", "http")
+	c.SetDefault("web.influxdb.host", "localhost")
 	c.SetDefault("web.influxdb.port", "8086")
 	c.SetDefault("web.influxdb.org", "scrutiny")
 	c.SetDefault("web.influxdb.bucket", "metrics")
 	c.SetDefault("web.influxdb.init_username", "admin")
 	c.SetDefault("web.influxdb.init_password", "password12345")
+	c.SetDefault("web.influxdb.token", "scrutiny-default-admin-token")
 	c.SetDefault("web.influxdb.retention_policy", true)
 
 	//c.SetDefault("disks.include", []string{})

webapp/backend/pkg/constants.go

@@ -4,25 +4,34 @@ const DeviceProtocolAta = "ATA"
 const DeviceProtocolScsi = "SCSI"
 const DeviceProtocolNvme = "NVMe"
 
-const SmartAttributeStatusPassed = 0
-const SmartAttributeStatusFailed = 1
-const SmartAttributeStatusWarning = 2
-
-const SmartWhenFailedFailingNow = "FAILING_NOW"
-const SmartWhenFailedInThePast = "IN_THE_PAST"
-
-//const SmartStatusPassed = "passed"
-//const SmartStatusFailed = "failed"
-
-type DeviceStatus int
+type AttributeStatus uint8
 
 const (
-	DeviceStatusPassed         DeviceStatus = 0
-	DeviceStatusFailedSmart    DeviceStatus = iota
-	DeviceStatusFailedScrutiny DeviceStatus = iota
+	// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
+	AttributeStatusPassed          AttributeStatus = 0
+	AttributeStatusFailedSmart     AttributeStatus = 1
+	AttributeStatusWarningScrutiny AttributeStatus = 2
+	AttributeStatusFailedScrutiny  AttributeStatus = 4
 )
 
-func Set(b, flag DeviceStatus) DeviceStatus    { return b | flag }
-func Clear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
-func Toggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
-func Has(b, flag DeviceStatus) bool            { return b&flag != 0 }
+const AttributeWhenFailedFailingNow = "FAILING_NOW"
+const AttributeWhenFailedInThePast = "IN_THE_PAST"
+
+func AttributeStatusSet(b, flag AttributeStatus) AttributeStatus    { return b | flag }
+func AttributeStatusClear(b, flag AttributeStatus) AttributeStatus  { return b &^ flag }
+func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ flag }
+func AttributeStatusHas(b, flag AttributeStatus) bool               { return b&flag != 0 }
+
+type DeviceStatus uint8
+
+const (
+	// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
+	DeviceStatusPassed         DeviceStatus = 0
+	DeviceStatusFailedSmart    DeviceStatus = 1
+	DeviceStatusFailedScrutiny DeviceStatus = 2
+)
+
+func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus    { return b | flag }
+func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
+func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
+func DeviceStatusHas(b, flag DeviceStatus) bool            { return b&flag != 0 }

webapp/backend/pkg/database/interface.go

@@ -19,6 +19,7 @@ type DeviceRepo interface {
 	UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error)
 	UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error)
 	GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error)
+	DeleteDevice(ctx context.Context, wwn string) error
 
 	SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error)
 	GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error)

webapp/backend/pkg/database/migrations/m20220503120000/device.go

@@ -5,6 +5,7 @@ import (
 	"time"
 )
 
+// Deprecated: m20220503120000.Device is deprecated, only used by db migrations
 type Device struct {
 	//GORM attributes, see: http://gorm.io/docs/conventions.html
 	CreatedAt time.Time

webapp/backend/pkg/database/migrations/m20220509170100/device.go

@@ -0,0 +1,41 @@
+package m20220509170100
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"time"
+)
+
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN string `json:"wwn" gorm:"primary_key"`
+
+	DeviceName     string `json:"device_name"`
+	DeviceUUID	   string `json:"device_uuid"`
+	DeviceSerialID	   string `json:"device_serial_id"`
+	DeviceLabel	   string `json:"device_label"`
+
+	Manufacturer   string `json:"manufacturer"`
+	ModelName      string `json:"model_name"`
+	InterfaceType  string `json:"interface_type"`
+	InterfaceSpeed string `json:"interface_speed"`
+	SerialNumber   string `json:"serial_number"`
+	Firmware       string `json:"firmware"`
+	RotationSpeed  int    `json:"rotational_speed"`
+	Capacity       int64  `json:"capacity"`
+	FormFactor     string `json:"form_factor"`
+	SmartSupport   bool   `json:"smart_support"`
+	DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string `json:"label"`
+	HostId string `json:"host_id"`
+
+	// Data set by Scrutiny
+	DeviceStatus pkg.DeviceStatus `json:"device_status"`
+}
+

webapp/backend/pkg/database/scrutiny_repository.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
@@ -11,6 +12,9 @@ import (
 	"github.com/sirupsen/logrus"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"time"
 )
 
@@ -75,36 +79,38 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 	// Create a new client using an InfluxDB server base URL and an authentication token
-	influxdbUrl := fmt.Sprintf("http://%s:%s", appConfig.GetString("web.influxdb.host"), appConfig.GetString("web.influxdb.port"))
+	influxdbUrl := fmt.Sprintf("%s://%s:%s", appConfig.GetString("web.influxdb.scheme"), appConfig.GetString("web.influxdb.host"), appConfig.GetString("web.influxdb.port"))
 	globalLogger.Debugf("InfluxDB url: %s", influxdbUrl)
 
 	client := influxdb2.NewClient(influxdbUrl, appConfig.GetString("web.influxdb.token"))
 
-	if !appConfig.IsSet("web.influxdb.token") {
-		globalLogger.Debugf("No influxdb token found, running first-time setup...")
+	//if !appConfig.IsSet("web.influxdb.token") {
+	globalLogger.Debugf("Determine Influxdb setup status...")
+	influxSetupComplete, err := InfluxSetupComplete(influxdbUrl)
+	if err != nil {
+		return nil, fmt.Errorf("failed to check influxdb setup status - %w", err)
+	}
+
+	if !influxSetupComplete {
+		globalLogger.Debugf("Influxdb un-initialized, running first-time setup...")
 
 		// if no token is provided, but we have a valid server, we're going to assume this is the first setup of our server.
 		// we will initialize with a predetermined username & password, that you should change.
 
 		// metrics bucket will have a retention period of 8 days (since it will be down-sampled once a week)
 		// in seconds (60seconds * 60minutes * 24hours * 15 days) = 1_296_000 (see EnsureBucket() function)
-		onboardingResponse, err := client.Setup(
+		_, err := client.SetupWithToken(
 			backgroundContext,
 			appConfig.GetString("web.influxdb.init_username"),
 			appConfig.GetString("web.influxdb.init_password"),
 			appConfig.GetString("web.influxdb.org"),
 			appConfig.GetString("web.influxdb.bucket"),
-			0)
+			0,
+			appConfig.GetString("web.influxdb.token"),
+		)
 		if err != nil {
 			return nil, err
 		}
-
-		appConfig.Set("web.influxdb.token", *onboardingResponse.Auth.Token)
-		// we should write the config file out here. Ignore failures.
-		err = appConfig.WriteConfig()
-		if err != nil {
-			globalLogger.Infof("ignoring error while writing influxdb info to config: %v", err)
-		}
 	}
 
 	// Use blocking write client for writes to desired bucket
@@ -176,6 +182,37 @@ func (sr *scrutinyRepository) Close() error {
 	return nil
 }
 
+func InfluxSetupComplete(influxEndpoint string) (bool, error) {
+	influxUri, err := url.Parse(influxEndpoint)
+	if err != nil {
+		return false, err
+	}
+	influxUri, err = influxUri.Parse("/api/v2/setup")
+	if err != nil {
+		return false, err
+	}
+
+	res, err := http.Get(influxUri.String())
+	if err != nil {
+		return false, err
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return false, err
+	}
+
+	type SetupStatus struct {
+		Allowed bool `json:"allowed"`
+	}
+	var data SetupStatus
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return false, err
+	}
+	return !data.Allowed, nil
+}
+
 func (sr *scrutinyRepository) EnsureBuckets(ctx context.Context, org *domain.Organization) error {
 
 	var mainBucketRetentionRule domain.RetentionRule

webapp/backend/pkg/database/scrutiny_repository_device.go

@@ -7,6 +7,7 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"gorm.io/gorm/clause"
+	"time"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -18,7 +19,7 @@ import (
 func (sr *scrutinyRepository) RegisterDevice(ctx context.Context, dev models.Device) error {
 	if err := sr.gormClient.WithContext(ctx).Clauses(clause.OnConflict{
 		Columns:   []clause.Column{{Name: "wwn"}},
-		DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}),
+		DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type", "device_uuid", "device_serial_id", "device_label"}),
 	}).Create(&dev).Error; err != nil {
 		return err
 	}
@@ -57,7 +58,7 @@ func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string
 		return device, fmt.Errorf("Could not get device from DB: %v", err)
 	}
 
-	device.DeviceStatus = pkg.Set(device.DeviceStatus, status)
+	device.DeviceStatus = pkg.DeviceStatusSet(device.DeviceStatus, status)
 	return device, sr.gormClient.Model(&device).Updates(device).Error
 }
 
@@ -72,3 +73,33 @@ func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, wwn string)
 
 	return device, nil
 }
+
+func (sr *scrutinyRepository) DeleteDevice(ctx context.Context, wwn string) error {
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).Delete(&models.Device{}).Error; err != nil {
+		return err
+	}
+
+	//delete data from influxdb.
+	buckets := []string{
+		sr.appConfig.GetString("web.influxdb.bucket"),
+		fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket")),
+		fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket")),
+		fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket")),
+	}
+
+	for _, bucket := range buckets {
+		sr.logger.Infof("Deleting data for %s in bucket: %s", wwn, bucket)
+		if err := sr.influxClient.DeleteAPI().DeleteWithName(
+			ctx,
+			sr.appConfig.GetString("web.influxdb.org"),
+			bucket,
+			time.Now().AddDate(-10, 0, 0),
+			time.Now(),
+			fmt.Sprintf(`device_wwn="%s"`, wwn),
+		); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

webapp/backend/pkg/database/scrutiny_repository_migrations.go

@@ -2,14 +2,18 @@ package database
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	"github.com/go-gormigrate/gormigrate/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api/http"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	log "github.com/sirupsen/logrus"
 	"gorm.io/gorm"
 	"strconv"
 	"time"
@@ -22,9 +26,12 @@ import (
 
 func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 
-	sr.logger.Infoln("Database migration starting")
+	sr.logger.Infoln("Database migration starting. Please wait, this process may take a long time....")
 
-	m := gormigrate.New(sr.gormClient, gormigrate.DefaultOptions, []*gormigrate.Migration{
+	gormMigrateOptions := gormigrate.DefaultOptions
+	gormMigrateOptions.UseTransaction = true
+
+	m := gormigrate.New(sr.gormClient, gormMigrateOptions, []*gormigrate.Migration{
 		{
 			ID: "20201107210306", // v0.3.13 (pre-influxdb schema). 9fac3c6308dc6cb6cd5bbc43a68cd93e8fb20b87
 			Migrate: func(tx *gorm.DB) error {
@@ -38,16 +45,6 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 					&m20201107210306.SmartNvmeAttribute{},
 				)
 			},
-			Rollback: func(tx *gorm.DB) error {
-				return tx.Migrator().DropTable(
-					&m20201107210306.Device{},
-					&m20201107210306.Smart{},
-					&m20201107210306.SmartAtaAttribute{},
-					&m20201107210306.SmartNvmeAttribute{},
-					&m20201107210306.SmartNvmeAttribute{},
-					"self_tests",
-				)
-			},
 		},
 		{
 			ID: "20220503113100", // backwards compatible - influxdb schema
@@ -137,7 +134,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 								smartTags,
 								smartFields,
 								postSmartResults.Date, ctx)
-							if err != nil {
+							if ignorePastRetentionPolicyError(err) != nil {
 								return err
 							}
 
@@ -147,7 +144,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 								tempTags,
 								tempFields,
 								postSmartResults.Date, ctx)
-							if err != nil {
+							if ignorePastRetentionPolicyError(err) != nil {
 								return err
 							}
 						}
@@ -166,7 +163,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 								smartFields,
 								postSmartResults.Date, ctx)
 
-							if err != nil {
+							if ignorePastRetentionPolicyError(err) != nil {
 								return err
 							}
 
@@ -176,7 +173,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 								tempTags,
 								tempFields,
 								postSmartResults.Date, ctx)
-							if err != nil {
+							if ignorePastRetentionPolicyError(err) != nil {
 								return err
 							}
 						}
@@ -194,7 +191,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 								smartTags,
 								smartFields,
 								postSmartResults.Date, ctx)
-							if err != nil {
+							if ignorePastRetentionPolicyError(err) != nil {
 								return err
 							}
 
@@ -204,7 +201,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 								tempTags,
 								tempFields,
 								postSmartResults.Date, ctx)
-							if err != nil {
+							if ignorePastRetentionPolicyError(err) != nil {
 								return err
 							}
 						}
@@ -221,7 +218,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 								smartTags,
 								smartFields,
 								postSmartResults.Date, ctx)
-							if err != nil {
+							if ignorePastRetentionPolicyError(err) != nil {
 								return err
 							}
 
@@ -231,7 +228,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 								tempTags,
 								tempFields,
 								postSmartResults.Date, ctx)
-							if err != nil {
+							if ignorePastRetentionPolicyError(err) != nil {
 								return err
 							}
 						}
@@ -257,20 +254,44 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 					return err
 				}
 
-				//migrate the device database to the current version
+				//migrate the device database
 				return tx.AutoMigrate(m20220503120000.Device{})
 			},
 		},
+		{
+			ID: "m20220509170100", // addl udev device data
+			Migrate: func(tx *gorm.DB) error {
+
+				//migrate the device database.
+				// adding addl columns (device_label, device_uuid, device_serial_id)
+				return tx.AutoMigrate(m20220509170100.Device{})
+			},
+		},
 	})
 
 	if err := m.Migrate(); err != nil {
-		sr.logger.Errorf("Database migration failed with error: %w", err)
+		sr.logger.Errorf("Database migration failed with error. \n Please open a github issue at https://github.com/AnalogJ/scrutiny and attach a copy of your scrutiny.db file. \n %v", err)
 		return err
 	}
 	sr.logger.Infoln("Database migration completed successfully")
 	return nil
 }
 
+// helpers
+
+//When adding data to influxdb, an error may be returned if the data point is outside the range of the retention policy.
+//This function will ignore retention policy errors, and allow the migration to continue.
+func ignorePastRetentionPolicyError(err error) error {
+	var influxDbWriteError *http.Error
+	if errors.As(err, &influxDbWriteError) {
+		if influxDbWriteError.StatusCode == 422 {
+			log.Infoln("ignoring error: attempted to writePoint past retention period duration")
+			return nil
+		}
+	}
+	return err
+}
+
 // Deprecated
 func m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.SmartTemperature) {
 	//extract temperature data for every datapoint

webapp/backend/pkg/models/device.go

@@ -21,6 +21,10 @@ type Device struct {
 	WWN string `json:"wwn" gorm:"primary_key"`
 
 	DeviceName     string `json:"device_name"`
+	DeviceUUID     string `json:"device_uuid"`
+	DeviceSerialID string `json:"device_serial_id"`
+	DeviceLabel    string `json:"device_label"`
+
 	Manufacturer   string `json:"manufacturer"`
 	ModelName      string `json:"model_name"`
 	InterfaceType  string `json:"interface_type"`
@@ -162,7 +166,7 @@ func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
 	dv.DeviceProtocol = info.Device.Protocol
 
 	if !info.SmartStatus.Passed {
-		dv.DeviceStatus = pkg.Set(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
+		dv.DeviceStatus = pkg.DeviceStatusSet(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
 	}
 
 	return nil

webapp/backend/pkg/models/measurements/smart.go

@@ -110,7 +110,7 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er
 	sm.PowerCycleCount = info.PowerCycleCount
 	sm.PowerOnHours = info.PowerOnTime.Hours
 	if !info.SmartStatus.Passed {
-		sm.Status = pkg.DeviceStatusFailedSmart
+		sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedSmart)
 	}
 
 	sm.DeviceProtocol = info.Device.Protocol
@@ -148,8 +148,9 @@ func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTa
 		}
 		attrModel.PopulateAttributeStatus()
 		sm.Attributes[strconv.Itoa(collectorAttr.ID)] = &attrModel
-		if attrModel.Status == pkg.SmartAttributeStatusFailed {
-			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+
+		if pkg.AttributeStatusHas(attrModel.Status, pkg.AttributeStatusFailedScrutiny) {
+			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
 		}
 	}
 }
@@ -171,15 +172,15 @@ func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.Nv
 		"power_on_hours":       (&SmartNvmeAttribute{AttributeId: "power_on_hours", Value: nvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1}).PopulateAttributeStatus(),
 		"unsafe_shutdowns":     (&SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Value: nvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1}).PopulateAttributeStatus(),
 		"media_errors":         (&SmartNvmeAttribute{AttributeId: "media_errors", Value: nvmeSmartHealthInformationLog.MediaErrors, Threshold: 0}).PopulateAttributeStatus(),
-		"num_err_log_entries":  (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: nvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0}).PopulateAttributeStatus(),
+		"num_err_log_entries":  (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: nvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: -1}).PopulateAttributeStatus(),
 		"warning_temp_time":    (&SmartNvmeAttribute{AttributeId: "warning_temp_time", Value: nvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1}).PopulateAttributeStatus(),
 		"critical_comp_time":   (&SmartNvmeAttribute{AttributeId: "critical_comp_time", Value: nvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1}).PopulateAttributeStatus(),
 	}
 
 	//find analyzed attribute status
 	for _, val := range sm.Attributes {
-		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
-			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		if pkg.AttributeStatusHas(val.GetStatus(), pkg.AttributeStatusFailedScrutiny) {
+			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
 		}
 	}
 }
@@ -204,8 +205,8 @@ func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog
 
 	//find analyzed attribute status
 	for _, val := range sm.Attributes {
-		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
-			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		if pkg.AttributeStatusHas(val.GetStatus(), pkg.AttributeStatusFailedScrutiny) {
+			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
 		}
 	}
 }

webapp/backend/pkg/models/measurements/smart_ata_attribute.go

@@ -18,13 +18,13 @@ type SmartAtaAttribute struct {
 	WhenFailed  string `json:"when_failed"`
 
 	//Generated data
-	TransformedValue int64   `json:"transformed_value"`
-	Status           int64   `json:"status"`
-	StatusReason     string  `json:"status_reason,omitempty"`
-	FailureRate      float64 `json:"failure_rate,omitempty"`
+	TransformedValue int64               `json:"transformed_value"`
+	Status           pkg.AttributeStatus `json:"status"`
+	StatusReason     string              `json:"status_reason,omitempty"`
+	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
-func (sa *SmartAtaAttribute) GetStatus() int64 {
+func (sa *SmartAtaAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }
 
@@ -43,7 +43,7 @@ func (sa *SmartAtaAttribute) Flatten() map[string]interface{} {
 
 		//Generated Data
 		fmt.Sprintf("attr.%s.transformed_value", idString): sa.TransformedValue,
-		fmt.Sprintf("attr.%s.status", idString):            sa.Status,
+		fmt.Sprintf("attr.%s.status", idString):            int64(sa.Status),
 		fmt.Sprintf("attr.%s.status_reason", idString):     sa.StatusReason,
 		fmt.Sprintf("attr.%s.failure_rate", idString):      sa.FailureRate,
 	}
@@ -77,7 +77,7 @@ func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
 	case "transformed_value":
 		sa.TransformedValue = val.(int64)
 	case "status":
-		sa.Status = val.(int64)
+		sa.Status = pkg.AttributeStatus(val.(int64))
 	case "status_reason":
 		sa.StatusReason = val.(string)
 	case "failure_rate":
@@ -89,14 +89,16 @@ func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
 //populate attribute status, using SMART Thresholds & Observed Metadata
 // Chainable
 func (sa *SmartAtaAttribute) PopulateAttributeStatus() *SmartAtaAttribute {
-	if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedFailingNow {
+	if strings.ToUpper(sa.WhenFailed) == pkg.AttributeWhenFailedFailingNow {
 		//this attribute has previously failed
-		sa.Status = pkg.SmartAttributeStatusFailed
-		sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
+		sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedSmart)
+		sa.StatusReason += "Attribute is failing manufacturer SMART threshold"
+		//if the Smart Status is failed, we should exit early, no need to look at thresholds.
+		return sa
 
-	} else if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedInThePast {
-		sa.Status = pkg.SmartAttributeStatusWarning
-		sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
+	} else if strings.ToUpper(sa.WhenFailed) == pkg.AttributeWhenFailedInThePast {
+		sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
+		sa.StatusReason += "Attribute has previously failed manufacturer SMART threshold"
 	}
 
 	if smartMetadata, ok := thresholds.AtaMetadata[sa.AttributeId]; ok {
@@ -136,16 +138,16 @@ func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttri
 
 			if smartMetadata.Critical {
 				if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = pkg.SmartAttributeStatusFailed
-					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
+					sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
+					sa.StatusReason += "Observed Failure Rate for Critical Attribute is greater than 10%"
 				}
 			} else {
 				if obsThresh.AnnualFailureRate >= 0.20 {
-					sa.Status = pkg.SmartAttributeStatusFailed
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
+					sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
+					sa.StatusReason += "Observed Failure Rate for Non-Critical Attribute is greater than 20%"
 				} else if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = pkg.SmartAttributeStatusWarning
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
+					sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
+					sa.StatusReason += "Observed Failure Rate for Non-Critical Attribute is greater than 10%"
 				}
 			}
 
@@ -155,7 +157,7 @@ func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttri
 	}
 	// no bucket found
 	if smartMetadata.Critical {
-		sa.Status = pkg.SmartAttributeStatusWarning
+		sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
 		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
 	}
 

webapp/backend/pkg/models/measurements/smart_attribute.go

@@ -1,7 +1,9 @@
 package measurements
 
+import "github.com/analogj/scrutiny/webapp/backend/pkg"
+
 type SmartAttribute interface {
 	Flatten() (fields map[string]interface{})
 	Inflate(key string, val interface{})
-	GetStatus() int64
+	GetStatus() pkg.AttributeStatus
 }

webapp/backend/pkg/models/measurements/smart_nvme_attribute.go

@@ -12,13 +12,13 @@ type SmartNvmeAttribute struct {
 	Value       int64  `json:"value"`
 	Threshold   int64  `json:"thresh"`
 
-	TransformedValue int64   `json:"transformed_value"`
-	Status           int64   `json:"status"`
-	StatusReason     string  `json:"status_reason,omitempty"`
-	FailureRate      float64 `json:"failure_rate,omitempty"`
+	TransformedValue int64               `json:"transformed_value"`
+	Status           pkg.AttributeStatus `json:"status"`
+	StatusReason     string              `json:"status_reason,omitempty"`
+	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
-func (sa *SmartNvmeAttribute) GetStatus() int64 {
+func (sa *SmartNvmeAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }
 
@@ -30,7 +30,7 @@ func (sa *SmartNvmeAttribute) Flatten() map[string]interface{} {
 
 		//Generated Data
 		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
-		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            int64(sa.Status),
 		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
 		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
 	}
@@ -54,7 +54,7 @@ func (sa *SmartNvmeAttribute) Inflate(key string, val interface{}) {
 	case "transformed_value":
 		sa.TransformedValue = val.(int64)
 	case "status":
-		sa.Status = val.(int64)
+		sa.Status = pkg.AttributeStatus(val.(int64))
 	case "status_reason":
 		sa.StatusReason = val.(string)
 	case "failure_rate":
@@ -72,8 +72,8 @@ func (sa *SmartNvmeAttribute) PopulateAttributeStatus() *SmartNvmeAttribute {
 			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
 			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
 				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = pkg.SmartAttributeStatusFailed
-				sa.StatusReason = "Attribute is failing recommended SMART threshold"
+				sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
+				sa.StatusReason += "Attribute is failing recommended SMART threshold"
 			}
 		}
 	}

webapp/backend/pkg/models/measurements/smart_scsci_attribute.go

@@ -12,13 +12,13 @@ type SmartScsiAttribute struct {
 	Value       int64  `json:"value"`
 	Threshold   int64  `json:"thresh"`
 
-	TransformedValue int64   `json:"transformed_value"`
-	Status           int64   `json:"status"`
-	StatusReason     string  `json:"status_reason,omitempty"`
-	FailureRate      float64 `json:"failure_rate,omitempty"`
+	TransformedValue int64               `json:"transformed_value"`
+	Status           pkg.AttributeStatus `json:"status"`
+	StatusReason     string              `json:"status_reason,omitempty"`
+	FailureRate      float64             `json:"failure_rate,omitempty"`
 }
 
-func (sa *SmartScsiAttribute) GetStatus() int64 {
+func (sa *SmartScsiAttribute) GetStatus() pkg.AttributeStatus {
 	return sa.Status
 }
 
@@ -30,7 +30,7 @@ func (sa *SmartScsiAttribute) Flatten() map[string]interface{} {
 
 		//Generated Data
 		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
-		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            int64(sa.Status),
 		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
 		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
 	}
@@ -54,7 +54,7 @@ func (sa *SmartScsiAttribute) Inflate(key string, val interface{}) {
 	case "transformed_value":
 		sa.TransformedValue = val.(int64)
 	case "status":
-		sa.Status = val.(int64)
+		sa.Status = pkg.AttributeStatus(val.(int64))
 	case "status_reason":
 		sa.StatusReason = val.(string)
 	case "failure_rate":
@@ -73,7 +73,7 @@ func (sa *SmartScsiAttribute) PopulateAttributeStatus() *SmartScsiAttribute {
 			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
 			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
 				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = pkg.SmartAttributeStatusFailed
+				sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
 				sa.StatusReason = "Attribute is failing recommended SMART threshold"
 			}
 		}

webapp/backend/pkg/models/measurements/smart_test.go

@@ -328,9 +328,12 @@ func TestFromCollectorSmartInfo(t *testing.T) {
 	require.Equal(t, 18, len(smartMdl.Attributes))
 
 	//check that temperature was correctly parsed
-
 	require.Equal(t, int64(163210330144), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).RawValue)
 	require.Equal(t, int64(32), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).TransformedValue)
+
+	//ensure that Scrutiny warning for a non critical attribute does not set device status to failed.
+	require.Equal(t, pkg.AttributeStatusWarningScrutiny, smartMdl.Attributes["3"].GetStatus())
+
 }
 
 func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) {
@@ -381,6 +384,70 @@ func TestFromCollectorSmartInfo_Fail_ScrutinySmart(t *testing.T) {
 	require.Equal(t, 17, len(smartMdl.Attributes))
 }
 
+func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-ata-failed-scrutiny.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusFailedScrutiny, smartMdl.Status)
+	require.Equal(t, pkg.AttributeStatusFailedScrutiny, smartMdl.Attributes["199"].GetStatus(),
+		"scrutiny should detect that %d failed (status: %d, %s)",
+		smartMdl.Attributes["199"].(*measurements.SmartAtaAttribute).AttributeId,
+		smartMdl.Attributes["199"].GetStatus(), smartMdl.Attributes["199"].(*measurements.SmartAtaAttribute).StatusReason,
+	)
+
+	require.Equal(t, 14, len(smartMdl.Attributes))
+}
+
+//TODO: Scrutiny Warn
+//TODO: Smart + Scrutiny Warn
+
+func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-nvme-failed.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusFailedScrutiny, smartMdl.Status)
+	require.Equal(t, pkg.AttributeStatusFailedScrutiny, smartMdl.Attributes["media_errors"].GetStatus(),
+		"scrutiny should detect that %s failed (status: %d, %s)",
+		smartMdl.Attributes["media_errors"].(*measurements.SmartNvmeAttribute).AttributeId,
+		smartMdl.Attributes["media_errors"].GetStatus(),
+		smartMdl.Attributes["media_errors"].(*measurements.SmartNvmeAttribute).StatusReason,
+	)
+
+	require.Equal(t, 16, len(smartMdl.Attributes))
+}
+
 func TestFromCollectorSmartInfo_Nvme(t *testing.T) {
 	//setup
 	smartDataFile, err := os.Open("../testdata/smart-nvme.json")

webapp/backend/pkg/models/testdata/smart-nvme-failed.json

@@ -0,0 +1,107 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-5.13.0-40-generic",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-x",
+      "-j",
+      "/dev/nvme0"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/nvme0",
+    "info_name": "/dev/nvme0",
+    "type": "nvme",
+    "protocol": "NVMe"
+  },
+  "model_name": "Samsung SSD 970 EVO 500GB",
+  "serial_number": "S466NX0M776250H",
+  "firmware_version": "2B2QEXE7",
+  "nvme_pci_vendor": {
+    "id": 5197,
+    "subsystem_id": 5197
+  },
+  "nvme_ieee_oui_identifier": 9528,
+  "nvme_total_capacity": 500107862016,
+  "nvme_unallocated_capacity": 0,
+  "nvme_controller_id": 4,
+  "nvme_number_of_namespaces": 1,
+  "nvme_namespaces": [
+    {
+      "id": 1,
+      "size": {
+        "blocks": 976773168,
+        "bytes": 500107862016
+      },
+      "capacity": {
+        "blocks": 976773168,
+        "bytes": 500107862016
+      },
+      "utilization": {
+        "blocks": 327275384,
+        "bytes": 167564996608
+      },
+      "formatted_lba_size": 512,
+      "eui64": {
+        "oui": 9528,
+        "ext_id": 376106710327
+      }
+    }
+  ],
+  "user_capacity": {
+    "blocks": 976773168,
+    "bytes": 500107862016
+  },
+  "logical_block_size": 512,
+  "local_time": {
+    "time_t": 1652220188,
+    "asctime": "Tue May 10 22:03:08 2022 UTC"
+  },
+  "smart_status": {
+    "passed": true,
+    "nvme": {
+      "value": 0
+    }
+  },
+  "nvme_smart_health_information_log": {
+    "critical_warning": 0,
+    "temperature": 35,
+    "available_spare": 99,
+    "available_spare_threshold": 10,
+    "percentage_used": 3,
+    "data_units_read": 17176794,
+    "data_units_written": 65602088,
+    "host_reads": 118020838,
+    "host_writes": 874050000,
+    "controller_busy_time": 7601,
+    "power_cycles": 25,
+    "power_on_hours": 12798,
+    "unsafe_shutdowns": 10,
+    "media_errors": 7,
+    "num_err_log_entries": 62,
+    "warning_temp_time": 0,
+    "critical_comp_time": 0,
+    "temperature_sensors": [
+      35,
+      39
+    ]
+  },
+  "temperature": {
+    "current": 35
+  },
+  "power_cycle_count": 25,
+  "power_on_time": {
+    "hours": 12798
+  }
+}

webapp/backend/pkg/thresholds/ata_attribute_metadata.go

@@ -445,50 +445,6 @@ var AtaMetadata = map[int]AtaAttributeMetadata{
 		Ideal:       ObservedThresholdIdealLow,
 		Critical:    false,
 		Description: "This attribute indicates the count of full hard disk power on/off cycles.",
-		ObservedThresholds: []ObservedThreshold{
-			{
-				Low:               0,
-				High:              13,
-				AnnualFailureRate: 0.019835987118930823,
-				ErrorInterval:     []float64{0.016560870164523494, 0.023569242386797896},
-			},
-			{
-				Low:               13,
-				High:              26,
-				AnnualFailureRate: 0.038210930067894826,
-				ErrorInterval:     []float64{0.03353859179329295, 0.0433520775718649},
-			},
-			{
-				Low:               26,
-				High:              39,
-				AnnualFailureRate: 0.11053528307302571,
-				ErrorInterval:     []float64{0.09671061589521368, 0.1257816678419765},
-			},
-			{
-				Low:               39,
-				High:              52,
-				AnnualFailureRate: 0.16831189443375036,
-				ErrorInterval:     []float64{0.1440976510675928, 0.19543066007594895},
-			},
-			{
-				Low:               52,
-				High:              65,
-				AnnualFailureRate: 0.20630344262550107,
-				ErrorInterval:     []float64{0.1693965932069108, 0.2488633537247856},
-			},
-			{
-				Low:               65,
-				High:              78,
-				AnnualFailureRate: 0.1030972634140512,
-				ErrorInterval:     []float64{0.06734655535304743, 0.15106137807407605},
-			},
-			{
-				Low:               78,
-				High:              91,
-				AnnualFailureRate: 0.12354840389522469,
-				ErrorInterval:     []float64{0.06578432170016109, 0.21127153335749593},
-			},
-		},
 	},
 	13: {
 		ID:          13,

webapp/backend/pkg/version/version.go

@@ -2,4 +2,4 @@ package version
 
 // VERSION is the app-global version string, which will be replaced with a
 // new value during packaging
-const VERSION = "0.4.3"
+const VERSION = "0.4.9"

webapp/backend/pkg/web/handler/delete_device.go

@@ -0,0 +1,22 @@
+package handler
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"net/http"
+)
+
+func DeleteDevice(c *gin.Context) {
+	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	err := deviceRepo.DeleteDevice(c, c.Param("wwn"))
+	if err != nil {
+		logger.Errorln("An error occurred while deleting device", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"success": true})
+}

webapp/backend/pkg/web/server.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 type AppEngine struct {
@@ -47,6 +48,8 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 			api.POST("/device/:wwn/smart", handler.UploadDeviceMetrics)    //used by Collector to upload data
 			api.POST("/device/:wwn/selftest", handler.UploadDeviceSelfTests)
 			api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details
+			api.DELETE("/device/:wwn", handler.DeleteDevice)          //used by UI to delete device
+
 		}
 	}
 
@@ -66,6 +69,11 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 }
 
 func (ae *AppEngine) Start() error {
+	//set the gin mode
+	gin.SetMode(gin.ReleaseMode)
+	if strings.ToLower(ae.Config.GetString("log.level")) == "debug" {
+		gin.SetMode(gin.DebugMode)
+	}
 
 	logger := logrus.New()
 	//set default log level

webapp/backend/pkg/web/server_test.go

@@ -93,6 +93,7 @@ func (suite *ServerTestSuite) TestHealthRoute() {
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
 
+	fakeConfig.EXPECT().GetString("web.influxdb.scheme").Return("http").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
 	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
@@ -132,6 +133,7 @@ func (suite *ServerTestSuite) TestRegisterDevicesRoute() {
 	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.scheme").Return("http").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
 	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
@@ -171,6 +173,7 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() {
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.scheme").Return("http").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
 	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
@@ -219,6 +222,7 @@ func (suite *ServerTestSuite) TestPopulateMultiple() {
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.scheme").Return("http").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
 	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
@@ -314,6 +318,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() {
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.scheme").Return("http").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
 	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
@@ -352,6 +357,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() {
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.scheme").Return("http").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
 	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
@@ -390,6 +396,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() {
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.scheme").Return("http").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
 	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
@@ -428,6 +435,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() {
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.scheme").Return("http").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
 	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
@@ -465,6 +473,7 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() {
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.scheme").Return("http").AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
 	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
@@ -509,5 +518,5 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() {
 
 	//assert
 	require.Equal(suite.T(), "a4c8e8ed-11a0-4c97-9bba-306440f1b944", deviceSummary.Data.Summary["a4c8e8ed-11a0-4c97-9bba-306440f1b944"].Device.WWN)
-	require.Equal(suite.T(), pkg.DeviceStatusFailedScrutiny, deviceSummary.Data.Summary["a4c8e8ed-11a0-4c97-9bba-306440f1b944"].Device.DeviceStatus)
+	require.Equal(suite.T(), pkg.DeviceStatusPassed, deviceSummary.Data.Summary["a4c8e8ed-11a0-4c97-9bba-306440f1b944"].Device.DeviceStatus)
 }

webapp/frontend/git.version.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+if [[ -z "${CI}" ]]; then
+    echo "running locally (not in Github Actions). generating version file from git client"
+    GIT_TAG=`git describe --tags`
+    GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
+
+    if [[ "$GIT_BRANCH" == "master" ]]; then
+        VERSION_INFO="${GIT_TAG}"
+    else
+        VERSION_INFO="${GIT_BRANCH}#${GIT_TAG}"
+    fi
+else
+    echo "running in Github Actions, generating version file from environmental variables"
+    # https://docs.github.com/en/actions/learn-github-actions/environment-variables
+    VERSION_INFO="${GITHUB_REF_NAME}"
+
+    if [[ "$GITHUB_REF_TYPE" == "branch" ]]; then
+            VERSION_INFO="${VERSION_INFO}#${GITHUB_SHA::7}"
+    fi
+fi
+
+echo "writing version file (version: ${VERSION_INFO})"
+cat <<EOT > src/environments/versions.ts
+// this file is automatically generated by git.version.ts script
+export const versionInfo = {
+    version: '${VERSION_INFO}',
+};
+EOT

webapp/frontend/src/@treo/services/config/config.service.ts

@@ -2,6 +2,9 @@ import { Inject, Injectable } from '@angular/core';
 import { BehaviorSubject, Observable } from 'rxjs';
 import * as _ from 'lodash';
 import { TREO_APP_CONFIG } from '@treo/services/config/config.constants';
+import { AppConfig } from 'app/core/config/app.config';
+
+const SCRUTINY_CONFIG_LOCAL_STORAGE_KEY = 'scrutiny';
 
 @Injectable({
     providedIn: 'root'
@@ -14,10 +17,18 @@ export class TreoConfigService
     /**
      * Constructor
      */
-    constructor(@Inject(TREO_APP_CONFIG) config: any)
+    constructor(@Inject(TREO_APP_CONFIG) defaultConfig: any)
     {
+        let currentScrutinyConfig = defaultConfig
+
+        let localConfigStr = localStorage.getItem(SCRUTINY_CONFIG_LOCAL_STORAGE_KEY)
+        if (localConfigStr){
+            //check localstorage for a value
+            let localConfig = JSON.parse(localConfigStr)
+            currentScrutinyConfig = Object.assign({}, localConfig, currentScrutinyConfig) // make sure defaults are available if missing from localStorage.
+        }
         // Set the private defaults
-        this._config = new BehaviorSubject(config);
+        this._config = new BehaviorSubject(currentScrutinyConfig);
     }
 
     // -----------------------------------------------------------------------------------------------------
@@ -27,20 +38,29 @@ export class TreoConfigService
     /**
      * Setter and getter for config
      */
+    //Setter
     set config(value: any)
     {
         // Merge the new config over to the current config
-        const config = _.merge({}, this._config.getValue(), value);
+        let config = _.merge({}, this._config.getValue(), value);
+
+        //Store the config in localstorage
+        localStorage.setItem(SCRUTINY_CONFIG_LOCAL_STORAGE_KEY, JSON.stringify(config));
 
         // Execute the observable
         this._config.next(config);
     }
 
+    //Getter
     get config$(): Observable<any>
     {
         return this._config.asObservable();
     }
 
+    // -----------------------------------------------------------------------------------------------------
+    // @ Private methods
+    // -----------------------------------------------------------------------------------------------------
+
     // -----------------------------------------------------------------------------------------------------
     // @ Public methods
     // -----------------------------------------------------------------------------------------------------

webapp/frontend/src/app/core/config/app.config.ts

@@ -1,7 +1,7 @@
 import { Layout } from "app/layout/layout.types";
 
 // Theme type
-export type Theme = "light" | "dark";
+export type Theme = "light" | "dark" | "system";
 
 /**
  * AppConfig interface. Update this interface to strictly type your config
@@ -11,6 +11,12 @@ export interface AppConfig
 {
     theme: Theme;
     layout: Layout;
+
+    // Dashboard options
+    dashboardDisplay: string;
+    dashboardSort: string;
+
+    temperatureUnit: string;
 }
 
 /**
@@ -23,6 +29,11 @@ export interface AppConfig
  */
 export const appConfig: AppConfig = {
     theme : "light",
-    layout: "material"
+    layout: "material",
+
+    dashboardDisplay: "name",
+    dashboardSort: "status",
+
+    temperatureUnit: "celsius",
 };
 

webapp/frontend/src/app/data/mock/summary/data.ts

@@ -11,6 +11,9 @@ export const summary = {
                     "DeletedAt": null,
                     "wwn": "0x5000c500673e6b5f",
                     "device_name": "sdg",
+                    "device_label": "14TB-WD-DRIVE2",
+                    "device_uuid": "",
+                    "device_serial_id": "ata-ST6000DX000-1H217Z-Z4DXXXXX",
                     "manufacturer": "ATA",
                     "model_name": "ST6000DX000-1H217Z",
                     "interface_type": "SCSI",
@@ -35,6 +38,9 @@ export const summary = {
                     "DeletedAt": null,
                     "wwn": "0x5000cca252c859cc",
                     "device_name": "sdd",
+                    "device_label": "14TB-WD-DRIVE1",
+                    "device_uuid": "806cf4bc-d160-4d96-8ee9-3ab7cf2a2e1f",
+                    "device_serial_id": "ata-WDC_WD80EFAX-68LHPN0-7SGLXXXXX",
                     "manufacturer": "ATA",
                     "model_name": "WDC_WD80EFAX-68LHPN0",
                     "interface_type": "SCSI",
@@ -68,6 +74,9 @@ export const summary = {
                     "DeletedAt": null,
                     "wwn": "0x5000cca264eb01d7",
                     "device_name": "sdb",
+                    "device_label": "14TB-WD-DRIVE5",
+                    "device_uuid": "8125ec6d-a7e4-4950-ac84-72d6a4d67128",
+                    "device_serial_id": "ata-WDC_WD140EDFZ-11A0VA0-9RK1XXXXX",
                     "manufacturer": "ATA",
                     "model_name": "WDC_WD140EDFZ-11A0VA0",
                     "interface_type": "SCSI",
@@ -101,6 +110,9 @@ export const summary = {
                     "DeletedAt": null,
                     "wwn": "0x5000cca264ebc248",
                     "device_name": "sde",
+                    "device_label": "14TB-WD-DRIVE3",
+                    "device_uuid": "9eb60cde-d6d0-4172-b520-b241a6a5477f",
+                    "device_serial_id": "ata-WDC_WD140EDFZ-11A0VA0-9RK3XXXXX",
                     "manufacturer": "ATA",
                     "model_name": "WDC_WD140EDFZ-11A0VA0",
                     "interface_type": "SCSI",
@@ -125,6 +137,9 @@ export const summary = {
                     "DeletedAt": null,
                     "wwn": "0x5000cca264ec3183",
                     "device_name": "sdc",
+                    "device_label": "14TB-WD-DRIVE6",
+                    "device_uuid": "e1378723-7861-49b9-8e01-0bd063f0ecdd",
+                    "device_serial_id": "ata-WDC_WD140EDFZ-11A0VA0-9RK4XXXXX",
                     "manufacturer": "ATA",
                     "model_name": "WDC_WD140EDFZ-11A0VA0",
                     "interface_type": "SCSI",
@@ -138,7 +153,7 @@ export const summary = {
                     "device_protocol": "",
                     "device_type": "",
                     "label": "",
-                    "host_id": "",
+                    "host_id": "custom host id",
                     "device_status": 1
                 },
                 "smart": {
@@ -542,6 +557,9 @@ export const summary = {
                     "DeletedAt": null,
                     "wwn": "0x50014ee20b2a72a9",
                     "device_name": "sdf",
+                    "device_label": "8.0TB-WD-4",
+                    "device_uuid": "fc684dcc-aa2f-44f3-a958-d302dc7dd46d",
+                    "device_serial_id": "ata-WDC_WD60EFRX-68MYMN1-WXL1HXXXXX",
                     "manufacturer": "ATA",
                     "model_name": "WDC_WD60EFRX-68MYMN1",
                     "interface_type": "SCSI",
@@ -566,6 +584,9 @@ export const summary = {
                     "DeletedAt": null,
                     "wwn": "0x5002538e40a22954",
                     "device_name": "sda",
+                    "device_label": "",
+                    "device_uuid": "",
+                    "device_serial_id": "ata-Samsung_SSD_860_EVO_500GB-S3YZNB0KBXXXXXX",
                     "manufacturer": "ATA",
                     "model_name": "Samsung_SSD_860_EVO_500GB",
                     "interface_type": "SCSI",

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component.html

@@ -0,0 +1,10 @@
+<h2 mat-dialog-title>Delete {{data.title}}?</h2>
+<mat-dialog-content>This will delete all data associated with this device (including all historical data).</mat-dialog-content>
+<mat-dialog-actions>
+    <button mat-button mat-dialog-close>Cancel</button>
+    <!-- The mat-dialog-close directive optionally accepts a value as a result for the dialog. -->
+    <button class="red-600" mat-button (click)="onDeleteClick()">
+        <mat-icon class="icon-size-20 mr-3"
+                  [svgIcon]="'delete_forever'"></mat-icon>
+        Delete</button>
+</mat-dialog-actions>

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component.spec.ts

@@ -0,0 +1,25 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { DashboardDeviceDeleteDialogComponent } from './dashboard-device-delete-dialog.component';
+
+describe('DashboardDeviceDeleteDialogComponent', () => {
+  let component: DashboardDeviceDeleteDialogComponent;
+  let fixture: ComponentFixture<DashboardDeviceDeleteDialogComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [ DashboardDeviceDeleteDialogComponent ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(DashboardDeviceDeleteDialogComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component.ts

@@ -0,0 +1,30 @@
+import { Component, OnInit, Inject } from '@angular/core';
+import {MAT_DIALOG_DATA, MatDialogRef} from '@angular/material/dialog';
+import {DashboardDeviceDeleteDialogService} from "./dashboard-device-delete-dialog.service";
+import {Subject} from "rxjs";
+
+@Component({
+  selector: 'app-dashboard-device-delete-dialog',
+  templateUrl: './dashboard-device-delete-dialog.component.html',
+  styleUrls: ['./dashboard-device-delete-dialog.component.scss']
+})
+export class DashboardDeviceDeleteDialogComponent implements OnInit {
+
+    constructor(
+        public dialogRef: MatDialogRef<DashboardDeviceDeleteDialogComponent>,
+        @Inject(MAT_DIALOG_DATA) public data: {wwn: string, title: string},
+        private _deleteService: DashboardDeviceDeleteDialogService,
+    ) {
+    }
+
+  ngOnInit(): void {
+  }
+
+  onDeleteClick(): void {
+      this._deleteService.deleteDevice(this.data.wwn)
+          .subscribe((data) => {
+              this.dialogRef.close(data);
+          });
+
+  }
+}

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.module.ts

@@ -0,0 +1,52 @@
+import { NgModule } from '@angular/core';
+import { RouterModule } from '@angular/router';
+import { Overlay } from '@angular/cdk/overlay';
+import { MAT_AUTOCOMPLETE_SCROLL_STRATEGY, MatAutocompleteModule } from '@angular/material/autocomplete';
+import { MatButtonModule } from '@angular/material/button';
+import { MatSelectModule } from '@angular/material/select';
+import { MatFormFieldModule } from '@angular/material/form-field';
+import { MatIconModule } from '@angular/material/icon';
+import { MatInputModule } from '@angular/material/input';
+import { SharedModule } from 'app/shared/shared.module';
+import {DashboardDeviceDeleteDialogComponent} from 'app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component'
+import { MatButtonToggleModule} from "@angular/material/button-toggle";
+import {MatTabsModule} from "@angular/material/tabs";
+import {MatSliderModule} from "@angular/material/slider";
+import {MatSlideToggleModule} from "@angular/material/slide-toggle";
+import {MatTooltipModule} from "@angular/material/tooltip";
+import {dashboardRoutes} from "../../../modules/dashboard/dashboard.routing";
+import {MatDividerModule} from "@angular/material/divider";
+import {MatMenuModule} from "@angular/material/menu";
+import {MatProgressBarModule} from "@angular/material/progress-bar";
+import {MatSortModule} from "@angular/material/sort";
+import {MatTableModule} from "@angular/material/table";
+import {NgApexchartsModule} from "ng-apexcharts";
+import { MatDialogModule } from '@angular/material/dialog';
+
+@NgModule({
+    declarations: [
+        DashboardDeviceDeleteDialogComponent
+    ],
+    imports     : [
+        RouterModule.forChild([]),
+        RouterModule.forChild(dashboardRoutes),
+        MatButtonModule,
+        MatDividerModule,
+        MatTooltipModule,
+        MatIconModule,
+        MatMenuModule,
+        MatProgressBarModule,
+        MatSortModule,
+        MatTableModule,
+        NgApexchartsModule,
+        SharedModule,
+        MatDialogModule
+    ],
+    exports     : [
+        DashboardDeviceDeleteDialogComponent,
+    ],
+    providers   : []
+})
+export class DashboardDeviceDeleteDialogModule
+{
+}

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.service.ts

@@ -0,0 +1,34 @@
+import { Injectable } from '@angular/core';
+import { HttpClient } from '@angular/common/http';
+import { BehaviorSubject, Observable } from 'rxjs';
+import { tap } from 'rxjs/operators';
+import { getBasePath } from 'app/app.routing';
+
+@Injectable({
+    providedIn: 'root'
+})
+export class DashboardDeviceDeleteDialogService
+{
+
+
+    /**
+     * Constructor
+     *
+     * @param {HttpClient} _httpClient
+     */
+    constructor(
+        private _httpClient: HttpClient
+    )
+    {
+    }
+
+    // -----------------------------------------------------------------------------------------------------
+    // @ Public methods
+    // -----------------------------------------------------------------------------------------------------
+
+
+    deleteDevice(wwn: string): Observable<any>
+    {
+        return this._httpClient.delete( `${getBasePath()}/api/device/${wwn}`, {});
+    }
+}

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html

@@ -0,0 +1,67 @@
+<div [ngClass]="{ 'border-green': deviceSummary.device.device_status == 0 && deviceSummary.smart,
+                                'border-red': deviceSummary.device.device_status != 0 }"
+     class="relative flex flex-col flex-auto p-6 pr-3 pb-3 bg-card rounded border-l-4 shadow-md overflow-hidden">
+    <div class="absolute bottom-0 right-0 w-24 h-24 -m-6">
+        <mat-icon class="icon-size-96 opacity-12 text-green"
+                  *ngIf="deviceSummary.device.device_status == 0 && deviceSummary.smart"
+                  [svgIcon]="'heroicons_outline:check-circle'"></mat-icon>
+        <mat-icon class="icon-size-96 opacity-12 text-red"
+                  *ngIf="deviceSummary.device.device_status != 0"
+                  [svgIcon]="'heroicons_outline:exclamation-circle'"></mat-icon>
+        <mat-icon class="icon-size-96 opacity-12 text-yellow"
+                  *ngIf="!deviceSummary.smart"
+                  [svgIcon]="'heroicons_outline:question-mark-circle'"></mat-icon>
+    </div>
+    <div class="flex items-center">
+        <div class="flex flex-col">
+            <a [routerLink]="'/device/'+ deviceSummary.device.wwn"
+               class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceSummary.device | deviceTitle:config.dashboardDisplay}}</a>
+            <div [ngClass]="classDeviceLastUpdatedOn(deviceSummary)" class="font-medium text-sm" *ngIf="deviceSummary.smart">
+                Last Updated on {{deviceSummary.smart.collector_date | date:'MMMM dd, yyyy - HH:mm' }}
+            </div>
+        </div>
+        <div class="ml-auto" *ngIf="deviceSummary.device">
+            <button mat-icon-button
+                    [matMenuTriggerFor]="previousStatementMenu">
+                <mat-icon [svgIcon]="'more_vert'"></mat-icon>
+            </button>
+            <mat-menu #previousStatementMenu="matMenu">
+                <a mat-menu-item [routerLink]="'/device/'+ deviceSummary.device.wwn">
+                                <span class="flex items-center">
+                                    <mat-icon class="icon-size-20 mr-3"
+                                              [svgIcon]="'assessment'"></mat-icon>
+                                    <span>View Details</span>
+                                </span>
+                </a>
+                <a mat-menu-item (click)="openDeleteDialog()">
+                                <span class="flex items-center">
+                                    <mat-icon class="icon-size-20 mr-3"
+                                              [svgIcon]="'delete_forever'"></mat-icon>
+                                    <span>Delete Device</span>
+                                </span>
+                </a>
+            </mat-menu>
+        </div>
+    </div>
+    <div class="flex flex-row flex-wrap mt-4 -mx-6">
+        <div class="flex flex-col mx-6 my-3 xs:w-full">
+            <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Status</div>
+            <div class="mt-2 font-medium text-3xl leading-none" *ngIf="deviceSummary.smart?.collector_date; else unknownStatus">{{ deviceStatusString(deviceSummary.device.device_status) | titlecase}}</div>
+            <ng-template #unknownStatus><div class="mt-2 font-medium text-3xl leading-none">No Data</div></ng-template>
+        </div>
+        <div class="flex flex-col mx-6 my-3 xs:w-full">
+            <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Temperature</div>
+            <div class="mt-2 font-medium text-3xl leading-none" *ngIf="deviceSummary.smart?.collector_date; else unknownTemp">{{ deviceSummary.smart?.temp | temperature:config.temperatureUnit:true }}</div>
+            <ng-template #unknownTemp><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
+        </div>
+        <div class="flex flex-col mx-6 my-3 xs:w-full">
+            <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Capacity</div>
+            <div class="mt-2 font-medium text-3xl leading-none">{{ deviceSummary.device.capacity | fileSize}}</div>
+        </div>
+        <div class="flex flex-col mx-6 my-3 xs:w-full">
+            <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Powered On</div>
+            <div class="mt-2 font-medium text-3xl leading-none" *ngIf="deviceSummary.smart?.power_on_hours; else unknownPoweredOn">{{ humanizeDuration(deviceSummary.smart?.power_on_hours * 60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] }) }}</div>
+            <ng-template #unknownPoweredOn><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
+        </div>
+    </div>
+</div>

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts

@@ -0,0 +1,25 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { DashboardDeviceComponent } from './dashboard-device.component';
+
+describe('DashboardDeviceComponent', () => {
+  let component: DashboardDeviceComponent;
+  let fixture: ComponentFixture<DashboardDeviceComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [ DashboardDeviceComponent ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(DashboardDeviceComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts

@@ -0,0 +1,93 @@
+import { Component, Input, Output, OnInit, EventEmitter} from '@angular/core';
+import * as moment from "moment";
+import {takeUntil} from "rxjs/operators";
+import {AppConfig} from "app/core/config/app.config";
+import {TreoConfigService} from "@treo/services/config";
+import {Subject} from "rxjs";
+import  humanizeDuration from 'humanize-duration'
+import {MatDialog} from '@angular/material/dialog';
+import {DashboardDeviceDeleteDialogComponent} from "app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component";
+import {DeviceTitlePipe} from "app/shared/device-title.pipe";
+
+@Component({
+  selector: 'app-dashboard-device',
+  templateUrl: './dashboard-device.component.html',
+  styleUrls: ['./dashboard-device.component.scss']
+})
+export class DashboardDeviceComponent implements OnInit {
+    @Input() deviceSummary: any;
+    @Input() deviceWWN: string;
+    @Output() deviceDeleted = new EventEmitter<string>();
+
+    config: AppConfig;
+
+    private _unsubscribeAll: Subject<any>;
+
+    constructor(
+        private _configService: TreoConfigService,
+        public dialog: MatDialog,
+    ) {
+        // Set the private defaults
+        this._unsubscribeAll = new Subject();
+    }
+
+    ngOnInit(): void {
+        // Subscribe to config changes
+        this._configService.config$
+            .pipe(takeUntil(this._unsubscribeAll))
+            .subscribe((config: AppConfig) => {
+                this.config = config;
+            });
+    }
+
+
+    // -----------------------------------------------------------------------------------------------------
+    // @ Public methods
+    // -----------------------------------------------------------------------------------------------------
+
+    classDeviceLastUpdatedOn(deviceSummary){
+        if (deviceSummary.device.device_status !== 0) {
+            return 'text-red' // if the device has failed, always highlight in red
+        } else if(deviceSummary.device.device_status === 0 && deviceSummary.smart){
+            if(moment().subtract(14, 'd').isBefore(deviceSummary.smart.collector_date)){
+                // this device was updated in the last 2 weeks.
+                return 'text-green'
+            } else if(moment().subtract(1, 'm').isBefore(deviceSummary.smart.collector_date)){
+                // this device was updated in the last month
+                return 'text-yellow'
+            } else{
+                // last updated more than a month ago.
+                return 'text-red'
+            }
+
+        } else {
+            return ''
+        }
+    }
+
+    deviceStatusString(deviceStatus){
+        if(deviceStatus == 0){
+            return "passed"
+        } else {
+            return "failed"
+        }
+    }
+
+    readonly humanizeDuration = humanizeDuration;
+
+
+
+    openDeleteDialog(): void {
+        const dialogRef = this.dialog.open(DashboardDeviceDeleteDialogComponent, {
+            // width: '250px',
+            data: {wwn: this.deviceWWN, title: DeviceTitlePipe.deviceTitleWithFallback(this.deviceSummary.device, this.config.dashboardDisplay)}
+        });
+
+        dialogRef.afterClosed().subscribe(result => {
+            console.log('The dialog was closed', result);
+            if(result.success){
+                this.deviceDeleted.emit(this.deviceWWN)
+            }
+        });
+    }
+}

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.module.ts

@@ -0,0 +1,53 @@
+import { NgModule } from '@angular/core';
+import { RouterModule } from '@angular/router';
+import { Overlay } from '@angular/cdk/overlay';
+import { MAT_AUTOCOMPLETE_SCROLL_STRATEGY, MatAutocompleteModule } from '@angular/material/autocomplete';
+import { MatButtonModule } from '@angular/material/button';
+import { MatSelectModule } from '@angular/material/select';
+import { MatFormFieldModule } from '@angular/material/form-field';
+import { MatIconModule } from '@angular/material/icon';
+import { MatInputModule } from '@angular/material/input';
+import { SharedModule } from 'app/shared/shared.module';
+import {DashboardDeviceComponent} from 'app/layout/common/dashboard-device/dashboard-device.component'
+import { MatDialogModule } from "@angular/material/dialog";
+import { MatButtonToggleModule} from "@angular/material/button-toggle";
+import {MatTabsModule} from "@angular/material/tabs";
+import {MatSliderModule} from "@angular/material/slider";
+import {MatSlideToggleModule} from "@angular/material/slide-toggle";
+import {MatTooltipModule} from "@angular/material/tooltip";
+import {dashboardRoutes} from "../../../modules/dashboard/dashboard.routing";
+import {MatDividerModule} from "@angular/material/divider";
+import {MatMenuModule} from "@angular/material/menu";
+import {MatProgressBarModule} from "@angular/material/progress-bar";
+import {MatSortModule} from "@angular/material/sort";
+import {MatTableModule} from "@angular/material/table";
+import {NgApexchartsModule} from "ng-apexcharts";
+import {DashboardDeviceDeleteDialogModule} from "../dashboard-device-delete-dialog/dashboard-device-delete-dialog.module";
+
+@NgModule({
+    declarations: [
+        DashboardDeviceComponent
+    ],
+    imports     : [
+        RouterModule.forChild([]),
+        RouterModule.forChild(dashboardRoutes),
+        MatButtonModule,
+        MatDividerModule,
+        MatTooltipModule,
+        MatIconModule,
+        MatMenuModule,
+        MatProgressBarModule,
+        MatSortModule,
+        MatTableModule,
+        NgApexchartsModule,
+        SharedModule,
+        DashboardDeviceDeleteDialogModule
+    ],
+    exports     : [
+        DashboardDeviceComponent,
+    ],
+    providers   : []
+})
+export class DashboardDeviceModule
+{
+}

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html

@@ -1,77 +1,110 @@
 <h2 mat-dialog-title>Scrutiny Settings</h2>
 <mat-dialog-content class="mat-typography">
 
-    <form class="flex flex-col p-8 pb-0 overflow-hidden">
-        <div class="flex flex-col gt-xs:flex-row">
-            <mat-form-field class="flex-auto gt-xs:pr-3">
-                <mat-label>Sort By</mat-label>
-                <mat-select [value]="'status'">
-                    <mat-option  value="status">Status</mat-option>
-                    <mat-option value="name" disabled>Name</mat-option>
-                    <mat-option value="label" disabled>Label</mat-option>
+    <div class="flex flex-col p-8 pb-0 overflow-hidden">
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Dark Mode</mat-label>
+                <mat-select [(ngModel)]="theme">
+                    <mat-option value="system">System</mat-option>
+                    <mat-option value="dark">Dark</mat-option>
+                    <mat-option value="light">Light</mat-option>
                 </mat-select>
             </mat-form-field>
         </div>
 
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Display Title</mat-label>
+                <mat-select [(ngModel)]="dashboardDisplay">
+                    <mat-option value="name">Name</mat-option>
+                    <mat-option value="serial_id">Serial ID</mat-option>
+                    <mat-option value="uuid">UUID</mat-option>
+                    <mat-option value="label">Label</mat-option>
+                </mat-select>
+            </mat-form-field>
+
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pl-3">
+                <mat-label>Sort By</mat-label>
+                <mat-select [(ngModel)]="dashboardSort">
+                    <mat-option value="status">Status</mat-option>
+                    <mat-option value="title">Title</mat-option>
+                    <mat-option value="age">Age</mat-option>
+                </mat-select>
+            </mat-form-field>
+        </div>
+
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Temperature Display Unit</mat-label>
+                <mat-select [(ngModel)]="temperatureUnit">
+                    <mat-option value="celsius">Celsius</mat-option>
+                    <mat-option value="fahrenheit">Fahrenheit</mat-option>
+                </mat-select>
+            </mat-form-field>
+
+        </div>
+
         <div class="flex">
             <mat-tab-group mat-align-tabs="start">
                 <mat-tab label="Ata">
 
-                    <div class="flex flex-col mt-5 gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Critical Error Threshold</mat-label>
-                            <input matInput [value]="'10%'">
+                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
+                            <input disabled matInput [value]="'10%'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Critical Warning Threshold</mat-label>
-                            <input matInput>
+                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
+                            <input disabled matInput>
                         </mat-form-field>
                     </div>
 
-                    <div class="flex flex-col gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Error Threshold</mat-label>
-                            <input matInput [value]="'20%'">
+                            <mat-label class="text-hint">Error Threshold</mat-label>
+                            <input disabled matInput [value]="'20%'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Warning Threshold</mat-label>
-                            <input matInput [value]="'10%'">
+                            <mat-label class="text-hint">Warning Threshold</mat-label>
+                            <input disabled matInput [value]="'10%'">
                         </mat-form-field>
                     </div>
 
                 </mat-tab>
                 <mat-tab label="NVMe">
 
-                    <div class="flex flex-col mt-5 gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Critical Error Threshold</mat-label>
-                            <input matInput [value]="'enabled'">
+                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
+                            <input disabled matInput [value]="'enabled'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Critical Warning Threshold</mat-label>
-                            <input matInput>
+                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
+                            <input disabled matInput>
                         </mat-form-field>
                     </div>
 
                 </mat-tab>
                 <mat-tab label="SCSI">
-                    <div class="flex flex-col mt-5 gt-md:flex-row">
+                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
                         <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label>Critical Error Threshold</mat-label>
-                            <input matInput [value]="'enabled'">
+                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
+                            <input disabled matInput [value]="'enabled'">
                         </mat-form-field>
                         <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label>Critical Warning Threshold</mat-label>
-                            <input matInput>
+                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
+                            <input disabled matInput>
                         </mat-form-field>
                     </div>
                 </mat-tab>
             </mat-tab-group>
         </div>
-    </form>
+    </div>
 
 </mat-dialog-content>
 <mat-dialog-actions align="end">
     <button mat-button mat-dialog-close>Cancel</button>
-    <button mat-button matTooltip="not yet implemented" [mat-dialog-close]="true" cdkFocusInitial>Save</button>
+    <button mat-button mat-dialog-close (click)="saveSettings()" cdkFocusInitial>Save</button>
 </mat-dialog-actions>

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts

@@ -1,4 +1,8 @@
 import { Component, OnInit } from '@angular/core';
+import {AppConfig} from 'app/core/config/app.config';
+import { TreoConfigService } from '@treo/services/config';
+import {Subject} from "rxjs";
+import {takeUntil} from "rxjs/operators";
 
 @Component({
   selector: 'app-dashboard-settings',
@@ -7,11 +11,51 @@ import { Component, OnInit } from '@angular/core';
 })
 export class DashboardSettingsComponent implements OnInit {
 
-  constructor() { }
+    dashboardDisplay: string;
+    dashboardSort: string;
+    temperatureUnit: string;
+    theme: string;
+
+    // Private
+    private _unsubscribeAll: Subject<any>;
+
+    constructor(
+        private _configService: TreoConfigService,
+    ) {
+        // Set the private defaults
+        this._unsubscribeAll = new Subject();
+    }
 
   ngOnInit(): void {
+      // Subscribe to config changes
+      this._configService.config$
+          .pipe(takeUntil(this._unsubscribeAll))
+          .subscribe((config: AppConfig) => {
+
+              // Store the config
+              this.dashboardDisplay = config.dashboardDisplay;
+              this.dashboardSort = config.dashboardSort;
+              this.temperatureUnit = config.temperatureUnit;
+              this.theme = config.theme;
+
+          });
+
   }
-    formatLabel(value: number) {
+
+  saveSettings(): void {
+
+
+        const newSettings = {
+            dashboardDisplay: this.dashboardDisplay,
+            dashboardSort: this.dashboardSort,
+            temperatureUnit: this.temperatureUnit,
+            theme: this.theme
+        }
+        this._configService.config = newSettings
+        console.log(`Saved Settings: ${JSON.stringify(newSettings)}`)
+  }
+
+    formatLabel(value: number): number {
         return value;
     }
 }

webapp/frontend/src/app/layout/layout.component.ts

@@ -23,6 +23,7 @@ export class LayoutComponent implements OnInit, OnDestroy
 
     // Private
     private _unsubscribeAll: Subject<any>;
+    private systemPrefersDark: boolean;
 
     /**
      * Constructor
@@ -43,6 +44,9 @@ export class LayoutComponent implements OnInit, OnDestroy
     {
         // Set the private defaults
         this._unsubscribeAll = new Subject();
+
+        this.systemPrefersDark = window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches;
+
     }
 
     // -----------------------------------------------------------------------------------------------------
@@ -66,7 +70,7 @@ export class LayoutComponent implements OnInit, OnDestroy
                 this.theme = config.theme;
 
                 // Update the selected theme class name on body
-                const themeName = 'treo-theme-' + config.theme;
+                const themeName = 'treo-theme-' + this.determineTheme(config);
                 this._document.body.classList.forEach((className) => {
                     if ( className.startsWith('treo-theme-') && className !== themeName )
                     {
@@ -105,6 +109,17 @@ export class LayoutComponent implements OnInit, OnDestroy
     // @ Private methods
     // -----------------------------------------------------------------------------------------------------
 
+    /**
+     * Checks if theme should be set to dark based on config & system settings
+     */
+    private determineTheme(config:AppConfig): string {
+        if (config.theme === 'system') {
+            return this.systemPrefersDark ? 'dark' : 'light'
+        } else {
+            return config.theme
+        }
+    }
+
     /**
      * Update the selected layout
      */

webapp/frontend/src/app/layout/layouts/horizontal/material/material.component.html

@@ -39,7 +39,7 @@
 
                 <!-- Spacer -->
                 <div class="spacer"></div>
-
+                <code>{{appVersion}}</code>
 
                 <!-- Shortcuts -->
 <!--                <shortcuts [shortcuts]="data.shortcuts"></shortcuts>-->
@@ -48,6 +48,7 @@
 <!--                <notifications [notifications]="data.notifications"></notifications>-->
 
 
+
             </div>
 
 

webapp/frontend/src/app/layout/layouts/horizontal/material/material.component.ts

@@ -4,6 +4,7 @@ import { Subject } from 'rxjs';
 import { takeUntil } from 'rxjs/operators';
 import { TreoMediaWatcherService } from '@treo/services/media-watcher';
 import { TreoNavigationService } from '@treo/components/navigation';
+import {versionInfo} from 'environments/versions';
 
 @Component({
     selector     : 'material-layout',
@@ -13,6 +14,7 @@ import { TreoNavigationService } from '@treo/components/navigation';
 })
 export class MaterialLayoutComponent implements OnInit, OnDestroy
 {
+    appVersion: string;
     data: any;
     isScreenSmall: boolean;
 
@@ -46,6 +48,8 @@ export class MaterialLayoutComponent implements OnInit, OnDestroy
         // Set the defaults
         this.fixedHeader = false;
         this.fixedFooter = false;
+
+        this.appVersion = versionInfo.version
     }
 
     // -----------------------------------------------------------------------------------------------------

webapp/frontend/src/app/modules/dashboard/dashboard.component.html

@@ -47,71 +47,15 @@
                 </div>
             </div>
 
-            <div class="flex flex-wrap w-full">
-                <div *ngFor="let summary of data.data.summary | keyvalue" class="flex gt-sm:w-1/2 min-w-80 p-4">
-                    <div [ngClass]="{ 'border-green': summary.value.device.device_status == 0 && summary.value.smart,
-                                    'border-red': summary.value.device.device_status != 0 }"
-                         class="relative flex flex-col flex-auto p-6 pr-3 pb-3 bg-card rounded border-l-4 shadow-md overflow-hidden">
-                        <div class="absolute bottom-0 right-0 w-24 h-24 -m-6">
-                            <mat-icon class="icon-size-96 opacity-12 text-green"
-                                      *ngIf="summary.value.device.device_status == 0 && summary.value.smart"
-                                      [svgIcon]="'heroicons_outline:check-circle'"></mat-icon>
-                            <mat-icon class="icon-size-96 opacity-12 text-red"
-                                      *ngIf="summary.value.device.device_status != 0"
-                                      [svgIcon]="'heroicons_outline:exclamation-circle'"></mat-icon>
-                            <mat-icon class="icon-size-96 opacity-12 text-yellow"
-                                      *ngIf="!summary.value.smart"
-                                      [svgIcon]="'heroicons_outline:question-mark-circle'"></mat-icon>
-                        </div>
-                        <div class="flex items-center">
-                            <div class="flex flex-col">
-                                <a [routerLink]="'/device/'+ summary.value.device.wwn"
-                                   class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceTitle(summary.value.device)}}</a>
-                                <div [ngClass]="classDeviceLastUpdatedOn(summary.value)" class="font-medium text-sm" *ngIf="summary.value.smart">
-                                    Last Updated on {{summary.value.smart.collector_date | date:'MMMM dd, yyyy - HH:mm' }}
-                                </div>
-                            </div>
-                            <div class="ml-auto" *ngIf="summary.value.device">
-                                <button mat-icon-button
-                                        [matMenuTriggerFor]="previousStatementMenu">
-                                    <mat-icon [svgIcon]="'more_vert'"></mat-icon>
-                                </button>
-                                <mat-menu #previousStatementMenu="matMenu">
-                                    <a mat-menu-item [routerLink]="'/device/'+ summary.value.device.wwn">
-                                    <span class="flex items-center">
-                                        <mat-icon class="icon-size-20 mr-3"
-                                                  [svgIcon]="'payment'"></mat-icon>
-                                        <span>View Details</span>
-                                    </span>
-                                    </a>
-                                </mat-menu>
-                            </div>
-                        </div>
-                        <div class="flex flex-row flex-wrap mt-4 -mx-6">
-                            <div class="flex flex-col mx-6 my-3 xs:w-full">
-                                <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Status</div>
-                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="summary.value.smart?.collector_date; else unknownStatus">{{ deviceStatusString(summary.value.device.device_status) | titlecase}}</div>
-                                <ng-template #unknownStatus><div class="mt-2 font-medium text-3xl leading-none">No Data</div></ng-template>
-                            </div>
-                            <div class="flex flex-col mx-6 my-3 xs:w-full">
-                                <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Temperature</div>
-                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="summary.value.smart?.collector_date; else unknownTemp">{{ summary.value.smart?.temp }}°C</div>
-                                <ng-template #unknownTemp><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
-                            </div>
-                            <div class="flex flex-col mx-6 my-3 xs:w-full">
-                                <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Capacity</div>
-                                <div class="mt-2 font-medium text-3xl leading-none">{{ summary.value.device.capacity | fileSize}}</div>
-                            </div>
-                            <div class="flex flex-col mx-6 my-3 xs:w-full">
-                                <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Powered On</div>
-                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="summary.value.smart?.power_on_hours; else unknownPoweredOn">{{ humanizeDuration(summary.value.smart?.power_on_hours * 60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] }) }}</div>
-                                <ng-template #unknownPoweredOn><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
-                            </div>
-                        </div>
-                    </div>
+
+            <div class="flex flex-wrap w-full" *ngFor="let hostId of hostGroups | keyvalue">
+                <h3 class="ml-4" *ngIf="hostId.key">{{hostId.key}}</h3>
+                <div class="flex flex-wrap w-full">
+                    <app-dashboard-device (deviceDeleted)="onDeviceDeleted($event)" class="flex gt-sm:w-1/2 min-w-80 p-4" *ngFor="let deviceSummary of (deviceSummariesForHostGroup(hostId.value) | deviceSort:config.dashboardSort:config.dashboardDisplay )" [deviceWWN]="deviceSummary.device.wwn" [deviceSummary]="deviceSummary"></app-dashboard-device>
                 </div>
             </div>
 
+
             <!-- Drive Temperatures -->
             <div class="flex flex-auto w-full min-w-80 h-90 p-4">
                 <div class="flex flex-col flex-auto bg-card shadow-md rounded overflow-hidden">
@@ -123,22 +67,22 @@
                             </div>
                             <div>
                                 <button class="h-8 min-h-8 px-2"
-                                        matTooltip="not yet implemented"
                                         mat-button
                                         [matMenuTriggerFor]="tempRangeMenu">
-                                    <span class="font-medium text-sm text-hint">1 week</span>
+                                    <span class="font-medium text-sm text-hint">{{tempDurationKey}}</span>
                                 </button>
                                 <mat-menu #tempRangeMenu="matMenu">
-                                    <button mat-menu-item>1 month</button>
-                                    <button mat-menu-item>12 months</button>
-                                    <button mat-menu-item>all time</button>
+                                    <button (click)="changeSummaryTempDuration('forever')" mat-menu-item>forever</button>
+                                    <button (click)="changeSummaryTempDuration('year')" mat-menu-item>year</button>
+                                    <button (click)="changeSummaryTempDuration('month')" mat-menu-item>month</button>
+                                    <button (click)="changeSummaryTempDuration('week')" mat-menu-item>week</button>
                                 </mat-menu>
                             </div>
                         </div>
 
                     </div>
                     <div class="flex flex-col flex-auto">
-                        <apx-chart *ngIf="temperatureOptions" class="flex-auto w-full h-full"
+                        <apx-chart #tempChart *ngIf="temperatureOptions" class="flex-auto w-full h-full"
                                    [chart]="temperatureOptions.chart"
                                    [colors]="temperatureOptions.colors"
                                    [fill]="temperatureOptions.fill"
@@ -167,6 +111,4 @@
         <code>scrutiny-collector-metrics run</code>
 
     </div>
-
-
 </ng-template>

webapp/frontend/src/app/modules/dashboard/dashboard.component.ts

@@ -3,12 +3,15 @@ import { MatSort } from '@angular/material/sort';
 import { MatTableDataSource } from '@angular/material/table';
 import { Subject } from 'rxjs';
 import { takeUntil } from 'rxjs/operators';
-import { ApexOptions } from 'ng-apexcharts';
+import {ApexOptions, ChartComponent} from 'ng-apexcharts';
 import { DashboardService } from 'app/modules/dashboard/dashboard.service';
-import * as moment from "moment";
 import {MatDialog} from '@angular/material/dialog';
 import { DashboardSettingsComponent } from 'app/layout/common/dashboard-settings/dashboard-settings.component';
-import  humanizeDuration from 'humanize-duration'
+import {AppConfig} from "app/core/config/app.config";
+import {TreoConfigService} from "@treo/services/config";
+import {Router} from "@angular/router";
+import {TemperaturePipe} from "app/shared/temperature.pipe";
+import {DeviceTitlePipe} from "app/shared/device-title.pipe";
 
 @Component({
     selector       : 'example',
@@ -20,10 +23,14 @@ import  humanizeDuration from 'humanize-duration'
 export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
 {
     data: any;
+    hostGroups: { [hostId: string]: string[] } = {}
     temperatureOptions: ApexOptions;
+    tempDurationKey: string = "forever"
+    config: AppConfig;
 
     // Private
     private _unsubscribeAll: Subject<any>;
+    @ViewChild("tempChart", { static: false }) tempChart: ChartComponent;
 
     /**
      * Constructor
@@ -32,7 +39,9 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
      */
     constructor(
         private _smartService: DashboardService,
-        public dialog: MatDialog
+        private _configService: TreoConfigService,
+        public dialog: MatDialog,
+        private router: Router,
     )
     {
         // Set the private defaults
@@ -49,6 +58,28 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
      */
     ngOnInit(): void
     {
+
+        // Subscribe to config changes
+        this._configService.config$
+            .pipe(takeUntil(this._unsubscribeAll))
+            .subscribe((config: AppConfig) => {
+
+                //check if the old config and the new config do not match.
+                let oldConfig = JSON.stringify(this.config)
+                let newConfig = JSON.stringify(config)
+
+                if(oldConfig != newConfig){
+                    console.log(`Configuration updated: ${newConfig} vs ${oldConfig}`)
+                    // Store the config
+                    this.config = config;
+
+                    if(oldConfig){
+                        console.log("reloading component...")
+                        this.refreshComponent()
+                    }
+                }
+            });
+
         // Get the data
         this._smartService.data$
             .pipe(takeUntil(this._unsubscribeAll))
@@ -57,6 +88,15 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
                 // Store the data
                 this.data = data;
 
+                //generate group data.
+                for(let wwn in this.data.data.summary){
+                    let hostid = this.data.data.summary[wwn].device.host_id
+                    let hostDeviceList = this.hostGroups[hostid] || []
+                    hostDeviceList.push(wwn)
+                    this.hostGroups[hostid] = hostDeviceList
+                }
+                console.log(this.hostGroups)
+
                 // Prepare the chart data
                 this._prepareChartData();
             });
@@ -81,6 +121,14 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
     // -----------------------------------------------------------------------------------------------------
     // @ Private methods
     // -----------------------------------------------------------------------------------------------------
+    private refreshComponent(){
+
+        let currentUrl = this.router.url;
+        this.router.routeReuseStrategy.shouldReuseRoute = () => false;
+        this.router.onSameUrlNavigation = 'reload';
+        this.router.navigate([currentUrl]);
+    }
+
     private _deviceDataTemperatureSeries() {
         var deviceTemperatureSeries = []
 
@@ -91,8 +139,11 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
             if (!deviceSummary.temp_history){
                 continue
             }
+
+            let deviceName = DeviceTitlePipe.deviceTitleWithFallback(deviceSummary.device, this.config.dashboardDisplay)
+
             var deviceSeriesMetadata = {
-                name: `/dev/${deviceSummary.device.device_name}`,
+                name: deviceName,
                 data: []
             }
 
@@ -100,7 +151,7 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
                 let newDate = new Date(tempHistory.date);
                 deviceSeriesMetadata.data.push({
                     x: newDate,
-                    y: tempHistory.temp
+                    y: TemperaturePipe.formatTemperature(tempHistory.temp, this.config.temperatureUnit, false)
                 })
             }
             deviceTemperatureSeries.push(deviceSeriesMetadata)
@@ -149,8 +200,9 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
                     format: 'MMM dd, yyyy HH:mm:ss'
                 },
                 y    : {
+
                     formatter: (value) => {
-                        return value + '°C';
+                        return TemperaturePipe.formatTemperature(value, this.config.temperatureUnit, true) as string;
                     }
                 }
             },
@@ -164,6 +216,16 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
     // @ Public methods
     // -----------------------------------------------------------------------------------------------------
 
+    deviceSummariesForHostGroup(hostGroupWWNs: string[]) {
+        let deviceSummaries = []
+        for(let wwn of hostGroupWWNs){
+            if(this.data.data.summary[wwn]){
+                deviceSummaries.push(this.data.data.summary[wwn])
+            }
+        }
+        return deviceSummaries
+    }
+
     openDialog() {
         const dialogRef = this.dialog.open(DashboardSettingsComponent);
 
@@ -172,48 +234,33 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
         });
     }
 
-    deviceTitle(disk){
-        let title = []
-
-        if (disk.host_id) title.push(disk.host_id)
-
-        title.push(`/dev/${disk.device_name}`)
-
-        if (disk.device_type && disk.device_type != 'scsi' && disk.device_type != 'ata'){
-            title.push(disk.device_type)
-        }
-
-        title.push(disk.model_name)
-
-        return title.join(' - ')
+    onDeviceDeleted(wwn: string) {
+        delete this.data.data.summary[wwn] // remove the device from the summary list.
     }
 
-    deviceStatusString(deviceStatus){
-        if(deviceStatus == 0){
-            return "passed"
-        } else {
-            return "failed"
-        }
-    }
+    /*
 
-    classDeviceLastUpdatedOn(deviceSummary){
-        if (deviceSummary.device.device_status !== 0) {
-            return 'text-red' // if the device has failed, always highlight in red
-        } else if(deviceSummary.device.device_status === 0 && deviceSummary.smart){
-            if(moment().subtract(14, 'd').isBefore(deviceSummary.smart.collector_date)){
-                // this device was updated in the last 2 weeks.
-                return 'text-green'
-            } else if(moment().subtract(1, 'm').isBefore(deviceSummary.smart.collector_date)){
-                // this device was updated in the last month
-                return 'text-yellow'
-            } else{
-                // last updated more than a month ago.
-                return 'text-red'
-            }
+    DURATION_KEY_WEEK    = "week"
+	DURATION_KEY_MONTH   = "month"
+	DURATION_KEY_YEAR    = "year"
+	DURATION_KEY_FOREVER = "forever"
+     */
 
-        } else {
-            return ''
-        }
+    changeSummaryTempDuration(durationKey: string){
+        this.tempDurationKey = durationKey
+
+        this._smartService.getSummaryTempData(durationKey)
+            .subscribe((data) => {
+
+                // given a list of device temp history, override the data in the "summary" object.
+                for(const wwn in this.data.data.summary) {
+                    // console.log(`Updating ${wwn}, length: ${this.data.data.summary[wwn].temp_history.length}`)
+                    this.data.data.summary[wwn].temp_history = data.data.temp_history[wwn] || []
+                }
+
+                // Prepare the chart series data
+                this.tempChart.updateSeries(this._deviceDataTemperatureSeries())
+            });
     }
 
     /**
@@ -227,6 +274,4 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
         return item.id || index;
     }
 
-    readonly humanizeDuration = humanizeDuration;
-
 }

webapp/frontend/src/app/modules/dashboard/dashboard.module.ts

@@ -13,12 +13,13 @@ import { MatTableModule } from '@angular/material/table';
 import { NgApexchartsModule } from 'ng-apexcharts';
 import { MatTooltipModule } from '@angular/material/tooltip'
 import { DashboardSettingsModule } from "app/layout/common/dashboard-settings/dashboard-settings.module";
+import { DashboardDeviceModule } from "app/layout/common/dashboard-device/dashboard-device.module";
 
 @NgModule({
     declarations: [
         DashboardComponent
     ],
-    imports     : [
+    imports: [
         RouterModule.forChild(dashboardRoutes),
         MatButtonModule,
         MatDividerModule,
@@ -30,7 +31,8 @@ import { DashboardSettingsModule } from "app/layout/common/dashboard-settings/da
         MatTableModule,
         NgApexchartsModule,
         SharedModule,
-        DashboardSettingsModule
+        DashboardSettingsModule,
+        DashboardDeviceModule
     ]
 })
 export class DashboardModule

webapp/frontend/src/app/modules/dashboard/dashboard.resolvers.ts

@@ -31,6 +31,6 @@ export class DashboardResolver implements Resolve<any>
      */
     resolve(route: ActivatedRouteSnapshot, state: RouterStateSnapshot): Observable<any>
     {
-        return this._dashboardService.getData();
+        return this._dashboardService.getSummaryData();
     }
 }

webapp/frontend/src/app/modules/dashboard/dashboard.service.ts

@@ -44,7 +44,7 @@ export class DashboardService
     /**
      * Get data
      */
-    getData(): Observable<any>
+    getSummaryData(): Observable<any>
     {
         return this._httpClient.get(getBasePath() + '/api/summary').pipe(
             tap((response: any) => {
@@ -52,4 +52,14 @@ export class DashboardService
             })
         );
     }
+
+    getSummaryTempData(durationKey: string): Observable<any>
+    {
+        let params = {}
+        if(durationKey){
+            params["duration_key"] = durationKey
+        }
+
+        return this._httpClient.get(getBasePath() + '/api/summary/temp', {params: params});
+    }
 }

webapp/frontend/src/app/modules/detail/detail.component.html

@@ -4,7 +4,7 @@
 
         <div class="flex items-center justify-between w-full my-4 px-4 xs:pr-0">
             <div class="mr-6">
-                <h2 class="m-0">Drive Details</h2>
+                <h2 class="m-0">Drive Details - {{device | deviceTitle:config.dashboardDisplay}} </h2>
                 <div class="text-secondary tracking-tight">Dive into S.M.A.R.T data</div>
             </div>
             <!-- Action buttons -->
@@ -51,11 +51,8 @@
 
         <!-- Card -->
         <div class="flex flex-auto w-1/4 p-4 lt-md:w-full">
-            <treo-card class="flex flex-auto p-4 pt-6 flex-col flex-auto filter-list">
-                <div class="flex items-center justify-between">
-                    <div class="text-2xl font-semibold leading-tight">/dev/{{device?.device_name}}</div>
-                </div>
-                <div class="flex flex-col my-2 grid grid-cols-2">
+            <treo-card class="flex flex-auto p-4 flex-col flex-auto filter-list">
+                <div class="flex flex-col grid grid-cols-2">
                     <div *ngIf="device" class="my-2 col-span-2 lt-md:col-span-1">
                         <div>
                             <span class="inline-flex items-center font-bold text-xs px-2 py-2px rounded-full tracking-wide uppercase"
@@ -64,7 +61,7 @@
                                 <span class="w-2 h-2 rounded-full mr-2"
                                       [ngClass]="{'bg-red': device?.device_status != 0,
                                   'bg-green': device?.device_status == 0}"></span>
-                                <span class="pr-2px leading-relaxed whitespace-no-wrap">{{device?.device_status == 0 ? 'passed' : 'failed'}}</span>
+                                <span class="pr-2px leading-relaxed whitespace-no-wrap">{{device?.device_status | deviceStatus}}</span>
                             </span>
                         </div>
                         <div class="text-secondary text-md">Status</div>
@@ -74,6 +71,16 @@
                         <div>{{device?.host_id}}</div>
                         <div class="text-secondary text-md">Host ID</div>
                     </div>
+
+                    <div *ngIf="device?.device_uuid" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.device_uuid}}</div>
+                        <div class="text-secondary text-md">Device UUID</div>
+                    </div>
+                    <div *ngIf="device?.device_label" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.device_label}}</div>
+                        <div class="text-secondary text-md">Device Label</div>
+                    </div>
+
                     <div *ngIf="device?.device_type && device?.device_type != 'ata' && device?.device_type != 'scsi'" class="my-2 col-span-2 lt-md:col-span-1">
                         <div>{{device?.device_type | uppercase}}</div>
                         <div class="text-secondary text-md">Device Type</div>
@@ -119,7 +126,7 @@
                         <div class="text-secondary text-md">Powered On</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{smart_results[0]?.temp}}°C</div>
+                        <div>{{smart_results[0]?.temp | temperature:config.temperatureUnit:true}}</div>
                         <div class="text-secondary text-md">Temperature</div>
                     </div>
                 </div>