(v0.5.0) Automated packaging of release by Packagr

Ensure logger creation is consistent between Web and Collector
Create logger in main, pass down to downstream functions (like gin)
In debug mode, print a copy of AppConfig
Better debugging for logger.

.github/workflows/build.yaml

@@ -1,85 +0,0 @@
-name: CI
-# This workflow is triggered on pushes & pull requests
-on: [pull_request]
-
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    container: techknowlogick/xgo:go-1.17.x
-
-    # Service containers to run with `build` (Required for end-to-end testing)
-    services:
-      influxdb:
-        image: influxdb:2.2
-        env:
-          DOCKER_INFLUXDB_INIT_MODE: setup
-          DOCKER_INFLUXDB_INIT_USERNAME: admin
-          DOCKER_INFLUXDB_INIT_PASSWORD: password12345
-          DOCKER_INFLUXDB_INIT_ORG: scrutiny
-          DOCKER_INFLUXDB_INIT_BUCKET: metrics
-          DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token
-        ports:
-          - 8086:8086
-    env:
-      PROJECT_PATH: /go/src/github.com/analogj/scrutiny
-      CGO_ENABLED: 1
-    steps:
-      - name: Git
-        run: |
-          apt-get update && apt-get install -y software-properties-common
-          add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git
-          git --version
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Test
-        run: |
-          mkdir -p $(dirname "$PROJECT_PATH")
-          cp -a $GITHUB_WORKSPACE $PROJECT_PATH
-          cd $PROJECT_PATH
-
-          go mod vendor
-          go test -race -coverprofile=coverage.txt -covermode=atomic -v -tags "static" $(go list ./... | grep -v /vendor/)
-      - name: Generate coverage report
-        uses: codecov/codecov-action@v2
-        with:
-          files: ${{ env.PROJECT_PATH }}/coverage.txt
-          flags: unittests
-          fail_ci_if_error: true
-          verbose: true
-      - name: Build Binaries
-        run: |
-
-          cd $PROJECT_PATH
-          make all
-
-      - name: Archive
-        uses: actions/upload-artifact@v2
-        with:
-          name: binaries.zip
-          path: |
-            /build/scrutiny-web-linux-amd64
-            /build/scrutiny-collector-metrics-linux-amd64
-            /build/scrutiny-web-linux-arm64
-            /build/scrutiny-collector-metrics-linux-arm64
-            /build/scrutiny-web-linux-arm-5
-            /build/scrutiny-collector-metrics-linux-arm-5
-            /build/scrutiny-web-linux-arm-6
-            /build/scrutiny-collector-metrics-linux-arm-6
-            /build/scrutiny-web-linux-arm-7
-            /build/scrutiny-collector-metrics-linux-arm-7
-            /build/scrutiny-web-windows-4.0-amd64.exe
-            /build/scrutiny-collector-metrics-windows-4.0-amd64.exe
-#            /build/scrutiny-web-darwin-arm64
-#            /build/scrutiny-collector-metrics-darwin-arm64
-#            /build/scrutiny-web-darwin-amd64
-#            /build/scrutiny-collector-metrics-darwin-amd64
-#            /build/scrutiny-web-freebsd-amd64
-#            /build/scrutiny-collector-metrics-freebsd-amd64
-      - uses: codecov/codecov-action@v2
-        with:
-          file: ${{ env.PROJECT_PATH }}/coverage.txt
-          flags: unittests
-          fail_ci_if_error: false
-
-

.github/workflows/ci.yaml

@@ -0,0 +1,114 @@
+name: CI
+# This workflow is triggered on pushes & pull requests
+on: [pull_request]
+
+jobs:
+  test-frontend:
+    name: Test Frontend
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Test Frontend
+        run: |
+          make binary-frontend-test-coverage
+      - name: Upload coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: ${{ github.workspace }}/webapp/frontend/coverage/lcov.info
+          retention-days: 1
+  test-backend:
+    name: Test Backend
+    runs-on: ubuntu-latest
+    container: ghcr.io/packagrio/packagr:latest-golang
+    # Service containers to run with `build` (Required for end-to-end testing)
+    services:
+      influxdb:
+        image: influxdb:2.2
+        env:
+          DOCKER_INFLUXDB_INIT_MODE: setup
+          DOCKER_INFLUXDB_INIT_USERNAME: admin
+          DOCKER_INFLUXDB_INIT_PASSWORD: password12345
+          DOCKER_INFLUXDB_INIT_ORG: scrutiny
+          DOCKER_INFLUXDB_INIT_BUCKET: metrics
+          DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token
+        ports:
+          - 8086:8086
+    env:
+      STATIC: true
+    steps:
+      - name: Git
+        run: |
+          apt-get update && apt-get install -y software-properties-common
+          add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git
+          git --version
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Test Backend
+        run: |
+          make binary-clean binary-test-coverage
+      - name: Upload coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: ${{ github.workspace }}/coverage.txt
+          retention-days: 1
+  test-coverage:
+    name: Test Coverage Upload
+    needs:
+      - test-backend
+      - test-frontend
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Download coverage reports
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage
+      - name: Upload coverage reports
+        uses: codecov/codecov-action@v2
+        with:
+          files: ${{ github.workspace }}/coverage.txt,${{ github.workspace }}/lcov.info
+          flags: unittests
+          fail_ci_if_error: true
+          verbose: true
+
+  build:
+    name: Build ${{ matrix.cfg.goos }}/${{ matrix.cfg.goarch }}
+    runs-on: ${{ matrix.cfg.on }}
+    env:
+      GOOS: ${{ matrix.cfg.goos }}
+      GOARCH: ${{ matrix.cfg.goarch }}
+      GOARM: ${{ matrix.cfg.goarm }}
+      STATIC: true
+    strategy:
+      matrix:
+        cfg:
+          - { on: ubuntu-latest,  goos: linux,      goarch: amd64 }
+          - { on: ubuntu-latest,  goos: linux,      goarch: arm,    goarm: 5 }
+          - { on: ubuntu-latest,  goos: linux,      goarch: arm,    goarm: 6 }
+          - { on: ubuntu-latest,  goos: linux,      goarch: arm,    goarm: 7 }
+          - { on: ubuntu-latest,  goos: linux,      goarch: arm64 }
+          - { on: macos-latest,   goos: darwin,     goarch: amd64 }
+          - { on: macos-latest,   goos: darwin,     goarch: arm64 }
+          - { on: macos-latest,   goos: freebsd,    goarch: amd64 }
+          - { on: windows-latest, goos: windows,    goarch: amd64 }
+          - { on: windows-latest, goos: windows,    goarch: arm64 }
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '^1.18.3'
+      - name: Build Binaries
+        run: |
+          make binary-clean binary-all
+      - name: Archive
+        uses: actions/upload-artifact@v2
+        with:
+          name: binaries.zip
+          path: |
+            scrutiny-web-*
+            scrutiny-collector-metrics-*

.github/workflows/docker-build.yaml

@@ -81,7 +81,7 @@ jobs:
           options: -v ${{ github.workspace }}:/work
           run: |
             cd /work
-            make frontend && echo "print contents of /work/dist" && ls -alt /work/dist
+            make binary-frontend && echo "print contents of /work/dist" && ls -alt /work/dist
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -141,7 +141,7 @@ jobs:
           options: -v ${{ github.workspace }}:/work
           run: |
             cd /work
-            make frontend && echo "print contents of /work/dist" && ls -alt /work/dist
+            make binary-frontend && echo "print contents of /work/dist" && ls -alt /work/dist
 
 
       - name: Set up QEMU

.github/workflows/docker-nightly.yaml

@@ -26,7 +26,7 @@ jobs:
           options: -v ${{ github.workspace }}:/work
           run: |
             cd /work
-            make frontend && echo "print contents of /work/dist" && ls -alt /work/dist
+            make binary-frontend && echo "print contents of /work/dist" && ls -alt /work/dist
 
 
       - name: Set up QEMU

.github/workflows/release-freebsd.yaml

@@ -1,83 +0,0 @@
-# compiles FreeBSD artifacts and attaches them to build
-name: Release FreeBSD
-
-on:
-  release:
-    # Only use the types keyword to narrow down the activity types that will trigger your workflow.
-    types: [published]
-  workflow_dispatch:
-    inputs:
-      tag_name:
-        description: 'tag to build artifacts for'
-        required: true
-        default: 'v0.0.0'
-jobs:
-
-  release-freebsd:
-    name: Release FreeBSD
-    runs-on: macos-10.15
-    env:
-      PROJECT_PATH: /go/src/github.com/analogj/scrutiny
-      GOPATH: /go
-      GOOS: freebsd
-      GOARCH: amd64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          ref: ${{github.event.release.tag_name || github.event.inputs.tag_name }}
-      - name: Build Binaries
-        uses: vmactions/freebsd-vm@v0.1.5
-        with:
-          envs: 'PROJECT_PATH GOPATH GOOS GOARCH'
-          usesh: true
-          #TODO: lock go version using https://www.jeremymorgan.com/tutorials/golang/how-to-install-go-freebsd/
-          prepare: pkg install -y curl go gmake
-          run: |
-            pwd
-            ls -lah
-            whoami
-            freebsd-version
-
-            mkdir -p $(dirname "$PROJECT_PATH")
-            cp -R $GITHUB_WORKSPACE $PROJECT_PATH
-            cd $PROJECT_PATH
-
-            mkdir -p $GITHUB_WORKSPACE/dist
-
-            echo "building web binary (OS = ${GOOS}, ARCH = ${GOARCH})"
-            go build -ldflags "-extldflags=-static -X main.goos=${GOOS} -X main.goarch=${GOARCH}" -o $GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH} -tags "static netgo sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go
-
-            chmod +x "$GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH}"
-            file "$GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH}" || true
-            ldd "$GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH}" || true
-
-            echo "building collector binary (OS = ${GOOS}, ARCH = ${GOARCH})"
-            go build -ldflags "-extldflags=-static -X main.goos=${GOOS} -X main.goarch=${GOARCH}" -o $GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH} -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go
-
-            chmod +x "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}"
-            file "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}" || true
-            ldd "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}" || true
-
-      - name: Release Asset - Collector - freebsd-amd64
-        id: upload-release-asset2
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: './dist/scrutiny-collector-metrics-freebsd-amd64'
-          asset_name: scrutiny-collector-metrics-freebsd-amd64
-          asset_content_type: application/octet-stream
-
-      - name: Release Asset - Web - freebsd-amd64
-        id: upload-release-asset1
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: './dist/scrutiny-web-freebsd-amd64'
-          asset_name: scrutiny-web-freebsd-amd64
-          asset_content_type: application/octet-stream
-

.github/workflows/release-frontend.yaml

@@ -19,11 +19,8 @@ jobs:
         run: "cd webapp/frontend && ./git.version.sh"
       - name: Build Frontend
         run: |
-          cd webapp/frontend
-          npm install -g @angular/cli@9.1.4
-          npm install
-          mkdir -p dist
-          npm run build:prod -- --output-path=dist
+          apt-get update && apt-get install -y make
+          make binary-frontend
           tar -czf scrutiny-web-frontend.tar.gz dist
       - name: Upload Frontend Asset
         id: upload-release-asset3
@@ -32,6 +29,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
         with:
           upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: './webapp/frontend/scrutiny-web-frontend.tar.gz'
+          asset_path: './scrutiny-web-frontend.tar.gz'
           asset_name: scrutiny-web-frontend.tar.gz
           asset_content_type: application/gzip

.github/workflows/release.yaml

@@ -13,10 +13,10 @@ on:
         default: 'webapp/backend/pkg/version/version.go'
 
 jobs:
-  build:
-    name: Build
+  release:
+    name: Create Release Commit
     runs-on: ubuntu-latest
-    container: techknowlogick/xgo:go-1.17.x
+    container: ghcr.io/packagrio/packagr:latest-golang
     # Service containers to run with `build` (Required for end-to-end testing)
     services:
       influxdb:
@@ -31,8 +31,7 @@ jobs:
         ports:
           - 8086:8086
     env:
-      PROJECT_PATH: /go/src/github.com/analogj/scrutiny
-      CGO_ENABLED: 1
+      STATIC: true
     steps:
       - name: Git
         run: |
@@ -53,34 +52,80 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }} # Leave this line unchanged
       - name: Test
         run: |
-          mkdir -p $(dirname "$PROJECT_PATH")
-          cp -a $GITHUB_WORKSPACE $PROJECT_PATH
-          cd $PROJECT_PATH
-
-          go mod vendor
-          go test -v -tags "static" $(go list ./... | grep -v /vendor/)
-
-      - name: Build Binaries
-        run: |
-
-          cd $PROJECT_PATH
-          make all
-
-          # restore modified dir to GH workspace.
-          cp -arf $PROJECT_PATH/. $GITHUB_WORKSPACE/
-
-          # copy all the build artifacts to the GH workspace
-          cp -arf /build/. $GITHUB_WORKSPACE/
-
-      - name: Commit Changes
+          make binary-clean binary-test-coverage
+      - name: Commit Changes Locally
         id: commit
         uses: packagrio/action-releasr-go@master
         env:
-          # This is necessary in order to push a commit to the repo
           GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }} # Leave this line unchanged
         with:
           version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
-      - name: Publish Release
+      - name: Upload workspace
+        uses: actions/upload-artifact@v3
+        with:
+          name: workspace
+          path: ${{ github.workspace }}/**/*
+          retention-days: 1
+
+  build:
+    name: Build ${{ matrix.cfg.goos }}/${{ matrix.cfg.goarch }}${{ matrix.cfg.goarm }}
+    needs: release
+    runs-on: ${{ matrix.cfg.on }}
+    env:
+      GOOS: ${{ matrix.cfg.goos }}
+      GOARCH: ${{ matrix.cfg.goarch }}
+      GOARM: ${{ matrix.cfg.goarm }}
+      STATIC: true
+    strategy:
+      matrix:
+        cfg:
+          - { on: ubuntu-latest,  goos: linux,      goarch: amd64 }
+          - { on: ubuntu-latest,  goos: linux,      goarch: arm,    goarm: 5 }
+          - { on: ubuntu-latest,  goos: linux,      goarch: arm,    goarm: 6 }
+          - { on: ubuntu-latest,  goos: linux,      goarch: arm,    goarm: 7 }
+          - { on: ubuntu-latest,  goos: linux,      goarch: arm64 }
+          - { on: macos-latest,   goos: darwin,     goarch: amd64 }
+          - { on: macos-latest,   goos: darwin,     goarch: arm64 }
+          - { on: macos-latest,   goos: freebsd,    goarch: amd64 }
+          - { on: windows-latest, goos: windows,    goarch: amd64 }
+          - { on: windows-latest, goos: windows,    goarch: arm64 }
+    steps:
+      - name: Download workspace
+        uses: actions/download-artifact@v3
+        with:
+          name: workspace
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18.3' # The Go version to download (if necessary) and use.
+      - name: Build Binaries
+        run: |
+          make binary-clean binary-all
+      - name: Archive
+        uses: actions/upload-artifact@v2
+        with:
+          name: binaries.zip
+          path: |
+            scrutiny-web-*
+            scrutiny-collector-metrics-*
+
+  release-publish:
+    name: Publish Release
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download workspace
+        uses: actions/download-artifact@v3
+        with:
+          name: workspace
+      - name: Download binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: binaries.zip
+      - name: List
+        shell: bash
+        run: |
+          ls -alt
+      - name: Publish Release & Assets
         id: publish
         uses: packagrio/action-publishr-go@master
         env:
@@ -89,15 +134,23 @@ jobs:
         with:
           version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
           upload_assets:
-            scrutiny-web-linux-amd64
+            scrutiny-collector-metrics-darwin-amd64
+            scrutiny-collector-metrics-darwin-arm64
+            scrutiny-collector-metrics-freebsd-amd64
             scrutiny-collector-metrics-linux-amd64
-            scrutiny-web-linux-arm64
-            scrutiny-collector-metrics-linux-arm64
-            scrutiny-web-linux-arm-5
             scrutiny-collector-metrics-linux-arm-5
-            scrutiny-web-linux-arm-6
             scrutiny-collector-metrics-linux-arm-6
-            scrutiny-web-linux-arm-7
             scrutiny-collector-metrics-linux-arm-7
-            scrutiny-web-windows-4.0-amd64.exe
-            scrutiny-collector-metrics-windows-4.0-amd64.exe
+            scrutiny-collector-metrics-linux-arm64
+            scrutiny-collector-metrics-windows-amd64.exe
+            scrutiny-collector-metrics-windows-arm64.exe
+            scrutiny-web-darwin-amd64
+            scrutiny-web-darwin-arm64
+            scrutiny-web-freebsd-amd64
+            scrutiny-web-linux-amd64
+            scrutiny-web-linux-arm-5
+            scrutiny-web-linux-arm-6
+            scrutiny-web-linux-arm-7
+            scrutiny-web-linux-arm64
+            scrutiny-web-windows-amd64.exe
+            scrutiny-web-windows-arm64.exe

.github/workflows/remove-old-artifacts.yaml

@@ -1,19 +0,0 @@
-name: Cleanup Artifacts
-
-on:
-  schedule:
-    # Every day at 1am
-    - cron: '0 1 * * *'
-
-jobs:
-  remove-old-artifacts:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - name: Remove old artifacts
-        uses: c-hive/gha-remove-artifacts@v1
-        with:
-          age: '1 day'
-          skip-tags: true
-          skip-recent: 5

CONTRIBUTING.md

@@ -9,8 +9,9 @@ Depending on the functionality you are adding, you may need to setup a developme
 
 # Modifying the Scrutiny Backend Server (API)
 
-1. install the [Go runtime](https://go.dev/doc/install) (v1.17+)
-2. download the `scrutiny-web-frontend.tar.gz` for the [latest release](https://github.com/AnalogJ/scrutiny/releases/latest). Extract to a folder named `dist`
+1. install the [Go runtime](https://go.dev/doc/install) (v1.18+)
+2. download the `scrutiny-web-frontend.tar.gz` for
+   the [latest release](https://github.com/AnalogJ/scrutiny/releases/latest). Extract to a folder named `dist`
 3. create a `scrutiny.yaml` config file
     ```yaml
     # config file for local development. store as scrutiny.yaml
@@ -62,7 +63,7 @@ The frontend is written in Angular. If you're working on the frontend and can us
 If you're developing a feature that requires changes to the backend and the frontend, or a frontend feature that requires real data, 
 you'll need to follow the steps below:
 
-1. install the [Go runtime](https://go.dev/doc/install) (v1.17+)
+1. install the [Go runtime](https://go.dev/doc/install) (v1.18+)
 2. install [NodeJS](https://nodejs.org/en/download/)
 3. create a `scrutiny.yaml` config file
     ```yaml

Makefile

@@ -1,66 +1,133 @@
-export CGO_ENABLED = 1
+.ONESHELL: # Applies to every targets in the file! .ONESHELL instructs make to invoke a single instance of the shell and provide it with the entire recipe, regardless of how many lines it contains.
+.SHELLFLAGS = -ec
+
+########################################################################################################################
+# Global Env Settings
+########################################################################################################################
 
 GO_WORKSPACE ?= /go/src/github.com/analogj/scrutiny
 
-BINARY=\
-  linux/amd64   \
-  linux/arm-5   \
-  linux/arm-6   \
-  linux/arm-7   \
-  linux/arm64   \
+COLLECTOR_BINARY_NAME = scrutiny-collector-metrics
+WEB_BINARY_NAME = scrutiny-web
+LD_FLAGS =
 
-.ONESHELL: # Applies to every targets in the file! .ONESHELL instructs make to invoke a single instance of the shell and provide it with the entire recipe, regardless of how many lines it contains.
-.PHONY: all $(BINARY)
-all: $(BINARY) windows/amd64
+STATIC_TAGS =
+# enable multiarch docker image builds
+DOCKER_TARGETARCH_BUILD_ARG =
+ifdef TARGETARCH
+DOCKER_TARGETARCH_BUILD_ARG := $(DOCKER_TARGETARCH_BUILD_ARG) --build-arg TARGETARCH=$(TARGETARCH)
+endif
 
-$(BINARY): OS = $(word 1,$(subst /, ,$*))
-$(BINARY): ARCH = $(word 2,$(subst /, ,$*))
-$(BINARY): build/scrutiny-web-%:
-	@echo "building web binary (OS = $(OS), ARCH = $(ARCH))"
-	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-web -tags "static netgo sqlite_omit_load_extension" ${GO_WORKSPACE}/webapp/backend/cmd/scrutiny/
+# enable to build static binaries.
+ifdef STATIC
+export CGO_ENABLED = 0
+LD_FLAGS := $(LD_FLAGS) -extldflags=-static
+STATIC_TAGS := $(STATIC_TAGS) -tags "static netgo"
+endif
+ifdef GOOS
+COLLECTOR_BINARY_NAME := $(COLLECTOR_BINARY_NAME)-$(GOOS)
+WEB_BINARY_NAME := $(WEB_BINARY_NAME)-$(GOOS)
+LD_FLAGS := $(LD_FLAGS) -X main.goos=$(GOOS)
+endif
+ifdef GOARCH
+COLLECTOR_BINARY_NAME := $(COLLECTOR_BINARY_NAME)-$(GOARCH)
+WEB_BINARY_NAME := $(WEB_BINARY_NAME)-$(GOARCH)
+LD_FLAGS := $(LD_FLAGS) -X main.goarch=$(GOARCH)
+endif
+ifdef GOARM
+COLLECTOR_BINARY_NAME := $(COLLECTOR_BINARY_NAME)-$(GOARM)
+WEB_BINARY_NAME := $(WEB_BINARY_NAME)-$(GOARM)
+endif
+ifeq ($(OS),Windows_NT)
+COLLECTOR_BINARY_NAME := $(COLLECTOR_BINARY_NAME).exe
+WEB_BINARY_NAME := $(WEB_BINARY_NAME).exe
+endif
 
-	chmod +x "/build/scrutiny-web-$(OS)-$(ARCH)"
-	file "/build/scrutiny-web-$(OS)-$(ARCH)" || true
-	ldd "/build/scrutiny-web-$(OS)-$(ARCH)" || true
+########################################################################################################################
+# Binary
+########################################################################################################################
+.PHONY: all
+all: binary-all
 
-	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
-	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
-
-	chmod +x "/build/scrutiny-collector-metrics-$(OS)-$(ARCH)"
-	file "/build/scrutiny-collector-metrics-$(OS)-$(ARCH)" || true
-	ldd "/build/scrutiny-collector-metrics-$(OS)-$(ARCH)" || true
-
-windows/amd64: export OS = windows
-windows/amd64: export ARCH = amd64
-windows/amd64:
-	@echo "building web binary (OS = $(OS), ARCH = $(ARCH))"
-	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-web -tags "static netgo sqlite_omit_load_extension" ${GO_WORKSPACE}/webapp/backend/cmd/scrutiny/
-
-	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
-	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
+.PHONY: binary-all
+binary-all: binary-collector binary-web
+	@echo "built binary-collector and binary-web targets"
 
 
-docker-collector:
-	@echo "building collector docker image"
-	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.collector -t analogj/scrutiny-dev:collector .
+.PHONY: binary-clean
+binary-clean:
+	go clean
 
-docker-web:
-	@echo "building web docker image"
-	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.web -t analogj/scrutiny-dev:web .
+.PHONY: binary-dep
+binary-dep:
+	go mod vendor
 
-docker-omnibus:
-	@echo "building omnibus docker image"
-	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile -t analogj/scrutiny-dev:omnibus .
+.PHONY: binary-test
+binary-test: binary-dep
+	go test -v $(STATIC_TAGS) ./...
 
+.PHONY: binary-test-coverage
+binary-test-coverage: binary-dep
+	go test -coverprofile=coverage.txt -covermode=atomic -v $(STATIC_TAGS) ./...
+
+.PHONY: binary-collector
+binary-collector: binary-dep
+	go build -ldflags "$(LD_FLAGS)" -o $(COLLECTOR_BINARY_NAME) $(STATIC_TAGS) ./collector/cmd/collector-metrics/
+ifneq ($(OS),Windows_NT)
+	chmod +x $(COLLECTOR_BINARY_NAME)
+	file $(COLLECTOR_BINARY_NAME) || true
+	ldd $(COLLECTOR_BINARY_NAME) || true
+	./$(COLLECTOR_BINARY_NAME) || true
+endif
+
+.PHONY: binary-web
+binary-web: binary-dep
+	go build -ldflags "$(LD_FLAGS)" -o $(WEB_BINARY_NAME) $(STATIC_TAGS) ./webapp/backend/cmd/scrutiny/
+ifneq ($(OS),Windows_NT)
+	chmod +x $(WEB_BINARY_NAME)
+	file $(WEB_BINARY_NAME) || true
+	ldd $(WEB_BINARY_NAME) || true
+	./$(WEB_BINARY_NAME) || true
+endif
+
+########################################################################################################################
+# Binary
+########################################################################################################################
+
+.PHONY: binary-frontend
 # reduce logging, disable angular-cli analytics for ci environment
-frontend: export NPM_CONFIG_LOGLEVEL = warn
-frontend: export NG_CLI_ANALYTICS = false
-frontend:
+binary-frontend: export NPM_CONFIG_LOGLEVEL = warn
+binary-frontend: export NG_CLI_ANALYTICS = false
+binary-frontend:
 	cd webapp/frontend
 	npm install -g @angular/cli@9.1.4
 	mkdir -p $(CURDIR)/dist
 	npm ci
 	npm run build:prod -- --output-path=$(CURDIR)/dist
 
-# clean:
-# 	rm scrutiny-collector-metrics-* scrutiny-web-*
+.PHONY: binary-frontend-test-coverage
+# reduce logging, disable angular-cli analytics for ci environment
+binary-frontend-test-coverage:
+	cd webapp/frontend
+	npm ci
+	npx ng test --watch=false --browsers=ChromeHeadless --code-coverage
+
+########################################################################################################################
+# Docker
+# NOTE: these docker make targets are only used for local development (not used by Github Actions/CI)
+# NOTE: docker-web and docker-omnibus require `make binary-frontend` or frontend.tar.gz content in /dist before executing.
+########################################################################################################################
+.PHONY: docker-collector
+docker-collector:
+	@echo "building collector docker image"
+	docker build $(DOCKER_TARGETARCH_BUILD_ARG) -f docker/Dockerfile.collector -t analogj/scrutiny-dev:collector .
+
+.PHONY: docker-web
+docker-web:
+	@echo "building web docker image"
+	docker build $(DOCKER_TARGETARCH_BUILD_ARG) -f docker/Dockerfile.web -t analogj/scrutiny-dev:web .
+
+.PHONY: docker-omnibus
+docker-omnibus:
+	@echo "building omnibus docker image"
+	docker build $(DOCKER_TARGETARCH_BUILD_ARG) -f docker/Dockerfile -t analogj/scrutiny-dev:omnibus .

README.md

@@ -232,18 +232,18 @@ scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
 
 # Supported Architectures
 
-
 | Architecture Name | Binaries | Docker |
 | --- | --- | --- |
-| amd64 | :white_check_mark: | :white_check_mark: |
-| arm-5 | :white_check_mark: |  |
-| arm-6 | :white_check_mark: |  |
-| arm-7 | :white_check_mark: | web/collector only. see [#236](https://github.com/AnalogJ/scrutiny/issues/236)  |
-| arm64 | :white_check_mark: | :white_check_mark: |
-| freebsd | collector only. see [#238](https://github.com/AnalogJ/scrutiny/issues/238) |  | 
-| macos-amd64 |  | :white_check_mark: |
-| macos-arm64 |  | :white_check_mark: |
-| windows-amd64 | :white_check_mark: |  |
+| linux-amd64 | :white_check_mark: | :white_check_mark: |
+| linux-arm-5 | :white_check_mark: |  |
+| linux-arm-6 | :white_check_mark: |  |
+| linux-arm-7 | :white_check_mark: | web/collector only. see [#236](https://github.com/AnalogJ/scrutiny/issues/236)  |
+| linux-arm64 | :white_check_mark: | :white_check_mark: |
+| freebsd-amd64 | :white_check_mark: |  | 
+| macos-amd64 | :white_check_mark: | :white_check_mark: |
+| macos-arm64 | :white_check_mark: | :white_check_mark: |
+| windows-amd64 | :white_check_mark: | WIP, see [#15](https://github.com/AnalogJ/scrutiny/issues/15) |
+| windows-arm64 | :white_check_mark: |  |
 
 
 # Contributing

collector/cmd/collector-metrics/collector-metrics.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/analogj/scrutiny/collector/pkg/collector"
 	"github.com/analogj/scrutiny/collector/pkg/config"
@@ -120,26 +121,16 @@ OPTIONS:
 						config.Set("api.endpoint", apiEndpoint)
 					}
 
-					collectorLogger := logrus.WithFields(logrus.Fields{
-						"type": "metrics",
-					})
-
-					if level, err := logrus.ParseLevel(config.GetString("log.level")); err == nil {
-						logrus.SetLevel(level)
-					} else {
-						logrus.SetLevel(logrus.InfoLevel)
-					}
-
-					if config.IsSet("log.file") && len(config.GetString("log.file")) > 0 {
-						logFile, err := os.OpenFile(config.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
-						if err != nil {
-							logrus.Errorf("Failed to open log file %s for output: %s", config.GetString("log.file"), err)
-							return err
-						}
+					collectorLogger, logFile, err := CreateLogger(config)
+					if logFile != nil {
 						defer logFile.Close()
-						logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
+					}
+					if err != nil {
+						return err
 					}
 
+					settingsData, err := json.MarshalIndent(config.AllSettings(), "", "\t")
+					collectorLogger.Debug(string(settingsData), err)
 					metricCollector, err := collector.CreateMetricsCollector(
 						config,
 						collectorLogger,
@@ -192,5 +183,28 @@ OPTIONS:
 	if err != nil {
 		log.Fatal(color.HiRedString("ERROR: %v", err))
 	}
-
+}
+
+func CreateLogger(appConfig config.Interface) (*logrus.Entry, *os.File, error) {
+	logger := logrus.WithFields(logrus.Fields{
+		"type": "metrics",
+	})
+
+	if level, err := logrus.ParseLevel(appConfig.GetString("log.level")); err == nil {
+		logger.Logger.SetLevel(level)
+	} else {
+		logger.Logger.SetLevel(logrus.InfoLevel)
+	}
+
+	var logFile *os.File
+	var err error
+	if appConfig.IsSet("log.file") && len(appConfig.GetString("log.file")) > 0 {
+		logFile, err = os.OpenFile(appConfig.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			logger.Logger.Errorf("Failed to open log file %s for output: %s", appConfig.GetString("log.file"), err)
+			return nil, logFile, err
+		}
+		logger.Logger.SetOutput(io.MultiWriter(os.Stderr, logFile))
+	}
+	return logger, logFile, nil
 }

collector/pkg/collector/metrics.go

@@ -9,6 +9,7 @@ import (
 	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/samber/lo"
 	"github.com/sirupsen/logrus"
 	"net/url"
 	"os"
@@ -56,11 +57,16 @@ func (mc *MetricsCollector) Run() error {
 		Logger: mc.logger,
 		Config: mc.config,
 	}
-	detectedStorageDevices, err := deviceDetector.Start()
+	rawDetectedStorageDevices, err := deviceDetector.Start()
 	if err != nil {
 		return err
 	}
 
+	//filter any device with empty wwn (they are invalid)
+	detectedStorageDevices := lo.Filter[models.Device](rawDetectedStorageDevices, func(dev models.Device, _ int) bool {
+		return len(dev.WWN) > 0
+	})
+
 	mc.logger.Infoln("Sending detected devices to API, for filtering & validation")
 	jsonObj, _ := json.Marshal(detectedStorageDevices)
 	mc.logger.Debugf("Detected devices: %v", string(jsonObj))

docker/Dockerfile

@@ -1,14 +1,16 @@
+########################################################################################################################
+# Omnibus Image
+# NOTE: this image requires the `make binary-frontend` target to have been run before `docker build` The `dist` directory must exist.
+########################################################################################################################
+
+
 ########
-FROM golang:1.17-bullseye as backendbuild
+FROM golang:1.18-bullseye as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
-
 COPY . /go/src/github.com/analogj/scrutiny
+RUN make binary-clean binary-all WEB_BINARY_NAME=scrutiny
 
-RUN go mod vendor && \
-    go build -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go && \
-    go build -o scrutiny-collector-selftest collector/cmd/collector-selftest/collector-selftest.go && \
-    go build -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
 
 ########
 FROM debian:bullseye-slim as runtime
@@ -34,11 +36,9 @@ COPY /rootfs /
 
 COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /opt/scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
 COPY dist /opt/scrutiny/web
 RUN chmod +x /opt/scrutiny/bin/scrutiny && \
-    chmod +x /opt/scrutiny/bin/scrutiny-collector-selftest && \
     chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \
     chmod 0644 /etc/cron.d/scrutiny && \
     rm -f /etc/cron.daily/* && \

docker/Dockerfile.collector

@@ -1,27 +1,28 @@
+########################################################################################################################
+# Collector Image
+########################################################################################################################
+
+
 ########
-FROM golang:1.17-bullseye as backendbuild
+FROM golang:1.18-bullseye as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
 COPY . /go/src/github.com/analogj/scrutiny
 
-RUN go mod vendor && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-selftest collector/cmd/collector-selftest/collector-selftest.go && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
+RUN make binary-clean binary-collector
 
 ########
 FROM debian:bullseye-slim as runtime
-WORKDIR /scrutiny
+WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 
 RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && update-ca-certificates
 
 COPY /docker/entrypoint-collector.sh /entrypoint-collector.sh
 COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /opt/scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
-RUN chmod +x /opt/scrutiny/bin/scrutiny-collector-selftest && \
-    chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \
+RUN chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \
     chmod +x /entrypoint-collector.sh && \
     chmod 0644 /etc/cron.d/scrutiny && \
     rm -f /etc/cron.daily/apt /etc/cron.daily/dpkg /etc/cron.daily/passwd

docker/Dockerfile.web

@@ -1,12 +1,18 @@
+########################################################################################################################
+# Web Image
+# NOTE: this image requires the `make binary-frontend` target to have been run before `docker build` The `dist` directory must exist.
+########################################################################################################################
+
+
 ########
-FROM golang:1.17-bullseye as backendbuild
+FROM golang:1.18-bullseye as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
 COPY . /go/src/github.com/analogj/scrutiny
 
-RUN go mod vendor && \
-    go build -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go
+RUN make binary-clean binary-all WEB_BINARY_NAME=scrutiny
+
 
 ########
 FROM debian:bullseye-slim as runtime

docker/Dockerfile.xgo

@@ -1,7 +0,0 @@
-FROM techknowlogick/xgo:go-1.17.x
-
-WORKDIR /go/src/github.com/analogj/scrutiny
-
-COPY . /go/src/github.com/analogj/scrutiny
-
-RUN make all

docker/Vagrantfile

@@ -1,18 +0,0 @@
-# This vagrant file is only used for local development & testing.
-
-Vagrant.configure("2") do |config|
-  config.vm.guest = :freebsd
-  config.vm.synced_folder ".", "/vagrant", id: "vagrant-root", disabled: true
-  config.vm.box = "freebsd/FreeBSD-11.0-CURRENT"
-  config.ssh.shell = "sh"
-  config.vm.base_mac = "080027D14C66"
-
-  config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--memory", "1024"]
-    vb.customize ["modifyvm", :id, "--cpus", "1"]
-    vb.customize ["modifyvm", :id, "--hwvirtex", "on"]
-    vb.customize ["modifyvm", :id, "--audio", "none"]
-    vb.customize ["modifyvm", :id, "--nictype1", "virtio"]
-    vb.customize ["modifyvm", :id, "--nictype2", "virtio"]
-  end
-end

docs/INSTALL_SYNOLOGY_COLLECTOR.md

@@ -91,9 +91,13 @@ wget https://raw.githubusercontent.com/smartmontools/smartmontools/master/smartm
 ```
 #!/bin/bash
 
-/volume1/\@Entware/scrutiny/bin/scrutiny-collector-metrics-linux-arm64 run --config /volume1/\@Entware/scrutiny/config/collector.yaml
+/volume1/\@Entware/scrutiny/bin/scrutiny-collector-metrics-linux-arm64 run --config /volume1/\@Entware/scrutiny/conf/collector.yaml
 ```
 
+**Make `run_collect.sh` executable**
+
+`chmod +x /volume1/\@Entware/scrutiny/bin/run_collect.sh`
+
 ## Set up Synology to run a scheduled task. 
 
 Log in to DSM and do the following:
@@ -131,4 +135,4 @@ Frequency: <Your desired frequency>
 
 ## Troubleshooting
 
-If you have any issues with your devices being detected, or incorrect data, please take a look at [TROUBLESHOOTING_DEVICE_COLLECTOR.md](./TROUBLESHOOTING_DEVICE_COLLECTOR.md)
+If you have any issues with your devices being detected, or incorrect data, please take a look at [TROUBLESHOOTING_DEVICE_COLLECTOR.md](./TROUBLESHOOTING_DEVICE_COLLECTOR.md)

docs/SUPPORTED_NAS_OS.md

@@ -1,17 +1,18 @@
-# Officially Supported NAS OS's
+# Officially Supported NAS/OS's
 
-These are the officially supported NAS OS's (with documentation and setup guides).
-Once a guide is created (in `docs/guides/`) it will be linked here.
+These are the officially supported NAS OS's (with documentation and setup guides). Once a guide is created (
+in `docs/guides/` or elsewhere) it will be linked here.
 
-- [ ] freenas/truenas
+- [x] [freenas/truenas](https://blog.stefandroid.com/2022/01/14/smart-scrutiny.html)
 - [x] [unraid](./INSTALL_UNRAID.md)
 - [ ] ESXI
 - [ ] Proxmox
-- [x] Synology(./INSTALL_SYNOLOGY_COLLECTOR.md)
+- [x] [Synology](./INSTALL_SYNOLOGY_COLLECTOR.md)
 - [ ] OMV
 - [ ] Amahi
 - [ ] Running in a LXC container
 - [x] [PFSense](./INSTALL_UNRAID.md)
-- [ ] QNAP
-- [ ] RockStor
-
+- [x] QNAP
+- [x] [RockStor](https://rockstor.com/docs/interface/docker-based-rock-ons/scrutiny.html)
+- [ ] Solaris/OmniOS CE Support
+- [ ] Kubernetes

docs/TESTERS.md

@@ -0,0 +1,20 @@
+# Testers
+
+Scrutiny supports many operating systems, CPU architectures and runtime environments. Unfortunately that makes it incredibly
+difficult to test. 
+Thankfully the following users have been gracious enough to test/validate Scrutiny works on their system.
+
+> NOTE: If you're interested in volunteering to test Scrutiny beta builds on your system, please [open an issue](https://github.com/AnalogJ/scrutiny/issues). 
+
+| Architecture Name | Binaries | Docker |
+| --- | --- | --- |
+| linux-amd64 | -- | @feroxy @rshxyz |
+| linux-arm-5 | -- |  |
+| linux-arm-6 | -- |  |
+| linux-arm-7 | @Zorlin | @martini1992  |
+| linux-arm64 | @SiM22 @Zorlin | @ViRb3 @agneevX @benamajin |
+| freebsd-amd64 | @BadCo-NZ @varunsridharan @martadinata666 @KenwoodFox @FingerlessGlov3s |  |
+| macos-amd64 | -- | -- |
+| macos-arm64 | -- | -- |
+| windows-amd64 | @gabrielv33 | -- |
+| windows-arm64 | -- | -- |

docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md

@@ -104,7 +104,7 @@ devices:
 As mentioned in the [README.md](/README.md), NVMe devices require both `--cap-add SYS_RAWIO` and `--cap-add SYS_ADMIN` 
 to allow smartctl permission to query your NVMe device SMART data [#26](https://github.com/AnalogJ/scrutiny/issues/26)
 
-When attaching NVMe devices using `--device=/dev/nvme..`, make sure to provide the device controller (`/dev/nvme0`) 
+When attaching NVMe devices using `--device=/dev/nvme..`, make sure to provide the device controller (`/dev/nvme0`)
 instead of the block device (`/dev/nvme0n1`). See [#209](https://github.com/AnalogJ/scrutiny/issues/209).
 
 > The character device /dev/nvme0 is the NVME device controller, and block devices like /dev/nvme0n1 are the NVME storage namespaces: the devices you use for actual storage, which will behave essentially as disks.
@@ -113,15 +113,29 @@ instead of the block device (`/dev/nvme0n1`). See [#209](https://github.com/Anal
 
 ### ATA
 
+### USB Devices
+
+The following information is extracted from [#266](https://github.com/AnalogJ/scrutiny/issues/266)
+
+External HDDs support two modes of operation usb-storage (old, slower, stable) and uas (new, faster, sometimes unstable)
+. On some external HDDs, uas mode does not properly pass through SMART information, or even causes hardware issues, so
+it has been disabled by the kernel. No amount of smartctl parameters will fix this, as it is being rejected by the
+kernel. This is especially true with Seagate HDDs. One solution is to force these devices into usb-storage mode, which
+will incur some performance penalty, but may work well enough for you. More info:
+
+- https://smartmontools.org/wiki/Supported_USB-Devices
+- https://smartmontools.org/wiki/SAT-with-UAS-Linux
+- https://forums.raspberrypi.com/viewtopic.php?t=245931
+
 ### Exit Codes
 
 If you see an error message similar to `smartctl returned an error code (2) while processing /dev/sda`, this means that
-`smartctl` (not Scrutiny) exited with an error code. Scrutiny will attempt to print a helpful error message to help you debug,
-but you can look at the table (and associated links) below to debug `smartctl`. 
+`smartctl` (not Scrutiny) exited with an error code. Scrutiny will attempt to print a helpful error message to help you
+debug, but you can look at the table (and associated links) below to debug `smartctl`.
 
 > smartctl Return Values
-> The return values of smartctl are defined by a bitmask. If all is well with the disk, the return value (exit status) of 
-> smartctl is 0 (all bits turned off). If a problem occurs, or an error, potential error, or fault is detected, then 
+> The return values of smartctl are defined by a bitmask. If all is well with the disk, the return value (exit status) of
+> smartctl is 0 (all bits turned off). If a problem occurs, or an error, potential error, or fault is detected, then
 > a non-zero status is returned. In this case, the eight different bits in the return value have the following meanings 
 > for ATA disks; some of these values may also be returned for SCSI disks.
 > 

docs/TROUBLESHOOTING_INFLUXDB.md

@@ -1,7 +1,19 @@
 # InfluxDB Troubleshooting
 
-## Installation 
-InfluxDB is a required dependency for Scrutiny v0.4.0+. 
+## Why??
+
+Scrutiny has many features, but the relevant one to this conversation is the "S.M.A.R.T metric tracking for historical
+trends". Basically Scrutiny not only shows you the current SMART values, but how they've changed over weeks, months (or
+even years).
+
+To efficiently handle that data at scale (and to make my life easier as a developer) I decided to add InfluxDB as a
+dependency. It's a dedicated timeseries database, as opposed to the general purpose sqlite DB I used before. I also did
+a bunch of testing and analysis before I made the change. With InfluxDB the memory footprint for Scrutiny (at idle) is ~
+100mb, which is still fairly reasonable.
+
+## Installation
+
+InfluxDB is a required dependency for Scrutiny v0.4.0+.
 
 https://docs.influxdata.com/influxdb/v2.2/install/
 
@@ -66,12 +78,319 @@ panic: failed to check influxdb setup status - parse "://:": missing protocol sc
 As discussed in [#248](https://github.com/AnalogJ/scrutiny/issues/248) and [#234](https://github.com/AnalogJ/scrutiny/issues/234),
 this usually related to either:
 
-- Upgrading from the LSIO Scrutiny image to the Official Scrutiny image, without removing LSIO specific environmental variables
-  - remove the `SCRUTINY_WEB=true` and `SCRUTINY_COLLECTOR=true` environmental variables. They were used by the LSIO image, but are unnecessary and cause issues with the official Scrutiny image.
-- Updated versions of the [LSIO Scrutiny images are broken](https://github.com/linuxserver/docker-scrutiny/issues/22), as they have not installed InfluxDB which is a required dependency of Scrutiny v0.4.x
-  - You can revert to an earlier version of the LSIO image (`lscr.io/linuxserver/scrutiny:060ac7b8-ls34`), or just change to the official Scrutiny image (`ghcr.io/analogj/scrutiny:master-omnibus`)
+- Upgrading from the LSIO Scrutiny image to the Official Scrutiny image, without removing LSIO specific environmental
+  variables
+  - remove the `SCRUTINY_WEB=true` and `SCRUTINY_COLLECTOR=true` environmental variables. They were used by the LSIO
+    image, but are unnecessary and cause issues with the official Scrutiny image.
+- Updated versions of the [LSIO Scrutiny images are broken](https://github.com/linuxserver/docker-scrutiny/issues/22),
+  as they have not installed InfluxDB which is a required dependency of Scrutiny v0.4.x
+  - You can revert to an earlier version of the LSIO image (`lscr.io/linuxserver/scrutiny:060ac7b8-ls34`), or just
+    change to the official Scrutiny image (`ghcr.io/analogj/scrutiny:master-omnibus`)
 
 Here's a couple of confirmed working docker-compose files that you may want to look at:
 
 - https://github.com/AnalogJ/scrutiny/blob/master/docker/example.hubspoke.docker-compose.yml
 - https://github.com/AnalogJ/scrutiny/blob/master/docker/example.omnibus.docker-compose.yml
+
+## Bring your own InfluxDB
+
+> WARNING: Most users should not follow these steps. This is ONLY for users who have an EXISTING InfluxDB installation which contains data from multiple services.
+> The Scrutiny Docker omnibus image includes an empty InfluxDB instance which it can configure.
+> If you're deploying manually or via Hub/Spoke, you can just follow the installation instructions, Scrutiny knows how
+> to run the first-time setup automatically.
+
+The goal here is to create an InfluxDB API key with minimal permissions for use by Scrutiny.
+
+- Create Scrutiny buckets (`metrics`, `metrics_weekly`, `metrics_monthly`, `metrics_yearly`) with placeholder config
+- Create Downsampling tasks (`tsk-weekly-aggr`, `tsk-monthly-aggr`, `tsk-yearly-aggr`) with placeholder script.
+- Create API token with restricted scope
+- NOTE: Placeholder bucket & task configuration will be replaced automatically by Scrutiny during startup
+
+The placeholder buckets and tasks need to be created before the API token can be created, as the resource ID's need to
+exist for the scope restriction to work.
+
+Scopes:
+
+- `orgs`: read - required for scrutiny to find it's configured org_id
+- `tasks`: scrutiny specific read/write access - Scrutiny only needs access to the downsampling tasks you created above
+- `buckets`: scrutiny specific read/write access - Scrutiny only needs access to the buckets you created above
+
+### Setup Environmental Variables
+
+```bash
+# replace the following values with correct values for your InfluxDB installation
+export INFLUXDB_ADMIN_TOKEN=pCqRq7xxxxxx-FZgNLfstIs0w==
+export INFLUXDB_ORG_ID=b2495xxxxx
+export INFLUXDB_HOSTNAME=http://localhost:8086
+
+# if you want to change the bucket name prefix below, you'll also need to update the setting in the scrutiny.yaml config file.
+export INFLUXDB_SCRUTINY_BUCKET_BASENAME=metrics
+```
+
+### Create placeholder buckets
+
+<details>
+  <summary>Click to expand!</summary>
+
+```bash
+curl -sS -X POST ${INFLUXDB_HOSTNAME}/api/v2/buckets \
+-H "Content-Type: application/json" \
+-H "Authorization: Token ${INFLUXDB_ADMIN_TOKEN}" \
+--data-binary @- << EOF
+{
+"name": "${INFLUXDB_SCRUTINY_BUCKET_BASENAME}",
+"orgID": "${INFLUXDB_ORG_ID}",
+"retentionRules": []
+}
+EOF
+
+curl -sS -X POST ${INFLUXDB_HOSTNAME}/api/v2/buckets \
+-H "Content-Type: application/json" \
+-H "Authorization: Token ${INFLUXDB_ADMIN_TOKEN}" \
+--data-binary @- << EOF
+{
+"name": "${INFLUXDB_SCRUTINY_BUCKET_BASENAME}_weekly",
+"orgID": "${INFLUXDB_ORG_ID}",
+"retentionRules": []
+}
+EOF
+
+curl -sS -X POST ${INFLUXDB_HOSTNAME}/api/v2/buckets \
+-H "Content-Type: application/json" \
+-H "Authorization: Token ${INFLUXDB_ADMIN_TOKEN}" \
+--data-binary @- << EOF
+{
+"name": "${INFLUXDB_SCRUTINY_BUCKET_BASENAME}_monthly",
+"orgID": "${INFLUXDB_ORG_ID}",
+"retentionRules": []
+}
+EOF
+
+curl -sS -X POST ${INFLUXDB_HOSTNAME}/api/v2/buckets \
+-H "Content-Type: application/json" \
+-H "Authorization: Token ${INFLUXDB_ADMIN_TOKEN}" \
+--data-binary @- << EOF
+{
+"name": "${INFLUXDB_SCRUTINY_BUCKET_BASENAME}_yearly",
+"orgID": "${INFLUXDB_ORG_ID}",
+"retentionRules": []
+}
+EOF
+```
+
+</details>
+
+### Create placeholder tasks
+
+<details>
+  <summary>Click to expand!</summary>
+
+```bash
+curl -sS -X POST ${INFLUXDB_HOSTNAME}/api/v2/tasks \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Token ${INFLUXDB_ADMIN_TOKEN}" \
+    --data-binary @- << EOF
+{
+  "orgID": "${INFLUXDB_ORG_ID}",
+  "flux": "option task = {name: \"tsk-weekly-aggr\", every: 1y} \nyield now()"
+}
+EOF
+
+curl -sS -X POST ${INFLUXDB_HOSTNAME}/api/v2/tasks \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Token ${INFLUXDB_ADMIN_TOKEN}" \
+    --data-binary @- << EOF
+{
+  "orgID": "${INFLUXDB_ORG_ID}",
+  "flux": "option task = {name: \"tsk-monthly-aggr\", every: 1y} \nyield now()"
+}
+EOF
+
+curl -sS -X POST ${INFLUXDB_HOSTNAME}/api/v2/tasks \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Token ${INFLUXDB_ADMIN_TOKEN}" \
+    --data-binary @- << EOF
+{
+  "orgID": "${INFLUXDB_ORG_ID}",
+  "flux": "option task = {name: \"tsk-yearly-aggr\", every: 1y} \nyield now()"
+}
+EOF
+
+```
+
+</details>
+
+### Create InfluxDB API Token
+
+<details>
+  <summary>Click to expand!</summary>
+
+```bash
+# replace these values with placeholder bucket and task ids from your InfluxDB installation. 
+export INFLUXDB_SCRUTINY_BASE_BUCKET_ID=1e0709xxxx
+export INFLUXDB_SCRUTINY_WEEKLY_BUCKET_ID=1af03dexxxxx
+export INFLUXDB_SCRUTINY_MONTHLY_BUCKET_ID=b3c59c7xxxxx
+export INFLUXDB_SCRUTINY_YEARLY_BUCKET_ID=f381d8cxxxxx
+
+export INFLUXDB_SCRUTINY_WEEKLY_TASK_ID=09a64ecxxxxx
+export INFLUXDB_SCRUTINY_MONTHLY_TASK_ID=09a64xxxxx
+export INFLUXDB_SCRUTINY_YEARLY_TASK_ID=09a64ecxxxxx
+
+
+curl -sS -X POST ${INFLUXDB_HOSTNAME}/api/v2/authorizations \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Token ${INFLUXDB_ADMIN_TOKEN}" \
+    --data-binary @- << EOF
+{
+  "description": "scrutiny - restricted scope token",
+  "orgID": "${INFLUXDB_ORG_ID}",
+  "permissions": [
+        {
+            "action": "read",
+            "resource": {
+                "type": "orgs"
+            }
+        },
+        {
+            "action": "read",
+            "resource": {
+                "type": "tasks"
+            }
+        },
+        {
+            "action": "write",
+            "resource": {
+                "type": "tasks",
+                "id": "${INFLUXDB_SCRUTINY_WEEKLY_TASK_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+        },
+        {
+            "action": "write",
+            "resource": {
+                "type": "tasks",
+                "id": "${INFLUXDB_SCRUTINY_MONTHLY_TASK_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+        },
+        {
+            "action": "write",
+            "resource": {
+                "type": "tasks",
+                "id": "${INFLUXDB_SCRUTINY_YEARLY_TASK_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+        },
+        {
+            "action": "read",
+            "resource": {
+                "type": "buckets",
+                "id": "${INFLUXDB_SCRUTINY_BASE_BUCKET_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+       },
+        {
+            "action": "write",
+            "resource": {
+                "type": "buckets",
+                "id": "${INFLUXDB_SCRUTINY_BASE_BUCKET_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+       },
+        {
+            "action": "read",
+            "resource": {
+                "type": "buckets",
+                "id": "${INFLUXDB_SCRUTINY_WEEKLY_BUCKET_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+       },
+        {
+            "action": "write",
+            "resource": {
+                "type": "buckets",
+                "id": "${INFLUXDB_SCRUTINY_WEEKLY_BUCKET_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+       },
+        {
+            "action": "read",
+            "resource": {
+                "type": "buckets",
+                "id": "${INFLUXDB_SCRUTINY_MONTHLY_BUCKET_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+       },
+        {
+            "action": "write",
+            "resource": {
+                "type": "buckets",
+                "id": "${INFLUXDB_SCRUTINY_MONTHLY_BUCKET_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+       },
+        {
+            "action": "read",
+            "resource": {
+                "type": "buckets",
+                "id": "${INFLUXDB_SCRUTINY_YEARLY_BUCKET_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+       },
+        {
+            "action": "write",
+            "resource": {
+                "type": "buckets",
+                "id": "${INFLUXDB_SCRUTINY_YEARLY_BUCKET_ID}",
+                "orgID": "${INFLUXDB_ORG_ID}"
+            }
+       }
+  ]
+}
+EOF
+```
+
+</details>
+
+### Save InfluxDB API Token
+
+After running the Curl command above, you'll see a JSON response that looks like the following:
+
+```json
+{
+  "token": "ksVU2t5SkQwYkvIxxxxxxxYt2xUt0uRKSbSF1Po0UQ==",
+  "status": "active",
+  "description": "scrutiny - restricted scope token",
+  "orgID": "b2495586xxxx",
+  "org": "my-org",
+  "user": "admin",
+  "permissions": [
+    {
+      "action": "read",
+      "resource": {
+        "type": "orgs"
+      }
+    },
+    {
+      "action": "read",
+      "resource": {
+        "type": "tasks"
+      }
+    },
+    {
+      "action": "write",
+      "resource": {
+        "type": "tasks",
+        "id": "09a64exxxxx",
+        "orgID": "b24955860xxxxx",
+        "org": "my-org"
+      }
+    },
+    ...
+  ]
+}
+```
+
+You must copy the token field from the JSON response, and save it in your `scrutiny.yaml` config file. After that's
+done, you can start the Scrutiny server
+

docs/TROUBLESHOOTING_NOTIFICATIONS.md

@@ -21,5 +21,6 @@ SCRUTINY_DEVICE_NAME - eg. /dev/sda
 SCRUTINY_DEVICE_TYPE - ATA/SCSI/NVMe
 SCRUTINY_DEVICE_SERIAL - eg. WDDJ324KSO
 SCRUTINY_MESSAGE - eg. "Scrutiny SMART error notification for device: %s\nFailure Type: %s\nDevice Name: %s\nDevice Serial: %s\nDevice Type: %s\nDate: %s"
+SCRUTINY_HOST_ID - (optional) eg. "my-custom-host-id"
 ```
 

docs/TROUBLESHOOTING_UDEV.md

@@ -0,0 +1,18 @@
+# Operating systems without udev
+
+Some operating systems do not come with `udev` out of the box, for example Alpine Linux. In these instances you will not be able to bind `/run/udev` to the container for sharing device metadata. Some operating systems offer `udev` as a package that can be installed separately, or an alternative (such as `eudev` in the case of Alpine Linux) that provides the same functionality.
+
+To install `eudev` in Alpine Linux (run as root):
+
+```
+apk add eudev
+setup-udev
+```
+
+Once your `udev` implementation is installed, create `/run/udev` with the following command:
+
+```
+udevadm trigger
+```
+
+On Alpine Linux, this also has the benefit of creating symlinks to device serial numbers in `/dev/disk/by-id`.

docs/dbdiagram.io.txt

@@ -1,62 +1,88 @@
 
 // SQLite Table(s)
-Table device {
-  created_at timestamp
 
-  wwn varchar [pk]
+Table Device  {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time
+	UpdatedAt time
+	DeletedAt time
 
-  //user provided
-  label varchar
-  host_id varchar
+	WWN string
 
-  // smartctl provided
-  device_name varchar
-  manufacturer varchar
-  model_name varchar
-  interface_type varchar
-  interface_speed varchar
-  serial_number varchar
-  firmware varchar
-  rotational_speed varchar
-  capacity varchar
-  form_factor varchar
-  smart_support varchar
-  device_protocol varchar
-  device_type varchar
+	DeviceName     string
+	DeviceUUID     string
+	DeviceSerialID string
+	DeviceLabel    string
 
+	Manufacturer   string
+	ModelName      string
+	InterfaceType  string
+	InterfaceSpeed string
+	SerialNumber   string
+	Firmware       string
+	RotationSpeed  int
+	Capacity       int64
+	FormFactor     string
+	SmartSupport   bool
+	DeviceProtocol string//protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string//device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string
+	HostId string
+
+	// Data set by Scrutiny
+	DeviceStatus enum
+}
+
+Table Setting {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+
+	SettingKeyName  string
+    SettingKeyDescription  string
+	SettingDataType string
+
+	SettingValueNumeric int64
+	SettingValueString  string
 }
 
 
 // InfluxDB Tables
-Table device_temperature {
-  //timestamp
-  created_at timestamp
-
-  //tags (indexed & queryable)
-  device_wwn varchar [pk]
-
-  //fields
-  temp bigint
- }
+Table SmartTemperature {
+    Date time
+	DeviceWWN      string //(tag)
+	Temp int64
+}
 
 
-Table smart_ata_results {
-  //timestamp
-  created_at timestamp
+Table Smart {
+    Date           time
+	DeviceWWN      string //(tag)
+	DeviceProtocol string
 
-  //tags (indexed & queryable)
-  device_wwn varchar [pk]
-  smart_status varchar
-  scrutiny_status varchar
+	//Metrics (fields)
+	Temp            int64
+	PowerOnHours    int64
+	PowerCycleCount int64
 
+	//Smart Status
+	Status enum
 
-
-  //fields
-  temp bigint
-  power_on_hours bigint
-  power_cycle_count bigint
-
+	//SMART Attributes (fields)
+	Attr_ID_AttributeId int
+	Attr_ID_Value       int64
+	Attr_ID_Threshold   int64
+	Attr_ID_Worst       int64
+    Attr_ID_RawValue    int64
+	Attr_ID_RawString   string
+	Attr_ID_WhenFailed  string
+	//Generated data
+	Attr_ID_TransformedValue int64
+	Attr_ID_Status           enum
+	Attr_ID_StatusReason     string
+	Attr_ID_FailureRate      float64
 
  }
 
-Ref: device.wwn < smart_ata_results.device_wwn
+Ref: Device.WWN < Smart.DeviceWWN
+Ref: Device.WWN < SmartTemperature.DeviceWWN

example.collector.yaml

@@ -87,9 +87,6 @@ devices:
 ########################################################################################################################
 
 #collect:
-#  metric:
-#    enable: true
-#    command: '-a -o on -S on'
 #  long:
 #    enable: false
 #    command: ''

example.scrutiny.yaml

@@ -73,8 +73,6 @@ log:
 #    - "join://shoutrrr:api-key@join/?devices=device1[,device2, ...][&icon=icon][&title=title]"
 #    - "script:///file/path/on/disk"
 #    - "https://www.example.com/path"
-#  filter_attributes: 'all' # options: 'all' or  'critical'
-#  level: 'fail' # options: 'fail', 'fail_scrutiny', 'fail_smart'
 
 ########################################################################################################################
 # FEATURES COMING SOON

go.mod

@@ -1,64 +1,64 @@
 module github.com/analogj/scrutiny
 
-go 1.17
+go 1.18
 
 require (
 	github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
 	github.com/containrrr/shoutrrr v0.4.4
 	github.com/fatih/color v1.10.0
 	github.com/gin-gonic/gin v1.6.3
+	github.com/glebarez/sqlite v1.4.5
 	github.com/go-gormigrate/gormigrate/v2 v2.0.0
 	github.com/golang/mock v1.4.3
 	github.com/influxdata/influxdb-client-go/v2 v2.9.0
 	github.com/jaypipes/ghw v0.6.1
-	github.com/jinzhu/gorm v1.9.16
 	github.com/mitchellh/mapstructure v1.2.2
+	github.com/samber/lo v1.25.0
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/viper v1.7.0
-	github.com/stretchr/testify v1.5.1
+	github.com/stretchr/testify v1.7.1
 	github.com/urfave/cli/v2 v2.2.0
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
-	gorm.io/driver/sqlite v1.1.3
-	gorm.io/gorm v1.20.2
+	gorm.io/gorm v1.23.5
 )
 
 require (
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
-	github.com/citilinkru/libudev v1.0.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/glebarez/go-sqlite v1.17.2 // indirect
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/go-playground/locales v0.13.0 // indirect
 	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/go-playground/validator/v10 v10.2.0 // indirect
 	github.com/golang/protobuf v1.4.2 // indirect
-	github.com/google/uuid v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
 	github.com/jaypipes/pcidb v0.5.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/jinzhu/now v1.1.1 // indirect
+	github.com/jinzhu/now v1.1.4 // indirect
 	github.com/json-iterator/go v1.1.9 // indirect
-	github.com/klauspost/compress v1.12.1 // indirect
+	github.com/klauspost/compress v1.11.7 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect
+	github.com/kvz/logstreamer v0.0.0-20201023134116-02d20f4338f5 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mattn/go-sqlite3 v1.14.4 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.1 // indirect
+	github.com/nxadm/tail v1.4.6 // indirect
+	github.com/onsi/ginkgo v1.14.2 // indirect
 	github.com/pelletier/go-toml v1.7.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/russross/blackfriday/v2 v2.0.1 // indirect
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/spf13/afero v1.2.2 // indirect
@@ -68,8 +68,9 @@ require (
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/ugorji/go/codec v1.1.7 // indirect
 	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
+	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
 	golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
-	golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect
+	golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
@@ -77,7 +78,12 @@ require (
 	gopkg.in/ini.v1 v1.55.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	gosrc.io/xmpp v0.5.1 // indirect
 	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
-	nhooyr.io/websocket v1.8.7 // indirect
+	modernc.org/libc v1.16.8 // indirect
+	modernc.org/mathutil v1.4.1 // indirect
+	modernc.org/memory v1.1.1 // indirect
+	modernc.org/sqlite v1.17.2 // indirect
+	nhooyr.io/websocket v1.8.6 // indirect
 )

go.sum

@@ -39,8 +39,6 @@ github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d/go.mod h1:0YChpVz
 github.com/chromedp/cdproto v0.0.0-20190926234355-1b4886c6fad6/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0=
 github.com/chromedp/chromedp v0.3.1-0.20190619195644-fd957a4d2901/go.mod h1:mJdvfrVn594N9tfiPecUidF6W5jPRKHymqHfzbobPsM=
 github.com/chromedp/chromedp v0.4.0/go.mod h1:DC3QUn4mJ24dwjcaGQLoZrhm4X/uPHZ6spDbS2uFhm4=
-github.com/citilinkru/libudev v1.0.0 h1:upErSdhsJGdiKxwxPmvcz43fwJJD9R+y1j8BqU4wHog=
-github.com/citilinkru/libudev v1.0.0/go.mod h1:yaNdhdtfJMs5flqeXzUOMO0mT9QnyNh/U/jdY4WhA/I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/containrrr/shoutrrr v0.4.4 h1:vHZ4E/76pKVY+Jyn/qhBz3X540Bn8NI5ppPHK4PyILY=
@@ -63,13 +61,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU=
 github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
-github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg=
 github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
@@ -84,6 +81,10 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/glebarez/go-sqlite v1.17.2 h1:gyTyFr2RFFQd2gp6fOOdfnTvUn99zwvVOrQFHA4S+DY=
+github.com/glebarez/go-sqlite v1.17.2/go.mod h1:lakPjzvnJ6uSIARV+5dPALDuSLL3879PlzHFMEpbceM=
+github.com/glebarez/sqlite v1.4.5 h1:oaJupO4X9iTn4sXRvP5Vs15BNvKh9dx5AQfciKlDvV4=
+github.com/glebarez/sqlite v1.4.5/go.mod h1:6D+bB+DdXlEC4mO+pUFJWixVcnrHTIAJ9U6Ynnn4Lxk=
 github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gormigrate/gormigrate/v2 v2.0.0 h1:e2A3Uznk4viUC4UuemuVgsNnvYZyOA8B3awlYk3UioU=
@@ -108,7 +109,6 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
@@ -139,15 +139,15 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -156,8 +156,8 @@ github.com/google/pprof v0.0.0-20190908185732-236ed259b199/go.mod h1:zfwlbNMJ+OI
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
-github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -254,13 +254,11 @@ github.com/jaypipes/ghw v0.6.1/go.mod h1:QOXppNRCLGYR1H+hu09FxZPqjNt09bqUZUnOL3R
 github.com/jaypipes/pcidb v0.5.0 h1:4W5gZ+G7QxydevI8/MmmKdnIPJpURqJ2JNXTzfLxF5c=
 github.com/jaypipes/pcidb v0.5.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o=
-github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
 github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas=
+github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -271,32 +269,30 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg=
 github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.12.1 h1:/+xsCsk06wE38cyiqOR/o7U2fSftcH72xD+BQXmja/g=
-github.com/klauspost/compress v1.12.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 h1:3tLzEnUizyN9YLWFTT9loC30lSBvh2y70LTDcZOTs1s=
-github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0/go.mod h1:8/LTPeDLaklcUjgSQBHbhBF1ibKAFxzS5o+H7USfMSA=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kvz/logstreamer v0.0.0-20201023134116-02d20f4338f5 h1:dkCjlgGN81ahDFtM9R1x16gFGTa7ZvgZfdtAfM9lWOs=
+github.com/kvz/logstreamer v0.0.0-20201023134116-02d20f4338f5/go.mod h1:8/LTPeDLaklcUjgSQBHbhBF1ibKAFxzS5o+H7USfMSA=
 github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -320,12 +316,12 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
-github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
-github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI=
-github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
+github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0=
+github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -346,17 +342,16 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.6 h1:11TGpSHY7Esh/i/qnq02Jo5oVrI1Gue8Slbq0ujPZFQ=
 github.com/nxadm/tail v1.4.6/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
 github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54=
-github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
@@ -381,6 +376,8 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
@@ -389,6 +386,8 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/samber/lo v1.25.0 h1:H8F6cB0RotRdgcRCivTByAQePaYhGMdOTJIj2QFS2I0=
+github.com/samber/lo v1.25.0/go.mod h1:2I7tgIv8Q1SG2xEIkRq0F2i2zgxVpnyPOP0d3Gj2r+A=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
@@ -432,14 +431,15 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/twitchyliquid64/golang-asm v0.0.0-20190126203739-365674df15fc/go.mod h1:NoCfSFWosfqMqmmD7hApkirIK9ozpHjxRnRxs1l413A=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
@@ -475,7 +475,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -486,6 +485,8 @@ golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -572,11 +573,12 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 h1:iGu644GcxtEcrInvDsQRCwJjtCIOlT2V7IRt6ah2Whw=
-golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 h1:D1v9ucDTYBtbz5vNuBbAhIMAGhQhJ6Ym5ah3maMVNX4=
+golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -616,7 +618,7 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -656,8 +658,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
@@ -675,20 +677,21 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/mysql v1.0.1 h1:omJoilUzyrAp0xNoio88lGJCroGdIOen9hq2A/+3ifw=
 gorm.io/driver/mysql v1.0.1/go.mod h1:KtqSthtg55lFp3S5kUXqlGaelnWpKitn4k1xZTnoiPw=
 gorm.io/driver/postgres v1.0.0 h1:Yh4jyFQ0a7F+JPU0Gtiam/eKmpT/XFc1FKxotGqc6FM=
 gorm.io/driver/postgres v1.0.0/go.mod h1:wtMFcOzmuA5QigNsgEIb7O5lhvH1tHAF1RbWmLWV4to=
+gorm.io/driver/sqlite v1.1.1 h1:qtWqNAEUyi7gYSUAJXeiAMz0lUOdakZF5ia9Fqnp5G4=
 gorm.io/driver/sqlite v1.1.1/go.mod h1:hm2olEcl8Tmsc6eZyxYSeznnsDaMqamBvEXLNtBg4cI=
-gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc=
-gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c=
 gorm.io/driver/sqlserver v1.0.2 h1:FzxAlw0/7hntMzSiNfotpYCo9Lz8dqWQGdmCGqIiFGo=
 gorm.io/driver/sqlserver v1.0.2/go.mod h1:gb0Y9QePGgqjzrVyTQUZeh9zkd5v0iz71cM1B4ZycEY=
 gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
 gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
-gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
-gorm.io/gorm v1.20.2 h1:bZzSEnq7NDGsrd+n3evOOedDrY5oLM5QPlCjZJUK2ro=
-gorm.io/gorm v1.20.2/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+gorm.io/gorm v1.23.5 h1:TnlF26wScKSvknUC/Rn8t0NLLM22fypYBlvj1+aH6dM=
+gorm.io/gorm v1.23.5/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 gosrc.io/xmpp v0.5.1 h1:Rgrm5s2rt+npGggJH3HakQxQXR8ZZz3+QRzakRQqaq4=
 gosrc.io/xmpp v0.5.1/go.mod h1:L3NFMqYOxyLz3JGmgFyWf7r9htE91zVGiK40oW4RwdY=
 gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
@@ -699,11 +702,36 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
+lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
+modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
+modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
+modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
+modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
+modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
+modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
+modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
+modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
+modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A=
+modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU=
+modernc.org/libc v1.16.7/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
+modernc.org/libc v1.16.8 h1:Ux98PaOMvolgoFX/YwusFOHBnanXdGRmWgI8ciI2z4o=
+modernc.org/libc v1.16.8/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
+modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8=
+modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU=
+modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
+modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/sqlite v1.17.2 h1:TjmF36Wi5QcPYqRoAacV1cAyJ7xB/CD0ExpVUEMebnw=
+modernc.org/sqlite v1.17.2/go.mod h1:GOQmuiXd6pTTes1Fi2s9apiCcD/wbKQtBZ0Nw6/etjM=
+modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
+modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw=
+modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8=
 mvdan.cc/sh v2.6.4+incompatible/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8=
 nhooyr.io/websocket v1.6.5/go.mod h1:F259lAzPRAH0htX2y3ehpJe09ih1aSHN7udWki1defY=
+nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k=
 nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
-nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
-nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

webapp/backend/cmd/scrutiny/scrutiny.go

@@ -1,12 +1,15 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/version"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/web"
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
+	"io"
+	"log"
 	"os"
 	"time"
 
@@ -107,7 +110,18 @@ OPTIONS:
 						config.Set("log.file", c.String("log-file"))
 					}
 
-					webServer := web.AppEngine{Config: config}
+					webLogger, logFile, err := CreateLogger(config)
+					if logFile != nil {
+						defer logFile.Close()
+					}
+					if err != nil {
+						return err
+					}
+
+					settingsData, err := json.Marshal(config.AllSettings())
+					webLogger.Debug(string(settingsData), err)
+
+					webServer := web.AppEngine{Config: config, Logger: webLogger}
 
 					return webServer.Start()
 				},
@@ -140,3 +154,27 @@ OPTIONS:
 	}
 
 }
+
+func CreateLogger(appConfig config.Interface) (*logrus.Entry, *os.File, error) {
+	logger := logrus.WithFields(logrus.Fields{
+		"type": "web",
+	})
+	//set default log level
+	if level, err := logrus.ParseLevel(appConfig.GetString("log.level")); err == nil {
+		logger.Logger.SetLevel(level)
+	} else {
+		logger.Logger.SetLevel(logrus.InfoLevel)
+	}
+
+	var logFile *os.File
+	var err error
+	if appConfig.IsSet("log.file") && len(appConfig.GetString("log.file")) > 0 {
+		logFile, err = os.OpenFile(appConfig.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			logger.Logger.Errorf("Failed to open log file %s for output: %s", appConfig.GetString("log.file"), err)
+			return nil, logFile, err
+		}
+		logger.Logger.SetOutput(io.MultiWriter(os.Stderr, logFile))
+	}
+	return logger, logFile, nil
+}

webapp/backend/pkg/config/config.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"github.com/analogj/go-util/utils"
-	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
 	"github.com/spf13/viper"
 	"log"
@@ -10,6 +9,8 @@ import (
 	"strings"
 )
 
+const DB_USER_SETTINGS_SUBKEY = "user"
+
 // When initializing this class the following methods must be called:
 // Config.New
 // Config.Init
@@ -39,8 +40,6 @@ func (c *configuration) Init() error {
 	c.SetDefault("log.file", "")
 
 	c.SetDefault("notify.urls", []string{})
-	c.SetDefault("notify.filter_attributes", pkg.NotifyFilterAttributesAll)
-	c.SetDefault("notify.level", pkg.NotifyLevelFail)
 
 	c.SetDefault("web.influxdb.scheme", "http")
 	c.SetDefault("web.influxdb.host", "localhost")
@@ -55,17 +54,6 @@ func (c *configuration) Init() error {
 	//c.SetDefault("disks.include", []string{})
 	//c.SetDefault("disks.exclude", []string{})
 
-	//c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh")
-	//c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh")
-	//c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh")
-
-	//c.SetDefault("collect.metric.enable", true)
-	//c.SetDefault("collect.metric.command", "-a -o on -S on")
-	//c.SetDefault("collect.long.enable", true)
-	//c.SetDefault("collect.long.command", "-a -o on -S on")
-	//c.SetDefault("collect.short.enable", true)
-	//c.SetDefault("collect.short.command", "-a -o on -S on")
-
 	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
 	c.SetConfigType("yaml")
 	//c.SetConfigName("drawbridge")
@@ -77,7 +65,18 @@ func (c *configuration) Init() error {
 	c.AutomaticEnv()
 
 	//CLI options will be added via the `Set()` function
-	return nil
+	return c.ValidateConfig()
+}
+
+func (c *configuration) SubKeys(key string) []string {
+	return c.Sub(key).AllKeys()
+}
+
+func (c *configuration) Sub(key string) Interface {
+	config := configuration{
+		Viper: c.Viper.Sub(key),
+	}
+	return &config
 }
 
 func (c *configuration) ReadConfig(configFilePath string) error {
@@ -120,24 +119,18 @@ func (c *configuration) ReadConfig(configFilePath string) error {
 // This function ensures that the merged config works correctly.
 func (c *configuration) ValidateConfig() error {
 
-	////deserialize Questions
-	//questionsMap := map[string]Question{}
-	//err := c.UnmarshalKey("questions", &questionsMap)
-	//
-	//if err != nil {
-	//	log.Printf("questions could not be deserialized correctly. %v", err)
-	//	return err
-	//}
-	//
-	//for _, v := range questionsMap {
-	//
-	//	typeContent, ok := v.Schema["type"].(string)
-	//	if !ok || len(typeContent) == 0 {
-	//		return errors.QuestionSyntaxError("`type` is required for questions")
-	//	}
-	//}
-	//
-	//
+	//the following keys are deprecated, and no longer supported
+	/*
+		- notify.filter_attributes (replaced by metrics.status.filter_attributes SETTING)
+		- notify.level (replaced by metrics.notify.level and metrics.status.threshold SETTING)
+	*/
+	//TODO add docs and upgrade doc.
+	if c.IsSet("notify.filter_attributes") {
+		return errors.ConfigValidationError("`notify.filter_attributes` configuration option is deprecated. Replaced by option in Dashboard Settings page")
+	}
+	if c.IsSet("notify.level") {
+		return errors.ConfigValidationError("`notify.level` configuration option is deprecated. Replaced by option in Dashboard Settings page")
+	}
 
 	return nil
 }

webapp/backend/pkg/config/config_test.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func Test_MergeConfigMap(t *testing.T) {
+	//setup
+	testConfig := configuration{
+		Viper: viper.New(),
+	}
+	testConfig.Set("user.dashboard_display", "hello")
+	testConfig.SetDefault("user.layout", "hello")
+
+	mergeSettings := map[string]interface{}{
+		"user": map[string]interface{}{
+			"dashboard_display": "dashboard_display",
+			"layout":            "layout",
+		},
+	}
+	//test
+	err := testConfig.MergeConfigMap(mergeSettings)
+
+	//verify
+	require.NoError(t, err)
+
+	// if using Set, the MergeConfigMap functionality will not override
+	// if using SetDefault, the MergeConfigMap will override correctly
+	require.Equal(t, "hello", testConfig.GetString("user.dashboard_display"))
+	require.Equal(t, "layout", testConfig.GetString("user.layout"))
+
+}

webapp/backend/pkg/config/interface.go

@@ -12,12 +12,17 @@ type Interface interface {
 	WriteConfig() error
 	Set(key string, value interface{})
 	SetDefault(key string, value interface{})
+	MergeConfigMap(cfg map[string]interface{}) error
 
+	Sub(key string) Interface
 	AllSettings() map[string]interface{}
+	AllKeys() []string
+	SubKeys(key string) []string
 	IsSet(key string) bool
 	Get(key string) interface{}
 	GetBool(key string) bool
 	GetInt(key string) int
+	GetInt64(key string) int64
 	GetString(key string) string
 	GetStringSlice(key string) []string
 	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error

webapp/backend/pkg/config/mock/mock_config.go

@@ -7,6 +7,7 @@ package mock_config
 import (
 	reflect "reflect"
 
+	config "github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	gomock "github.com/golang/mock/gomock"
 	viper "github.com/spf13/viper"
 )
@@ -34,6 +35,20 @@ func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
+// AllKeys mocks base method.
+func (m *MockInterface) AllKeys() []string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AllKeys")
+	ret0, _ := ret[0].([]string)
+	return ret0
+}
+
+// AllKeys indicates an expected call of AllKeys.
+func (mr *MockInterfaceMockRecorder) AllKeys() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllKeys", reflect.TypeOf((*MockInterface)(nil).AllKeys))
+}
+
 // AllSettings mocks base method.
 func (m *MockInterface) AllSettings() map[string]interface{} {
 	m.ctrl.T.Helper()
@@ -90,6 +105,20 @@ func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
 }
 
+// GetInt64 mocks base method.
+func (m *MockInterface) GetInt64(key string) int64 {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetInt64", key)
+	ret0, _ := ret[0].(int64)
+	return ret0
+}
+
+// GetInt64 indicates an expected call of GetInt64.
+func (mr *MockInterfaceMockRecorder) GetInt64(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt64", reflect.TypeOf((*MockInterface)(nil).GetInt64), key)
+}
+
 // GetString mocks base method.
 func (m *MockInterface) GetString(key string) string {
 	m.ctrl.T.Helper()
@@ -146,6 +175,20 @@ func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
 }
 
+// MergeConfigMap mocks base method.
+func (m *MockInterface) MergeConfigMap(cfg map[string]interface{}) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MergeConfigMap", cfg)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// MergeConfigMap indicates an expected call of MergeConfigMap.
+func (mr *MockInterfaceMockRecorder) MergeConfigMap(cfg interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MergeConfigMap", reflect.TypeOf((*MockInterface)(nil).MergeConfigMap), cfg)
+}
+
 // ReadConfig mocks base method.
 func (m *MockInterface) ReadConfig(configFilePath string) error {
 	m.ctrl.T.Helper()
@@ -184,6 +227,34 @@ func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
 }
 
+// Sub mocks base method.
+func (m *MockInterface) Sub(key string) config.Interface {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Sub", key)
+	ret0, _ := ret[0].(config.Interface)
+	return ret0
+}
+
+// Sub indicates an expected call of Sub.
+func (mr *MockInterfaceMockRecorder) Sub(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sub", reflect.TypeOf((*MockInterface)(nil).Sub), key)
+}
+
+// SubKeys mocks base method.
+func (m *MockInterface) SubKeys(key string) []string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SubKeys", key)
+	ret0, _ := ret[0].([]string)
+	return ret0
+}
+
+// SubKeys indicates an expected call of SubKeys.
+func (mr *MockInterfaceMockRecorder) SubKeys(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubKeys", reflect.TypeOf((*MockInterface)(nil).SubKeys), key)
+}
+
 // UnmarshalKey mocks base method.
 func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
 	m.ctrl.T.Helper()

webapp/backend/pkg/constants.go

@@ -4,17 +4,11 @@ const DeviceProtocolAta = "ATA"
 const DeviceProtocolScsi = "SCSI"
 const DeviceProtocolNvme = "NVMe"
 
-const NotifyFilterAttributesAll = "all"
-const NotifyFilterAttributesCritical = "critical"
-
-const NotifyLevelFail = "fail"
-const NotifyLevelFailScrutiny = "fail_scrutiny"
-const NotifyLevelFailSmart = "fail_smart"
-
+//go:generate stringer -type=AttributeStatus
+// AttributeStatus bitwise flag, 1,2,4,8,16,32,etc
 type AttributeStatus uint8
 
 const (
-	// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
 	AttributeStatusPassed          AttributeStatus = 0
 	AttributeStatusFailedSmart     AttributeStatus = 1
 	AttributeStatusWarningScrutiny AttributeStatus = 2
@@ -29,10 +23,11 @@ func AttributeStatusClear(b, flag AttributeStatus) AttributeStatus  { return b &
 func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ flag }
 func AttributeStatusHas(b, flag AttributeStatus) bool               { return b&flag != 0 }
 
+//go:generate stringer -type=DeviceStatus
+// DeviceStatus bitwise flag, 1,2,4,8,16,32,etc
 type DeviceStatus uint8
 
 const (
-	// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
 	DeviceStatusPassed         DeviceStatus = 0
 	DeviceStatusFailedSmart    DeviceStatus = 1
 	DeviceStatusFailedScrutiny DeviceStatus = 2
@@ -42,3 +37,29 @@ func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus    { return b | flag }
 func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
 func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
 func DeviceStatusHas(b, flag DeviceStatus) bool            { return b&flag != 0 }
+
+// Metrics Specific Filtering & Threshold Constants
+type MetricsNotifyLevel int64
+
+const (
+	MetricsNotifyLevelWarn MetricsNotifyLevel = 1
+	MetricsNotifyLevelFail MetricsNotifyLevel = 2
+)
+
+type MetricsStatusFilterAttributes int64
+
+const (
+	MetricsStatusFilterAttributesAll      MetricsStatusFilterAttributes = 0
+	MetricsStatusFilterAttributesCritical MetricsStatusFilterAttributes = 1
+)
+
+// MetricsStatusThreshold bitwise flag, 1,2,4,8,16,32,etc
+type MetricsStatusThreshold int64
+
+const (
+	MetricsStatusThresholdSmart    MetricsStatusThreshold = 1
+	MetricsStatusThresholdScrutiny MetricsStatusThreshold = 2
+
+	//shortcut
+	MetricsStatusThresholdBoth MetricsStatusThreshold = 3
+)

webapp/backend/pkg/database/interface.go

@@ -11,9 +11,6 @@ import (
 type DeviceRepo interface {
 	Close() error
 
-	//GetSettings()
-	//SaveSetting()
-
 	RegisterDevice(ctx context.Context, dev models.Device) error
 	GetDevices(ctx context.Context) ([]models.Device, error)
 	UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error)
@@ -28,4 +25,7 @@ type DeviceRepo interface {
 
 	GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
 	GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)
+
+	LoadSettings(ctx context.Context) (*models.Settings, error)
+	SaveSettings(ctx context.Context, settings models.Settings) error
 }

webapp/backend/pkg/database/migrations/m20220716214900/setting.go

@@ -0,0 +1,18 @@
+package m20220716214900
+
+import (
+	"gorm.io/gorm"
+)
+
+type Setting struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	gorm.Model
+
+	SettingKeyName        string `json:"setting_key_name"`
+	SettingKeyDescription string `json:"setting_key_description"`
+	SettingDataType       string `json:"setting_data_type"`
+
+	SettingValueNumeric int    `json:"setting_value_numeric"`
+	SettingValueString  string `json:"setting_value_string"`
+	SettingValueBool    bool   `json:"setting_value_bool"`
+}

webapp/backend/pkg/database/scrutiny_repository.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/glebarez/sqlite"
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
 	"github.com/influxdata/influxdb-client-go/v2/api"
 	"github.com/influxdata/influxdb-client-go/v2/domain"
 	"github.com/sirupsen/logrus"
-	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"io/ioutil"
 	"net/http"
@@ -62,7 +62,20 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field
 	// Gorm/SQLite setup
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 	globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
-	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
+
+	// When a transaction cannot lock the database, because it is already locked by another one,
+	// SQLite by default throws an error: database is locked. This behavior is usually not appropriate when
+	// concurrent access is needed, typically when multiple processes write to the same database.
+	// PRAGMA busy_timeout lets you set a timeout or a handler for these events. When setting a timeout,
+	// SQLite will try the transaction multiple times within this timeout.
+	// fixes #341
+	// https://rsqlite.r-dbi.org/reference/sqlitesetbusyhandler
+	// retrying for 30000 milliseconds, 30seconds - this would be unreasonable for a distributed multi-tenant application,
+	// but should be fine for local usage.
+	pragmaStr := sqlitePragmaString(map[string]string{
+		"busy_timeout": "30000",
+	})
+	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")+pragmaStr), &gorm.Config{
 		//TODO: figure out how to log database queries again.
 		//Logger: logger
 		DisableForeignKeyConstraintWhenMigrating: true,
@@ -242,21 +255,29 @@ func (sr *scrutinyRepository) EnsureBuckets(ctx context.Context, org *domain.Org
 
 	//create buckets (used for downsampling)
 	weeklyBucket := fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
-	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, weeklyBucket); foundErr != nil {
+	if foundWeeklyBucket, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, weeklyBucket); foundErr != nil {
 		// metrics_weekly bucket will have a retention period of 8+1 weeks (since it will be down-sampled once a month)
 		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, weeklyBucket, weeklyBucketRetentionRule)
 		if err != nil {
 			return err
 		}
+	} else if sr.appConfig.GetBool("web.influxdb.retention_policy") {
+		//correctly set the retention period for the bucket (may not be able to do it during setup/creation)
+		foundWeeklyBucket.RetentionRules = domain.RetentionRules{weeklyBucketRetentionRule}
+		sr.influxClient.BucketsAPI().UpdateBucket(ctx, foundWeeklyBucket)
 	}
 
 	monthlyBucket := fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
-	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, monthlyBucket); foundErr != nil {
+	if foundMonthlyBucket, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, monthlyBucket); foundErr != nil {
 		// metrics_monthly bucket will have a retention period of 24+1 months (since it will be down-sampled once a year)
 		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, monthlyBucket, monthlyBucketRetentionRule)
 		if err != nil {
 			return err
 		}
+	} else if sr.appConfig.GetBool("web.influxdb.retention_policy") {
+		//correctly set the retention period for the bucket (may not be able to do it during setup/creation)
+		foundMonthlyBucket.RetentionRules = domain.RetentionRules{monthlyBucketRetentionRule}
+		sr.influxClient.BucketsAPI().UpdateBucket(ctx, foundMonthlyBucket)
 	}
 
 	yearlyBucket := fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
@@ -442,3 +463,16 @@ func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []str
 	}
 	return []string{DURATION_KEY_WEEK}
 }
+
+func sqlitePragmaString(pragmas map[string]string) string {
+	q := url.Values{}
+	for key, val := range pragmas {
+		q.Add("_pragma", key+"="+val)
+	}
+
+	queryStr := q.Encode()
+	if len(queryStr) > 0 {
+		return "?" + queryStr
+	}
+	return ""
+}

webapp/backend/pkg/database/scrutiny_repository_migrations.go

@@ -4,15 +4,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220716214900"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	_ "github.com/glebarez/sqlite"
 	"github.com/go-gormigrate/gormigrate/v2"
 	"github.com/influxdata/influxdb-client-go/v2/api/http"
-	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	log "github.com/sirupsen/logrus"
 	"gorm.io/gorm"
 	"strconv"
@@ -267,6 +269,85 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 				return tx.AutoMigrate(m20220509170100.Device{})
 			},
 		},
+		{
+			ID: "m20220709181300",
+			Migrate: func(tx *gorm.DB) error {
+
+				// delete devices with empty `wwn` field (they are impossible to delete manually), and are invalid.
+				return tx.Where("wwn = ?", "").Delete(&models.Device{}).Error
+			},
+		},
+		{
+			ID: "m20220716214900", // add settings table.
+			Migrate: func(tx *gorm.DB) error {
+
+				// adding the settings table.
+				err := tx.AutoMigrate(m20220716214900.Setting{})
+				if err != nil {
+					return err
+				}
+				//add defaults.
+
+				var defaultSettings = []m20220716214900.Setting{
+					{
+						SettingKeyName:        "theme",
+						SettingKeyDescription: "Frontend theme ('light' | 'dark' | 'system')",
+						SettingDataType:       "string",
+						SettingValueString:    "system", // options: 'light' | 'dark' | 'system'
+					},
+					{
+						SettingKeyName:        "layout",
+						SettingKeyDescription: "Frontend layout ('material')",
+						SettingDataType:       "string",
+						SettingValueString:    "material",
+					},
+					{
+						SettingKeyName:        "dashboard_display",
+						SettingKeyDescription: "Frontend device display title ('name' | 'serial_id' | 'uuid' | 'label')",
+						SettingDataType:       "string",
+						SettingValueString:    "name",
+					},
+					{
+						SettingKeyName:        "dashboard_sort",
+						SettingKeyDescription: "Frontend device sort by ('status' | 'title' | 'age')",
+						SettingDataType:       "string",
+						SettingValueString:    "status",
+					},
+					{
+						SettingKeyName:        "temperature_unit",
+						SettingKeyDescription: "Frontend temperature unit ('celsius' | 'fahrenheit')",
+						SettingDataType:       "string",
+						SettingValueString:    "celsius",
+					},
+					{
+						SettingKeyName:        "file_size_si_units",
+						SettingKeyDescription: "File size in SI units (true | false)",
+						SettingDataType:       "bool",
+						SettingValueBool:      false,
+					},
+
+					{
+						SettingKeyName:        "metrics.notify_level",
+						SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)",
+						SettingDataType:       "numeric",
+						SettingValueNumeric:   int(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn'
+					},
+					{
+						SettingKeyName:        "metrics.status_filter_attributes",
+						SettingKeyDescription: "Determines which attributes should impact device status",
+						SettingDataType:       "numeric",
+						SettingValueNumeric:   int(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or  'critical'
+					},
+					{
+						SettingKeyName:        "metrics.status_threshold",
+						SettingKeyDescription: "Determines which threshold should impact device status",
+						SettingDataType:       "numeric",
+						SettingValueNumeric:   int(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both'
+					},
+				}
+				return tx.Create(&defaultSettings).Error
+			},
+		},
 	})
 
 	if err := m.Migrate(); err != nil {
@@ -274,6 +355,30 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 		return err
 	}
 	sr.logger.Infoln("Database migration completed successfully")
+
+	//these migrations cannot be done within a transaction, so they are done as a separate group, with `UseTransaction = false`
+	sr.logger.Infoln("SQLite global configuration migrations starting. Please wait....")
+	globalMigrateOptions := gormigrate.DefaultOptions
+	globalMigrateOptions.UseTransaction = false
+	gm := gormigrate.New(sr.gormClient, globalMigrateOptions, []*gormigrate.Migration{
+		{
+			ID: "g20220802211500",
+			Migrate: func(tx *gorm.DB) error {
+				//shrink the Database (maybe necessary after 20220503113100)
+				if err := tx.Exec("VACUUM;").Error; err != nil {
+					return err
+				}
+				return nil
+			},
+		},
+	})
+
+	if err := gm.Migrate(); err != nil {
+		sr.logger.Errorf("SQLite global configuration migrations failed with error. \n Please open a github issue at https://github.com/AnalogJ/scrutiny and attach a copy of your scrutiny.db file. \n %v", err)
+		return err
+	}
+	sr.logger.Infoln("SQLite global configuration migrations completed successfully")
+
 	return nil
 }
 

webapp/backend/pkg/database/scrutiny_repository_settings.go

@@ -0,0 +1,85 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/mitchellh/mapstructure"
+	"strings"
+)
+
+// LoadSettings will retrieve settings from the database, store them in the AppConfig object, and return a Settings struct
+func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Settings, error) {
+	settingsEntries := []models.SettingEntry{}
+	if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
+		return nil, fmt.Errorf("Could not get settings from DB: %v", err)
+	}
+
+	// store retrieved settings in the AppConfig obj
+	for _, settingsEntry := range settingsEntries {
+		configKey := fmt.Sprintf("%s.%s", config.DB_USER_SETTINGS_SUBKEY, settingsEntry.SettingKeyName)
+
+		if settingsEntry.SettingDataType == "numeric" {
+			sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueNumeric)
+		} else if settingsEntry.SettingDataType == "string" {
+			sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueString)
+		} else if settingsEntry.SettingDataType == "bool" {
+			sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueBool)
+		}
+	}
+
+	// unmarshal the dbsetting object data to a settings object.
+	var settings models.Settings
+	err := sr.appConfig.UnmarshalKey(config.DB_USER_SETTINGS_SUBKEY, &settings)
+	if err != nil {
+		return nil, err
+	}
+	return &settings, nil
+}
+
+// testing
+// curl -d '{"metrics": { "notify_level": 5, "status_filter_attributes": 5, "status_threshold": 5 }}' -H "Content-Type: application/json" -X POST http://localhost:9090/api/settings
+// SaveSettings will update settings in AppConfig object, then save the settings to the database.
+func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error {
+	//save the entries to the appconfig
+	settingsMap := &map[string]interface{}{}
+	err := mapstructure.Decode(settings, &settingsMap)
+	if err != nil {
+		return err
+	}
+	settingsWrapperMap := map[string]interface{}{}
+	settingsWrapperMap[config.DB_USER_SETTINGS_SUBKEY] = *settingsMap
+	err = sr.appConfig.MergeConfigMap(settingsWrapperMap)
+	if err != nil {
+		return err
+	}
+	sr.logger.Debugf("after merge settings: %v", sr.appConfig.AllSettings())
+	//retrieve current settings from the database
+	settingsEntries := []models.SettingEntry{}
+	if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
+		return fmt.Errorf("Could not get settings from DB: %v", err)
+	}
+
+	//update settingsEntries
+	for ndx, settingsEntry := range settingsEntries {
+		configKey := fmt.Sprintf("%s.%s", config.DB_USER_SETTINGS_SUBKEY, strings.ToLower(settingsEntry.SettingKeyName))
+
+		if settingsEntry.SettingDataType == "numeric" {
+			settingsEntries[ndx].SettingValueNumeric = sr.appConfig.GetInt(configKey)
+		} else if settingsEntry.SettingDataType == "string" {
+			settingsEntries[ndx].SettingValueString = sr.appConfig.GetString(configKey)
+		} else if settingsEntry.SettingDataType == "bool" {
+			settingsEntries[ndx].SettingValueBool = sr.appConfig.GetBool(configKey)
+		}
+
+		// store in database.
+		//TODO: this should be `sr.gormClient.Updates(&settingsEntries).Error`
+		err := sr.gormClient.Model(&models.SettingEntry{}).Where([]uint{settingsEntry.ID}).Select("setting_value_numeric", "setting_value_string", "setting_value_bool").Updates(settingsEntries[ndx]).Error
+		if err != nil {
+			return err
+		}
+
+	}
+	return nil
+}

webapp/backend/pkg/database/scrutiny_repository_tasks.go

@@ -11,35 +11,71 @@ import (
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) error {
 	weeklyTaskName := "tsk-weekly-aggr"
+	weeklyTaskScript := sr.DownsampleScript("weekly", weeklyTaskName, "0 1 * * 0")
 	if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: weeklyTaskName}); findErr == nil && len(found) == 0 {
 		//weekly on Sunday at 1:00am
-		_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, weeklyTaskName, sr.DownsampleScript("weekly"), "0 1 * * 0", orgID)
+		_, err := sr.influxTaskApi.CreateTaskByFlux(ctx, weeklyTaskScript, orgID)
 		if err != nil {
 			return err
 		}
+	} else if len(found) == 1 {
+		//check if we should update
+		task := &found[0]
+		if weeklyTaskScript != task.Flux {
+			sr.logger.Infoln("updating weekly task script")
+			task.Flux = weeklyTaskScript
+			_, err := sr.influxTaskApi.UpdateTask(ctx, task)
+			if err != nil {
+				return err
+			}
+		}
 	}
 
 	monthlyTaskName := "tsk-monthly-aggr"
+	monthlyTaskScript := sr.DownsampleScript("monthly", monthlyTaskName, "30 1 1 * *")
 	if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: monthlyTaskName}); findErr == nil && len(found) == 0 {
 		//monthly on first day of the month at 1:30am
-		_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, monthlyTaskName, sr.DownsampleScript("monthly"), "30 1 1 * *", orgID)
+		_, err := sr.influxTaskApi.CreateTaskByFlux(ctx, monthlyTaskScript, orgID)
 		if err != nil {
 			return err
 		}
+	} else if len(found) == 1 {
+		//check if we should update
+		task := &found[0]
+		if monthlyTaskScript != task.Flux {
+			sr.logger.Infoln("updating monthly task script")
+			task.Flux = monthlyTaskScript
+			_, err := sr.influxTaskApi.UpdateTask(ctx, task)
+			if err != nil {
+				return err
+			}
+		}
 	}
 
 	yearlyTaskName := "tsk-yearly-aggr"
+	yearlyTaskScript := sr.DownsampleScript("yearly", yearlyTaskName, "0 2 1 1 *")
 	if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: yearlyTaskName}); findErr == nil && len(found) == 0 {
 		//yearly on the first day of the year at 2:00am
-		_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, yearlyTaskName, sr.DownsampleScript("yearly"), "0 2 1 1 *", orgID)
+		_, err := sr.influxTaskApi.CreateTaskByFlux(ctx, yearlyTaskScript, orgID)
 		if err != nil {
 			return err
 		}
+	} else if len(found) == 1 {
+		//check if we should update
+		task := &found[0]
+		if yearlyTaskScript != task.Flux {
+			sr.logger.Infoln("updating yearly task script")
+			task.Flux = yearlyTaskScript
+			_, err := sr.influxTaskApi.UpdateTask(ctx, task)
+			if err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }
 
-func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
+func (sr *scrutinyRepository) DownsampleScript(aggregationType string, name string, cron string) string {
 	var sourceBucket string // the source of the data
 	var destBucket string   // the destination for the aggregated data
 	var rangeStart string
@@ -88,30 +124,37 @@ func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
 	*/
 
 	return fmt.Sprintf(`
-  sourceBucket = "%s"
-  rangeStart = %s
-  rangeEnd = %s
-  aggWindow = %s
-  destBucket = "%s"
-  destOrg = "%s"
+option task = { 
+  name: "%s",
+  cron: "%s",
+}
 
-  from(bucket: sourceBucket)
-  |> range(start: rangeStart, stop: rangeEnd)
-  |> filter(fn: (r) => r["_measurement"] == "smart" )
-  |> group(columns: ["device_wwn", "_field"])
-  |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
-  |> to(bucket: destBucket, org: destOrg)
+sourceBucket = "%s"
+rangeStart = %s
+rangeEnd = %s
+aggWindow = %s
+destBucket = "%s"
+destOrg = "%s"
 
-  temp_data = from(bucket: sourceBucket)
-  |> range(start: rangeStart, stop: rangeEnd)
-  |> filter(fn: (r) => r["_measurement"] == "temp")
-  |> group(columns: ["device_wwn"])
-  |> toInt()
+from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "smart" )
+|> group(columns: ["device_wwn", "_field"])
+|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
+|> to(bucket: destBucket, org: destOrg)
 
-  temp_data
-  |> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
-  |> to(bucket: destBucket, org: destOrg)
+from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "temp")
+|> group(columns: ["device_wwn"])
+|> toInt()
+|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
+|> set(key: "_measurement", value: "temp")
+|> set(key: "_field", value: "temp")
+|> to(bucket: destBucket, org: destOrg)
 		`,
+		name,
+		cron,
 		sourceBucket,
 		rangeStart,
 		rangeEnd,

webapp/backend/pkg/database/scrutiny_repository_tasks_test.go

@@ -0,0 +1,164 @@
+package database
+
+import (
+	mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func Test_DownsampleScript_Weekly(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+
+	deviceRepo := scrutinyRepository{
+		appConfig: fakeConfig,
+	}
+
+	aggregationType := "weekly"
+
+	//test
+	influxDbScript := deviceRepo.DownsampleScript(aggregationType, "tsk-weekly-aggr", "0 1 * * 0")
+
+	//assert
+	require.Equal(t, `
+option task = { 
+  name: "tsk-weekly-aggr",
+  cron: "0 1 * * 0",
+}
+
+sourceBucket = "metrics"
+rangeStart = -2w
+rangeEnd = -1w
+aggWindow = 1w
+destBucket = "metrics_weekly"
+destOrg = "scrutiny"
+
+from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "smart" )
+|> group(columns: ["device_wwn", "_field"])
+|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
+|> to(bucket: destBucket, org: destOrg)
+
+from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "temp")
+|> group(columns: ["device_wwn"])
+|> toInt()
+|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
+|> set(key: "_measurement", value: "temp")
+|> set(key: "_field", value: "temp")
+|> to(bucket: destBucket, org: destOrg)
+		`, influxDbScript)
+}
+
+func Test_DownsampleScript_Monthly(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+
+	deviceRepo := scrutinyRepository{
+		appConfig: fakeConfig,
+	}
+
+	aggregationType := "monthly"
+
+	//test
+	influxDbScript := deviceRepo.DownsampleScript(aggregationType, "tsk-monthly-aggr", "30 1 1 * *")
+
+	//assert
+	require.Equal(t, `
+option task = { 
+  name: "tsk-monthly-aggr",
+  cron: "30 1 1 * *",
+}
+
+sourceBucket = "metrics_weekly"
+rangeStart = -2mo
+rangeEnd = -1mo
+aggWindow = 1mo
+destBucket = "metrics_monthly"
+destOrg = "scrutiny"
+
+from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "smart" )
+|> group(columns: ["device_wwn", "_field"])
+|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
+|> to(bucket: destBucket, org: destOrg)
+
+from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "temp")
+|> group(columns: ["device_wwn"])
+|> toInt()
+|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
+|> set(key: "_measurement", value: "temp")
+|> set(key: "_field", value: "temp")
+|> to(bucket: destBucket, org: destOrg)
+		`, influxDbScript)
+}
+
+func Test_DownsampleScript_Yearly(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+
+	deviceRepo := scrutinyRepository{
+		appConfig: fakeConfig,
+	}
+
+	aggregationType := "yearly"
+
+	//test
+	influxDbScript := deviceRepo.DownsampleScript(aggregationType, "tsk-yearly-aggr", "0 2 1 1 *")
+
+	//assert
+	require.Equal(t, `
+option task = { 
+  name: "tsk-yearly-aggr",
+  cron: "0 2 1 1 *",
+}
+
+sourceBucket = "metrics_monthly"
+rangeStart = -2y
+rangeEnd = -1y
+aggWindow = 1y
+destBucket = "metrics_yearly"
+destOrg = "scrutiny"
+
+from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "smart" )
+|> group(columns: ["device_wwn", "_field"])
+|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
+|> to(bucket: destBucket, org: destOrg)
+
+from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "temp")
+|> group(columns: ["device_wwn"])
+|> toInt()
+|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
+|> set(key: "_measurement", value: "temp")
+|> set(key: "_field", value: "temp")
+|> to(bucket: destBucket, org: destOrg)
+		`, influxDbScript)
+}

webapp/backend/pkg/database/scrutiny_repository_temperature_test.go

@@ -0,0 +1,185 @@
+package database
+
+import (
+	mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func Test_aggregateTempQuery_Week(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+
+	deviceRepo := scrutinyRepository{
+		appConfig: fakeConfig,
+	}
+
+	aggregationType := DURATION_KEY_WEEK
+
+	//test
+	influxDbScript := deviceRepo.aggregateTempQuery(aggregationType)
+
+	//assert
+	require.Equal(t, `import "influxdata/influxdb/schema"
+weekData = from(bucket: "metrics")
+|> range(start: -1w, stop: now())
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+weekData
+|> schema.fieldsAsCols()
+|> yield()`, influxDbScript)
+}
+
+func Test_aggregateTempQuery_Month(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+
+	deviceRepo := scrutinyRepository{
+		appConfig: fakeConfig,
+	}
+
+	aggregationType := DURATION_KEY_MONTH
+
+	//test
+	influxDbScript := deviceRepo.aggregateTempQuery(aggregationType)
+
+	//assert
+	require.Equal(t, `import "influxdata/influxdb/schema"
+weekData = from(bucket: "metrics")
+|> range(start: -1w, stop: now())
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+monthData = from(bucket: "metrics_weekly")
+|> range(start: -1mo, stop: -1w)
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+union(tables: [weekData, monthData])
+|> group(columns: ["device_wwn"])
+|> sort(columns: ["_time"], desc: false)
+|> schema.fieldsAsCols()`, influxDbScript)
+}
+
+func Test_aggregateTempQuery_Year(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+
+	deviceRepo := scrutinyRepository{
+		appConfig: fakeConfig,
+	}
+
+	aggregationType := DURATION_KEY_YEAR
+
+	//test
+	influxDbScript := deviceRepo.aggregateTempQuery(aggregationType)
+
+	//assert
+	require.Equal(t, `import "influxdata/influxdb/schema"
+weekData = from(bucket: "metrics")
+|> range(start: -1w, stop: now())
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+monthData = from(bucket: "metrics_weekly")
+|> range(start: -1mo, stop: -1w)
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+yearData = from(bucket: "metrics_monthly")
+|> range(start: -1y, stop: -1mo)
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+union(tables: [weekData, monthData, yearData])
+|> group(columns: ["device_wwn"])
+|> sort(columns: ["_time"], desc: false)
+|> schema.fieldsAsCols()`, influxDbScript)
+}
+
+func Test_aggregateTempQuery_Forever(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+
+	deviceRepo := scrutinyRepository{
+		appConfig: fakeConfig,
+	}
+
+	aggregationType := DURATION_KEY_FOREVER
+
+	//test
+	influxDbScript := deviceRepo.aggregateTempQuery(aggregationType)
+
+	//assert
+	require.Equal(t, `import "influxdata/influxdb/schema"
+weekData = from(bucket: "metrics")
+|> range(start: -1w, stop: now())
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+monthData = from(bucket: "metrics_weekly")
+|> range(start: -1mo, stop: -1w)
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+yearData = from(bucket: "metrics_monthly")
+|> range(start: -1y, stop: -1mo)
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+foreverData = from(bucket: "metrics_yearly")
+|> range(start: -10y, stop: -1y)
+|> filter(fn: (r) => r["_measurement"] == "temp" )
+|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+|> group(columns: ["device_wwn"])
+|> toInt()
+
+union(tables: [weekData, monthData, yearData, foreverData])
+|> group(columns: ["device_wwn"])
+|> sort(columns: ["_time"], desc: false)
+|> schema.fieldsAsCols()`, influxDbScript)
+}

webapp/backend/pkg/models/setting.go

@@ -1,5 +0,0 @@
-package models
-
-// Temperature Format
-// Date Format
-// Device History window

webapp/backend/pkg/models/setting_entry.go

@@ -0,0 +1,23 @@
+package models
+
+import (
+	"gorm.io/gorm"
+)
+
+// SettingEntry matches a setting row in the database
+type SettingEntry struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	gorm.Model
+
+	SettingKeyName        string `json:"setting_key_name" gorm:"unique;not null"`
+	SettingKeyDescription string `json:"setting_key_description"`
+	SettingDataType       string `json:"setting_data_type"`
+
+	SettingValueNumeric int    `json:"setting_value_numeric"`
+	SettingValueString  string `json:"setting_value_string"`
+	SettingValueBool    bool   `json:"setting_value_bool"`
+}
+
+func (s SettingEntry) TableName() string {
+	return "settings"
+}

webapp/backend/pkg/models/settings.go

@@ -0,0 +1,23 @@
+package models
+
+// Settings is made up of parsed SettingEntry objects retrieved from the database
+//type Settings struct {
+//	MetricsNotifyLevel            pkg.MetricsNotifyLevel            `json:"metrics.notify.level" mapstructure:"metrics.notify.level"`
+//	MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics.status.filter_attributes" mapstructure:"metrics.status.filter_attributes"`
+//	MetricsStatusThreshold        pkg.MetricsStatusThreshold        `json:"metrics.status.threshold" mapstructure:"metrics.status.threshold"`
+//}
+
+type Settings struct {
+	Theme            string `json:"theme" mapstructure:"theme"`
+	Layout           string `json:"layout" mapstructure:"layout"`
+	DashboardDisplay string `json:"dashboard_display" mapstructure:"dashboard_display"`
+	DashboardSort    string `json:"dashboard_sort" mapstructure:"dashboard_sort"`
+	TemperatureUnit  string `json:"temperature_unit" mapstructure:"temperature_unit"`
+	FileSizeSIUnits  bool   `json:"file_size_si_units" mapstructure:"file_size_si_units"`
+
+	Metrics struct {
+		NotifyLevel            int `json:"notify_level" mapstructure:"notify_level"`
+		StatusFilterAttributes int `json:"status_filter_attributes" mapstructure:"status_filter_attributes"`
+		StatusThreshold        int `json:"status_threshold" mapstructure:"status_threshold"`
+	} `json:"metrics" mapstructure:"metrics"`
+}

webapp/backend/pkg/notify/notify.go

@@ -29,20 +29,22 @@ const NotifyFailureTypeSmartFailure = "SmartFailure"
 const NotifyFailureTypeScrutinyFailure = "ScrutinyFailure"
 
 // ShouldNotify check if the error Message should be filtered (level mismatch or filtered_attributes)
-func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLevel string, notifyFilterAttributes string) bool {
+func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThreshold pkg.MetricsStatusThreshold, statusFilterAttributes pkg.MetricsStatusFilterAttributes) bool {
 	// 1. check if the device is healthy
 	if device.DeviceStatus == pkg.DeviceStatusPassed {
 		return false
 	}
 
+	//TODO: cannot check for warning notifyLevel yet.
+
 	// setup constants for comparison
 	var requiredDeviceStatus pkg.DeviceStatus
 	var requiredAttrStatus pkg.AttributeStatus
-	if notifyLevel == pkg.NotifyLevelFail {
+	if statusThreshold == pkg.MetricsStatusThresholdBoth {
 		// either scrutiny or smart failures should trigger an email
 		requiredDeviceStatus = pkg.DeviceStatusSet(pkg.DeviceStatusFailedSmart, pkg.DeviceStatusFailedScrutiny)
 		requiredAttrStatus = pkg.AttributeStatusSet(pkg.AttributeStatusFailedSmart, pkg.AttributeStatusFailedScrutiny)
-	} else if notifyLevel == pkg.NotifyLevelFailSmart {
+	} else if statusThreshold == pkg.MetricsStatusThresholdSmart {
 		//only smart failures
 		requiredDeviceStatus = pkg.DeviceStatusFailedSmart
 		requiredAttrStatus = pkg.AttributeStatusFailedSmart
@@ -53,9 +55,9 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev
 
 	// 2. check if the attributes that are failing should be filtered (non-critical)
 	// 3. for any unfiltered attribute, store the failure reason (Smart or Scrutiny)
-	if notifyFilterAttributes == pkg.NotifyFilterAttributesCritical {
+	if statusFilterAttributes == pkg.MetricsStatusFilterAttributesCritical {
 		hasFailingCriticalAttr := false
-		var statusFailingCrtiticalAttr pkg.AttributeStatus
+		var statusFailingCriticalAttr pkg.AttributeStatus
 
 		for attrId, attrData := range smartAttrs.Attributes {
 			//find failing attribute
@@ -64,7 +66,7 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev
 			}
 
 			// merge the status's of all critical attributes
-			statusFailingCrtiticalAttr = pkg.AttributeStatusSet(statusFailingCrtiticalAttr, attrData.GetStatus())
+			statusFailingCriticalAttr = pkg.AttributeStatusSet(statusFailingCriticalAttr, attrData.GetStatus())
 
 			//found a failing attribute, see if its critical
 			if device.IsScsi() && thresholds.ScsiMetadata[attrId].Critical {
@@ -89,7 +91,7 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev
 			return false
 		} else {
 			// check if any of the critical attributes have a status that we're looking for
-			return pkg.AttributeStatusHas(statusFailingCrtiticalAttr, requiredAttrStatus)
+			return pkg.AttributeStatusHas(statusFailingCriticalAttr, requiredAttrStatus)
 		}
 
 	} else {
@@ -99,12 +101,13 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev
 	}
 }
 
-// TODO: include host and/or user label for device.
+// TODO: include user label for device.
 type Payload struct {
-	DeviceType   string `json:"device_type"`   //ATA/SCSI/NVMe
-	DeviceName   string `json:"device_name"`   //dev/sda
-	DeviceSerial string `json:"device_serial"` //WDDJ324KSO
-	Test         bool   `json:"test"`          // false
+	HostId       string `json:"host_id,omitempty"` //host id (optional)
+	DeviceType   string `json:"device_type"`       //ATA/SCSI/NVMe
+	DeviceName   string `json:"device_name"`       //dev/sda
+	DeviceSerial string `json:"device_serial"`     //WDDJ324KSO
+	Test         bool   `json:"test"`              // false
 
 	//private, populated during init (marked as Public for JSON serialization)
 	Date        string `json:"date"`         //populated by Send function.
@@ -113,8 +116,9 @@ type Payload struct {
 	Message     string `json:"message"`
 }
 
-func NewPayload(device models.Device, test bool) Payload {
+func NewPayload(device models.Device, test bool, currentTime ...time.Time) Payload {
 	payload := Payload{
+		HostId:       strings.TrimSpace(device.HostId),
 		DeviceType:   device.DeviceType,
 		DeviceName:   device.DeviceName,
 		DeviceSerial: device.SerialNumber,
@@ -122,7 +126,13 @@ func NewPayload(device models.Device, test bool) Payload {
 	}
 
 	//validate that the Payload is populated
-	sendDate := time.Now()
+	var sendDate time.Time
+	if currentTime != nil && len(currentTime) > 0 {
+		sendDate = currentTime[0]
+	} else {
+		sendDate = time.Now()
+	}
+
 	payload.Date = sendDate.Format(time.RFC3339)
 	payload.FailureType = payload.GenerateFailureType(device.DeviceStatus)
 	payload.Subject = payload.GenerateSubject()
@@ -146,25 +156,39 @@ func (p *Payload) GenerateFailureType(deviceStatus pkg.DeviceStatus) string {
 
 func (p *Payload) GenerateSubject() string {
 	//generate a detailed failure message
-	return fmt.Sprintf("Scrutiny SMART error (%s) detected on device: %s", p.FailureType, p.DeviceName)
+	var subject string
+	if len(p.HostId) > 0 {
+		subject = fmt.Sprintf("Scrutiny SMART error (%s) detected on [host]device: [%s]%s", p.FailureType, p.HostId, p.DeviceName)
+	} else {
+		subject = fmt.Sprintf("Scrutiny SMART error (%s) detected on device: %s", p.FailureType, p.DeviceName)
+	}
+	return subject
 }
 
 func (p *Payload) GenerateMessage() string {
 	//generate a detailed failure message
-	message := fmt.Sprintf(
-		`Scrutiny SMART error notification for device: %s
-Failure Type: %s
-Device Name: %s
-Device Serial: %s
-Device Type: %s
 
-Date: %s`, p.DeviceName, p.FailureType, p.DeviceName, p.DeviceSerial, p.DeviceType, p.Date)
+	messageParts := []string{}
 
-	if p.Test {
-		message = "TEST NOTIFICATION:\n" + message
+	messageParts = append(messageParts, fmt.Sprintf("Scrutiny SMART error notification for device: %s", p.DeviceName))
+	if len(p.HostId) > 0 {
+		messageParts = append(messageParts, fmt.Sprintf("Host Id: %s", p.HostId))
 	}
 
-	return message
+	messageParts = append(messageParts,
+		fmt.Sprintf("Failure Type: %s", p.FailureType),
+		fmt.Sprintf("Device Name: %s", p.DeviceName),
+		fmt.Sprintf("Device Serial: %s", p.DeviceSerial),
+		fmt.Sprintf("Device Type: %s", p.DeviceType),
+		"",
+		fmt.Sprintf("Date: %s", p.Date),
+	)
+
+	if p.Test {
+		messageParts = append([]string{"TEST NOTIFICATION:"}, messageParts...)
+	}
+
+	return strings.Join(messageParts, "\n")
 }
 
 func New(logger logrus.FieldLogger, appconfig config.Interface, device models.Device, test bool) Notify {
@@ -285,6 +309,9 @@ func (n *Notify) SendScriptNotification(scriptUrl string) error {
 	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_TYPE=%s", n.Payload.DeviceType))
 	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_SERIAL=%s", n.Payload.DeviceSerial))
 	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.Message))
+	if len(n.Payload.HostId) > 0 {
+		copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_HOST_ID=%s", n.Payload.HostId))
+	}
 	err := utils.CmdExec(scriptPath, []string{}, "", copyEnv, "")
 	if err != nil {
 		n.Logger.Errorf("An error occurred while executing script %s: %v", scriptPath, err)

webapp/backend/pkg/notify/notify_test.go

@@ -1,11 +1,13 @@
 package notify
 
 import (
+	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	"github.com/stretchr/testify/require"
 	"testing"
+	"time"
 )
 
 func TestShouldNotify_MustSkipPassingDevices(t *testing.T) {
@@ -15,56 +17,56 @@ func TestShouldNotify_MustSkipPassingDevices(t *testing.T) {
 		DeviceStatus: pkg.DeviceStatusPassed,
 	}
 	smartAttrs := measurements.Smart{}
-	notifyLevel := pkg.NotifyLevelFail
-	notifyFilterAttributes := pkg.NotifyFilterAttributesAll
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
+	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
 }
 
-func TestShouldNotify_NotifyLevelFail_FailingSmartDevice(t *testing.T) {
+func TestShouldNotify_MetricsStatusThresholdBoth_FailingSmartDevice(t *testing.T) {
 	t.Parallel()
 	//setup
 	device := models.Device{
 		DeviceStatus: pkg.DeviceStatusFailedSmart,
 	}
 	smartAttrs := measurements.Smart{}
-	notifyLevel := pkg.NotifyLevelFail
-	notifyFilterAttributes := pkg.NotifyFilterAttributesAll
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
 
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
+	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
 }
 
-func TestShouldNotify_NotifyLevelFailSmart_FailingSmartDevice(t *testing.T) {
+func TestShouldNotify_MetricsStatusThresholdSmart_FailingSmartDevice(t *testing.T) {
 	t.Parallel()
 	//setup
 	device := models.Device{
 		DeviceStatus: pkg.DeviceStatusFailedSmart,
 	}
 	smartAttrs := measurements.Smart{}
-	notifyLevel := pkg.NotifyLevelFailSmart
-	notifyFilterAttributes := pkg.NotifyFilterAttributesAll
+	statusThreshold := pkg.MetricsStatusThresholdSmart
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
 
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
+	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
 }
 
-func TestShouldNotify_NotifyLevelFailScrutiny_FailingSmartDevice(t *testing.T) {
+func TestShouldNotify_MetricsStatusThresholdScrutiny_FailingSmartDevice(t *testing.T) {
 	t.Parallel()
 	//setup
 	device := models.Device{
 		DeviceStatus: pkg.DeviceStatusFailedSmart,
 	}
 	smartAttrs := measurements.Smart{}
-	notifyLevel := pkg.NotifyLevelFailScrutiny
-	notifyFilterAttributes := pkg.NotifyFilterAttributesAll
+	statusThreshold := pkg.MetricsStatusThresholdScrutiny
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
+	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
 }
 
-func TestShouldNotify_NotifyFilterAttributesCritical_WithCriticalAttrs(t *testing.T) {
+func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithCriticalAttrs(t *testing.T) {
 	t.Parallel()
 	//setup
 	device := models.Device{
@@ -75,14 +77,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithCriticalAttrs(t *testin
 			Status: pkg.AttributeStatusFailedSmart,
 		},
 	}}
-	notifyLevel := pkg.NotifyLevelFail
-	notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
 
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
+	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
 }
 
-func TestShouldNotify_NotifyFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) {
+func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) {
 	t.Parallel()
 	//setup
 	device := models.Device{
@@ -96,14 +98,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithMultipleCriticalAttrs(t
 			Status: pkg.AttributeStatusFailedScrutiny,
 		},
 	}}
-	notifyLevel := pkg.NotifyLevelFail
-	notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
 
 	//assert
-	require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
+	require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
 }
 
-func TestShouldNotify_NotifyFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) {
+func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) {
 	t.Parallel()
 	//setup
 	device := models.Device{
@@ -114,14 +116,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithNoCriticalAttrs(t *test
 			Status: pkg.AttributeStatusFailedSmart,
 		},
 	}}
-	notifyLevel := pkg.NotifyLevelFail
-	notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
+	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
 }
 
-func TestShouldNotify_NotifyFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) {
+func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) {
 	t.Parallel()
 	//setup
 	device := models.Device{
@@ -132,14 +134,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithNoFailingCriticalAttrs(
 			Status: pkg.AttributeStatusPassed,
 		},
 	}}
-	notifyLevel := pkg.NotifyLevelFail
-	notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
+	statusThreshold := pkg.MetricsStatusThresholdBoth
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
+	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
 }
 
-func TestShouldNotify_NotifyFilterAttributesCritical_NotifyLevelFailSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) {
+func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresholdSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) {
 	t.Parallel()
 	//setup
 	device := models.Device{
@@ -153,9 +155,90 @@ func TestShouldNotify_NotifyFilterAttributesCritical_NotifyLevelFailSmart_WithCr
 			Status: pkg.AttributeStatusFailedScrutiny,
 		},
 	}}
-	notifyLevel := pkg.NotifyLevelFailSmart
-	notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
+	statusThreshold := pkg.MetricsStatusThresholdSmart
+	notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
 
 	//assert
-	require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
+	require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
+}
+
+func TestNewPayload(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	device := models.Device{
+		SerialNumber: "FAKEWDDJ324KSO",
+		DeviceType:   pkg.DeviceProtocolAta,
+		DeviceName:   "/dev/sda",
+		DeviceStatus: pkg.DeviceStatusFailedScrutiny,
+	}
+	currentTime := time.Now()
+	//test
+
+	payload := NewPayload(device, false, currentTime)
+
+	//assert
+	require.Equal(t, "Scrutiny SMART error (ScrutinyFailure) detected on device: /dev/sda", payload.Subject)
+	require.Equal(t, fmt.Sprintf(`Scrutiny SMART error notification for device: /dev/sda
+Failure Type: ScrutinyFailure
+Device Name: /dev/sda
+Device Serial: FAKEWDDJ324KSO
+Device Type: ATA
+
+Date: %s`, currentTime.Format(time.RFC3339)), payload.Message)
+}
+
+func TestNewPayload_TestMode(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	device := models.Device{
+		SerialNumber: "FAKEWDDJ324KSO",
+		DeviceType:   pkg.DeviceProtocolAta,
+		DeviceName:   "/dev/sda",
+		DeviceStatus: pkg.DeviceStatusFailedScrutiny,
+	}
+	currentTime := time.Now()
+	//test
+
+	payload := NewPayload(device, true, currentTime)
+
+	//assert
+	require.Equal(t, "Scrutiny SMART error (EmailTest) detected on device: /dev/sda", payload.Subject)
+	require.Equal(t, fmt.Sprintf(`TEST NOTIFICATION:
+Scrutiny SMART error notification for device: /dev/sda
+Failure Type: EmailTest
+Device Name: /dev/sda
+Device Serial: FAKEWDDJ324KSO
+Device Type: ATA
+
+Date: %s`, currentTime.Format(time.RFC3339)), payload.Message)
+}
+
+func TestNewPayload_WithHostId(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	device := models.Device{
+		SerialNumber: "FAKEWDDJ324KSO",
+		DeviceType:   pkg.DeviceProtocolAta,
+		DeviceName:   "/dev/sda",
+		DeviceStatus: pkg.DeviceStatusFailedScrutiny,
+		HostId:       "custom-host",
+	}
+	currentTime := time.Now()
+	//test
+
+	payload := NewPayload(device, false, currentTime)
+
+	//assert
+	require.Equal(t, "Scrutiny SMART error (ScrutinyFailure) detected on [host]device: [custom-host]/dev/sda", payload.Subject)
+	require.Equal(t, fmt.Sprintf(`Scrutiny SMART error notification for device: /dev/sda
+Host Id: custom-host
+Failure Type: ScrutinyFailure
+Device Name: /dev/sda
+Device Serial: FAKEWDDJ324KSO
+Device Type: ATA
+
+Date: %s`, currentTime.Format(time.RFC3339)), payload.Message)
 }

webapp/backend/pkg/version/version.go

@@ -2,4 +2,4 @@ package version
 
 // VERSION is the app-global version string, which will be replaced with a
 // new value during packaging
-const VERSION = "0.4.14"
+const VERSION = "0.5.0"

webapp/backend/pkg/web/handler/delete_device.go

@@ -8,7 +8,7 @@ import (
 )
 
 func DeleteDevice(c *gin.Context) {
-	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
 	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 
 	err := deviceRepo.DeleteDevice(c, c.Param("wwn"))

webapp/backend/pkg/web/handler/get_device_details.go

@@ -9,7 +9,7 @@ import (
 )
 
 func GetDeviceDetails(c *gin.Context) {
-	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
 	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 
 	device, err := deviceRepo.GetDeviceDetails(c, c.Param("wwn"))

webapp/backend/pkg/web/handler/get_devices_summary.go

@@ -8,7 +8,7 @@ import (
 )
 
 func GetDevicesSummary(c *gin.Context) {
-	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
 	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 
 	summary, err := deviceRepo.GetSummary(c)
@@ -18,6 +18,7 @@ func GetDevicesSummary(c *gin.Context) {
 		return
 	}
 
+	//this must match DeviceSummaryWrapper (webapp/backend/pkg/models/device_summary.go)
 	c.JSON(http.StatusOK, gin.H{
 		"success": true,
 		"data": map[string]interface{}{

webapp/backend/pkg/web/handler/get_devices_summary_temp_history.go

@@ -8,7 +8,7 @@ import (
 )
 
 func GetDevicesSummaryTempHistory(c *gin.Context) {
-	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
 	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 
 	durationKey, exists := c.GetQuery("duration_key")

webapp/backend/pkg/web/handler/get_settings.go

@@ -0,0 +1,25 @@
+package handler
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"net/http"
+)
+
+func GetSettings(c *gin.Context) {
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	settings, err := deviceRepo.LoadSettings(c)
+	if err != nil {
+		logger.Errorln("An error occurred while retrieving settings", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"success":  true,
+		"settings": settings,
+	})
+}

webapp/backend/pkg/web/handler/register_devices.go

@@ -4,6 +4,7 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/gin-gonic/gin"
+	"github.com/samber/lo"
 	"github.com/sirupsen/logrus"
 	"net/http"
 )
@@ -12,7 +13,7 @@ import (
 // This function is run everytime a collector is about to start a run. It can be used to update device metadata.
 func RegisterDevices(c *gin.Context) {
 	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
-	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
 
 	var collectorDeviceWrapper models.DeviceWrapper
 	err := c.BindJSON(&collectorDeviceWrapper)
@@ -22,8 +23,13 @@ func RegisterDevices(c *gin.Context) {
 		return
 	}
 
+	//filter any device with empty wwn (they are invalid)
+	detectedStorageDevices := lo.Filter[models.Device](collectorDeviceWrapper.Data, func(dev models.Device, _ int) bool {
+		return len(dev.WWN) > 0
+	})
+
 	errs := []error{}
-	for _, dev := range collectorDeviceWrapper.Data {
+	for _, dev := range detectedStorageDevices {
 		//insert devices into DB (and update specified columns if device is already registered)
 		// update device fields that may change: (DeviceType, HostID)
 		if err := deviceRepo.RegisterDevice(c, dev); err != nil {
@@ -40,7 +46,7 @@ func RegisterDevices(c *gin.Context) {
 	} else {
 		c.JSON(http.StatusOK, models.DeviceWrapper{
 			Success: true,
-			Data:    collectorDeviceWrapper.Data,
+			Data:    detectedStorageDevices,
 		})
 		return
 	}

webapp/backend/pkg/web/handler/save_settings.go

@@ -0,0 +1,34 @@
+package handler
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"net/http"
+)
+
+func SaveSettings(c *gin.Context) {
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	var settings models.Settings
+	err := c.BindJSON(&settings)
+	if err != nil {
+		logger.Errorln("Cannot parse updated settings", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	err = deviceRepo.SaveSettings(c, settings)
+	if err != nil {
+		logger.Errorln("An error occurred while saving settings", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"success":  true,
+		"settings": settings,
+	})
+}

webapp/backend/pkg/web/handler/send_test_notification.go

@@ -13,7 +13,7 @@ import (
 // Send test notification
 func SendTestNotification(c *gin.Context) {
 	appConfig := c.MustGet("CONFIG").(config.Interface)
-	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
 
 	testNotify := notify.New(
 		logger,

webapp/backend/pkg/web/handler/upload_device_metrics.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
@@ -13,13 +14,17 @@ import (
 
 func UploadDeviceMetrics(c *gin.Context) {
 	//db := c.MustGet("DB").(*gorm.DB)
-	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	logger := c.MustGet("LOGGER").(*logrus.Entry)
 	appConfig := c.MustGet("CONFIG").(config.Interface)
 	//influxWriteDb := c.MustGet("INFLUXDB_WRITE").(*api.WriteAPIBlocking)
 	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 
 	//appConfig := c.MustGet("CONFIG").(config.Interface)
 
+	if c.Param("wwn") == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"success": false})
+	}
+
 	var collectorSmartData collector.SmartInfo
 	err := c.BindJSON(&collectorSmartData)
 	if err != nil {
@@ -63,7 +68,12 @@ func UploadDeviceMetrics(c *gin.Context) {
 	}
 
 	//check for error
-	if notify.ShouldNotify(updatedDevice, smartData, appConfig.GetString("notify.level"), appConfig.GetString("notify.filter_attributes")) {
+	if notify.ShouldNotify(
+		updatedDevice,
+		smartData,
+		pkg.MetricsStatusThreshold(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY))),
+		pkg.MetricsStatusFilterAttributes(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY))),
+	) {
 		//send notifications
 
 		liveNotify := notify.New(

webapp/backend/pkg/web/middleware/logger.go

@@ -28,11 +28,11 @@ import (
 var timeFormat = "02/Jan/2006:15:04:05 -0700"
 
 // Logger is the logrus logger handler
-func LoggerMiddleware(logger logrus.FieldLogger) gin.HandlerFunc {
+func LoggerMiddleware(logger *logrus.Entry) gin.HandlerFunc {
 
 	hostname, err := os.Hostname()
 	if err != nil {
-		hostname = "unknow"
+		hostname = "unknown"
 	}
 
 	return func(c *gin.Context) {

webapp/backend/pkg/web/middleware/repository.go

@@ -1,6 +1,7 @@
 package middleware
 
 import (
+	"context"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/gin-gonic/gin"
@@ -14,6 +15,14 @@ func RepositoryMiddleware(appConfig config.Interface, globalLogger logrus.FieldL
 		panic(err)
 	}
 
+	// ensure the settings have been loaded into the app config during startup.
+	_, err = deviceRepo.LoadSettings(context.Background())
+	if err != nil {
+		panic(err)
+	}
+
+	//settings.UpdateSettingEntries()
+
 	//TODO: determine where we can call defer deviceRepo.Close()
 	return func(c *gin.Context) {
 		c.Set("DEVICE_REPOSITORY", deviceRepo)

webapp/backend/pkg/web/server.go

@@ -9,18 +9,17 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/web/middleware"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"io"
 	"net/http"
-	"os"
 	"path/filepath"
 	"strings"
 )
 
 type AppEngine struct {
 	Config config.Interface
+	Logger *logrus.Entry
 }
 
-func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
+func (ae *AppEngine) Setup(logger *logrus.Entry) *gin.Engine {
 	r := gin.New()
 
 	r.Use(middleware.LoggerMiddleware(logger))
@@ -36,6 +35,10 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 		api := base.Group("/api")
 		{
 			api.GET("/health", func(c *gin.Context) {
+				//TODO:
+				// check if the /web folder is populated.
+				// check if access to influxdb
+				// check if access to sqlitedb.
 				c.JSON(http.StatusOK, gin.H{
 					"success": true,
 				})
@@ -50,6 +53,8 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 			api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details
 			api.DELETE("/device/:wwn", handler.DeleteDevice)          //used by UI to delete device
 
+			api.GET("/settings", handler.GetSettings)   //used to get settings
+			api.POST("/settings", handler.SaveSettings) //used to save settings
 		}
 	}
 
@@ -75,26 +80,6 @@ func (ae *AppEngine) Start() error {
 		gin.SetMode(gin.DebugMode)
 	}
 
-	logger := logrus.New()
-	//set default log level
-	logLevel, err := logrus.ParseLevel(ae.Config.GetString("log.level"))
-	if err != nil {
-		return err
-	}
-	logger.SetLevel(logLevel)
-	//set the log file if present
-	if len(ae.Config.GetString("log.file")) != 0 {
-		logFile, err := os.OpenFile(ae.Config.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
-		defer logFile.Close()
-		if err != nil {
-			logrus.Errorf("Failed to open log file %s for output: %s", ae.Config.GetString("log.file"), err)
-			return err
-		}
-
-		//configure the logrus default
-		logger.SetOutput(io.MultiWriter(os.Stderr, logFile))
-	}
-
 	//check if the database parent directory exists, fail here rather than in a handler.
 	if !utils.FileExists(filepath.Dir(ae.Config.GetString("web.database.location"))) {
 		return errors.ConfigValidationError(fmt.Sprintf(
@@ -102,7 +87,7 @@ func (ae *AppEngine) Start() error {
 			filepath.Dir(ae.Config.GetString("web.database.location"))))
 	}
 
-	r := ae.Setup(logger)
+	r := ae.Setup(ae.Logger)
 
 	return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port")))
 }

webapp/backend/pkg/web/server_test.go

@@ -3,7 +3,9 @@ package web_test
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
@@ -89,6 +91,8 @@ func (suite *ServerTestSuite) TestHealthRoute() {
 	mockCtrl := gomock.NewController(suite.T())
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes()
+	fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
@@ -111,7 +115,7 @@ func (suite *ServerTestSuite) TestHealthRoute() {
 		Config: fakeConfig,
 	}
 
-	router := ae.Setup(logrus.New())
+	router := ae.Setup(logrus.WithField("test", suite.T().Name()))
 
 	//test
 	w := httptest.NewRecorder()
@@ -130,6 +134,8 @@ func (suite *ServerTestSuite) TestRegisterDevicesRoute() {
 	mockCtrl := gomock.NewController(suite.T())
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes()
+	fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes()
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
@@ -150,7 +156,7 @@ func (suite *ServerTestSuite) TestRegisterDevicesRoute() {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup(logrus.New())
+	router := ae.Setup(logrus.WithField("test", suite.T().Name()))
 	file, err := os.Open("testdata/register-devices-req.json")
 	require.NoError(suite.T(), err)
 
@@ -170,6 +176,8 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() {
 	mockCtrl := gomock.NewController(suite.T())
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes()
+	fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
@@ -186,13 +194,14 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() {
 	} else {
 		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
 	}
-	fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail)
-	fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll)
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth))
 
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup(logrus.New())
+	router := ae.Setup(logrus.WithField("test", suite.T().Name()))
 	devicesfile, err := os.Open("testdata/register-devices-single-req.json")
 	require.NoError(suite.T(), err)
 
@@ -219,10 +228,13 @@ func (suite *ServerTestSuite) TestPopulateMultiple() {
 	mockCtrl := gomock.NewController(suite.T())
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes()
+	fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	//fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
 	fakeConfig.EXPECT().GetStringSlice("notify.urls").Return([]string{}).AnyTimes()
-	fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail)
-	fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll)
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth))
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
@@ -243,7 +255,7 @@ func (suite *ServerTestSuite) TestPopulateMultiple() {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup(logrus.New())
+	router := ae.Setup(logrus.WithField("test", suite.T().Name()))
 	devicesfile, err := os.Open("testdata/register-devices-req.json")
 	require.NoError(suite.T(), err)
 
@@ -319,6 +331,8 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() {
 	mockCtrl := gomock.NewController(suite.T())
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes()
+	fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
@@ -330,8 +344,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() {
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://unroutable.domain.example.asdfghj"})
-	fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail)
-	fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll)
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth))
 
 	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
 		// when running test suite in github actions, we run an influxdb service as a sidecar.
@@ -343,7 +358,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup(logrus.New())
+	router := ae.Setup(logrus.WithField("test", suite.T().Name()))
 
 	//test
 	wr := httptest.NewRecorder()
@@ -361,6 +376,8 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() {
 	mockCtrl := gomock.NewController(suite.T())
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes()
+	fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
@@ -372,8 +389,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() {
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///missing/path/on/disk"})
-	fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail)
-	fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll)
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth))
 
 	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
 		// when running test suite in github actions, we run an influxdb service as a sidecar.
@@ -385,7 +403,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup(logrus.New())
+	router := ae.Setup(logrus.WithField("test", suite.T().Name()))
 
 	//test
 	wr := httptest.NewRecorder()
@@ -403,6 +421,8 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() {
 	mockCtrl := gomock.NewController(suite.T())
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes()
+	fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
@@ -414,8 +434,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() {
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///usr/bin/env"})
-	fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail)
-	fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll)
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth))
 
 	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
 		// when running test suite in github actions, we run an influxdb service as a sidecar.
@@ -427,7 +448,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup(logrus.New())
+	router := ae.Setup(logrus.WithField("test", suite.T().Name()))
 
 	//test
 	wr := httptest.NewRecorder()
@@ -445,6 +466,8 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() {
 	mockCtrl := gomock.NewController(suite.T())
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes()
+	fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
@@ -456,8 +479,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() {
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"discord://invalidtoken@channel"})
-	fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail)
-	fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll)
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth))
 
 	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
 		// when running test suite in github actions, we run an influxdb service as a sidecar.
@@ -468,7 +492,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup(logrus.New())
+	router := ae.Setup(logrus.WithField("test", suite.T().Name()))
 
 	//test
 	wr := httptest.NewRecorder()
@@ -486,6 +510,8 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() {
 	mockCtrl := gomock.NewController(suite.T())
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes()
+	fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
 	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes()
@@ -497,8 +523,10 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() {
 	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
 	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{})
-	fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail)
-	fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll)
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll))
+	fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth))
+
 	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
 		// when running test suite in github actions, we run an influxdb service as a sidecar.
 		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
@@ -509,7 +537,7 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup(logrus.New())
+	router := ae.Setup(logrus.WithField("test", suite.T().Name()))
 	devicesfile, err := os.Open("testdata/register-devices-req-2.json")
 	require.NoError(suite.T(), err)
 

webapp/frontend/.gitignore

@@ -46,3 +46,5 @@ testem.log
 Thumbs.db
 
 /dist
+
+/coverage

webapp/frontend/angular.json

@@ -91,6 +91,7 @@
                 },
                 "test": {
                     "builder": "@angular-devkit/build-angular:karma",
+                    "defaultConfiguration": "production",
                     "options": {
                         "main": "src/test.ts",
                         "polyfills": "src/polyfills.ts",
@@ -101,10 +102,22 @@
                             "src/favicon-32x32.png",
                             "src/assets"
                         ],
+                        "stylePreprocessorOptions": {
+                            "includePaths": [
+                                "src/@treo/styles"
+                            ]
+                        },
                         "styles": [
-                            "src/styles.scss"
+                            "src/styles/vendors.scss",
+                            "src/@treo/styles/main.scss",
+                            "src/styles/styles.scss",
+                            "src/styles/tailwind.scss"
                         ],
-                        "scripts": []
+                        "scripts": [],
+                        "fileReplacements": [{
+                            "replace": "src/environments/environment.ts",
+                            "with": "src/environments/environment.prod.ts"
+                        }]
                     }
                 },
                 "lint": {

webapp/frontend/karma.conf.js

@@ -17,8 +17,8 @@ module.exports = function (config)
             clearContext: false // leave Jasmine Spec Runner output visible in browser
         },
         coverageIstanbulReporter: {
-            dir                  : require('path').join(__dirname, './coverage/treo'),
-            reports              : ['html', 'lcovonly', 'text-summary'],
+            dir: require('path').join(__dirname, './coverage'),
+            reports: ['html', 'lcovonly', 'text-summary'],
             fixWebpackSourcePaths: true
         },
         reporters               : ['progress', 'kjhtml'],

webapp/frontend/src/app/app.module.ts

@@ -1,22 +1,22 @@
-import { NgModule, enableProdMode } from '@angular/core';
-import { BrowserModule } from '@angular/platform-browser';
-import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
-import { ExtraOptions, PreloadAllModules, RouterModule } from '@angular/router';
-import { APP_BASE_HREF } from '@angular/common';
-import { MarkdownModule } from 'ngx-markdown';
-import { TreoModule } from '@treo';
-import { TreoConfigModule } from '@treo/services/config';
-import { TreoMockApiModule } from '@treo/lib/mock-api';
-import { CoreModule } from 'app/core/core.module';
-import { appConfig } from 'app/core/config/app.config';
-import { mockDataServices } from 'app/data/mock';
-import { LayoutModule } from 'app/layout/layout.module';
-import { AppComponent } from 'app/app.component';
-import { appRoutes, getAppBaseHref } from 'app/app.routing';
+import {enableProdMode, NgModule} from '@angular/core';
+import {BrowserModule} from '@angular/platform-browser';
+import {BrowserAnimationsModule} from '@angular/platform-browser/animations';
+import {ExtraOptions, PreloadAllModules, RouterModule} from '@angular/router';
+import {APP_BASE_HREF} from '@angular/common';
+import {MarkdownModule} from 'ngx-markdown';
+import {TreoModule} from '@treo';
+import {ScrutinyConfigModule} from 'app/core/config/scrutiny-config.module';
+import {TreoMockApiModule} from '@treo/lib/mock-api';
+import {CoreModule} from 'app/core/core.module';
+import {appConfig} from 'app/core/config/app.config';
+import {mockDataServices} from 'app/data/mock';
+import {LayoutModule} from 'app/layout/layout.module';
+import {AppComponent} from 'app/app.component';
+import {appRoutes, getAppBaseHref} from 'app/app.routing';
 
 const routerConfig: ExtraOptions = {
     scrollPositionRestoration: 'enabled',
-    preloadingStrategy       : PreloadAllModules
+    preloadingStrategy: PreloadAllModules
 };
 
 let dev = [
@@ -41,7 +41,7 @@ if (process.env.NODE_ENV === 'production') {
 
         // Treo & Treo Mock API
         TreoModule,
-        TreoConfigModule.forRoot(appConfig),
+        ScrutinyConfigModule.forRoot(appConfig),
         ...dev,
 
         // Core

webapp/frontend/src/app/core/config/app.config.ts

@@ -1,22 +1,58 @@
-import { Layout } from 'app/layout/layout.types';
+import {Layout} from 'app/layout/layout.types';
 
 // Theme type
 export type Theme = 'light' | 'dark' | 'system';
 
+// Device title to display on the dashboard
+export type DashboardDisplay = 'name' | 'serial_id' | 'uuid' | 'label'
+
+export type DashboardSort = 'status' | 'title' | 'age'
+
+export type TemperatureUnit = 'celsius' | 'fahrenheit'
+
+
+export enum MetricsNotifyLevel {
+    Warn = 1,
+    Fail = 2
+}
+
+export enum MetricsStatusFilterAttributes {
+    All = 0,
+    Critical = 1
+}
+
+export enum MetricsStatusThreshold {
+    Smart = 1,
+    Scrutiny = 2,
+
+    // shortcut
+    Both = 3
+}
+
 /**
  * AppConfig interface. Update this interface to strictly type your config
  * object.
  */
-export interface AppConfig
-{
-    theme: Theme;
-    layout: Layout;
+export interface AppConfig {
+    theme?: Theme;
+    layout?: Layout;
 
     // Dashboard options
-    dashboardDisplay: string;
-    dashboardSort: string;
+    dashboard_display?: DashboardDisplay;
+    dashboard_sort?: DashboardSort;
+
+    temperature_unit?: TemperatureUnit;
+
+    file_size_si_units?: boolean;
+
+    // Settings from Scrutiny API
+
+    metrics?: {
+        notify_level?: MetricsNotifyLevel
+        status_filter_attributes?: MetricsStatusFilterAttributes
+        status_threshold?: MetricsStatusThreshold
+    }
 
-    temperatureUnit: string;
 }
 
 /**
@@ -28,12 +64,19 @@ export interface AppConfig
  * "ConfigService".
  */
 export const appConfig: AppConfig = {
-    theme : 'light',
+    theme: 'light',
     layout: 'material',
 
-    dashboardDisplay: 'name',
-    dashboardSort: 'status',
+    dashboard_display: 'name',
+    dashboard_sort: 'status',
 
-    temperatureUnit: 'celsius',
+    temperature_unit: 'celsius',
+    file_size_si_units: false,
+
+    metrics: {
+        notify_level: MetricsNotifyLevel.Fail,
+        status_filter_attributes: MetricsStatusFilterAttributes.All,
+        status_threshold: MetricsStatusThreshold.Both
+    }
 };
 

webapp/frontend/src/app/core/config/scrutiny-config.module.ts

@@ -0,0 +1,33 @@
+import {ModuleWithProviders, NgModule} from '@angular/core';
+import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service';
+import {TREO_APP_CONFIG} from '@treo/services/config/config.constants';
+
+@NgModule()
+export class ScrutinyConfigModule {
+    /**
+     * Constructor
+     *
+     * @param {ScrutinyConfigService} _scrutinyConfigService
+     */
+    constructor(
+        private _scrutinyConfigService: ScrutinyConfigService
+    ) {
+    }
+
+    /**
+     * forRoot method for setting user configuration
+     *
+     * @param config
+     */
+    static forRoot(config: any): ModuleWithProviders {
+        return {
+            ngModule: ScrutinyConfigModule,
+            providers: [
+                {
+                    provide: TREO_APP_CONFIG,
+                    useValue: config
+                }
+            ]
+        };
+    }
+}

webapp/frontend/src/app/core/config/scrutiny-config.service.ts

@@ -0,0 +1,84 @@
+import {Inject, Injectable} from '@angular/core';
+import {HttpClient} from '@angular/common/http';
+import {TREO_APP_CONFIG} from '@treo/services/config/config.constants';
+import {BehaviorSubject, Observable} from 'rxjs';
+import {getBasePath} from '../../app.routing';
+import {map, tap} from 'rxjs/operators';
+import {AppConfig} from './app.config';
+import {merge} from 'lodash';
+
+@Injectable({
+    providedIn: 'root'
+})
+export class ScrutinyConfigService {
+    // Private
+    private _config: BehaviorSubject<AppConfig>;
+    private _defaultConfig: AppConfig;
+
+    constructor(
+        private _httpClient: HttpClient,
+        @Inject(TREO_APP_CONFIG) defaultConfig: AppConfig
+    ) {
+        // Set the private defaults
+        this._defaultConfig = defaultConfig
+        this._config = new BehaviorSubject(null);
+    }
+
+
+    // -----------------------------------------------------------------------------------------------------
+    // @ Accessors
+    // -----------------------------------------------------------------------------------------------------
+
+    /**
+     * Setter & getter for config
+     */
+    set config(value: AppConfig) {
+        // get the current config, merge the new values, and then submit. (setTheme only sets a single key, not the whole obj)
+        const mergedSettings = merge({}, this._config.getValue(), value);
+
+        console.log('saving settings...', mergedSettings)
+        this._httpClient.post(getBasePath() + '/api/settings', mergedSettings).pipe(
+            map((response: any) => {
+                console.log('settings resp')
+                return response.settings
+            }),
+            tap((settings: AppConfig) => {
+                this._config.next(settings);
+                return settings
+            })
+        ).subscribe(resp => {
+            console.log('updated settings', resp)
+        })
+    }
+
+    get config$(): Observable<AppConfig> {
+        if (this._config.getValue()) {
+            console.log('using cached settings:', this._config.getValue())
+            return this._config.asObservable()
+        } else {
+            console.log('retrieving settings')
+            return this._httpClient.get(getBasePath() + '/api/settings').pipe(
+                map((response: any) => {
+                    return response.settings
+                }),
+                tap((settings: AppConfig) => {
+                    this._config.next(settings);
+                    return this._config.asObservable()
+                })
+            );
+        }
+
+    }
+
+    // -----------------------------------------------------------------------------------------------------
+    // @ Public methods
+    // -----------------------------------------------------------------------------------------------------
+
+    /**
+     * Resets the config to the default
+     */
+    reset(): void {
+        // Set the config
+        this.config = this._defaultConfig
+    }
+}

webapp/frontend/src/app/core/models/device-details-response-wrapper.ts

@@ -0,0 +1,14 @@
+import {DeviceModel} from 'app/core/models/device-model';
+import {SmartModel} from 'app/core/models/measurements/smart-model';
+import {AttributeMetadataModel} from 'app/core/models/thresholds/attribute-metadata-model';
+
+// maps to webapp/backend/pkg/models/device_summary.go
+export interface DeviceDetailsResponseWrapper {
+    success: boolean;
+    errors?: any[];
+    data: {
+        device: DeviceModel;
+        smart_results: SmartModel[];
+    },
+    metadata: { [key: string]: AttributeMetadataModel } | { [key: number]: AttributeMetadataModel };
+}

webapp/frontend/src/app/core/models/device-model.ts

@@ -0,0 +1,26 @@
+// maps to webapp/backend/pkg/models/device.go
+export interface DeviceModel {
+    wwn: string;
+    device_name?: string;
+    device_uuid?: string;
+    device_serial_id?: string;
+    device_label?: string;
+
+    manufacturer: string;
+    model_name: string;
+    interface_type: string;
+    interface_speed: string;
+    serial_number: string;
+    firmware: string;
+    rotational_speed: number;
+    capacity: number;
+    form_factor: string;
+    smart_support: boolean;
+    device_protocol: string;
+    device_type: string;
+
+    label: string;
+    host_id: string;
+
+    device_status: number;
+}

webapp/frontend/src/app/core/models/device-summary-model.ts

@@ -0,0 +1,16 @@
+import {DeviceModel} from 'app/core/models/device-model';
+import {SmartTemperatureModel} from 'app/core/models/measurements/smart-temperature-model';
+
+// maps to webapp/backend/pkg/models/device_summary.go
+export interface DeviceSummaryModel {
+    device: DeviceModel;
+    smart?: SmartSummary;
+    temp_history?: SmartTemperatureModel[];
+}
+
+export interface SmartSummary {
+    collector_date?: string,
+    temp?: number
+    power_on_hours?: number
+}
+

webapp/frontend/src/app/core/models/device-summary-response-wrapper.ts

@@ -0,0 +1,10 @@
+import {DeviceSummaryModel} from 'app/core/models/device-summary-model';
+
+// maps to webapp/backend/pkg/models/device_summary.go
+export interface DeviceSummaryResponseWrapper {
+    success: boolean;
+    errors: any[];
+    data: {
+        summary: { [key: string]: DeviceSummaryModel }
+    }
+}

webapp/frontend/src/app/core/models/device-summary-temp-response-wrapper.ts

@@ -0,0 +1,9 @@
+import {SmartTemperatureModel} from './measurements/smart-temperature-model';
+
+export interface DeviceSummaryTempResponseWrapper {
+    success: boolean;
+    errors: any[];
+    data: {
+        temp_history: { [key: string]: SmartTemperatureModel[]; }
+    }
+}

webapp/frontend/src/app/core/models/measurements/smart-attribute-model.ts

@@ -0,0 +1,19 @@
+// maps to webapp/backend/pkg/models/measurements/smart_ata_attribute.go
+// maps to webapp/backend/pkg/models/measurements/smart_nvme_attribute.go
+// maps to webapp/backend/pkg/models/measurements/smart_scsi_attribute.go
+export interface SmartAttributeModel {
+    attribute_id: number | string
+    value: number
+    thresh: number
+    worst?: number
+    raw_value?: number
+    raw_string?: string
+    when_failed?: string
+
+    transformed_value: number
+    status: number
+    status_reason?: string
+    failure_rate?: number
+
+    chartData?: any[]
+}

webapp/frontend/src/app/core/models/measurements/smart-model.ts

@@ -0,0 +1,13 @@
+// maps to webapp/backend/pkg/models/measurements/smart.go
+import {SmartAttributeModel} from './smart-attribute-model';
+
+export interface SmartModel {
+    date: string;
+    device_wwn: string;
+    device_protocol: string;
+
+    temp: number;
+    power_on_hours: number;
+    power_cycle_count: number
+    attrs: { [key: string]: SmartAttributeModel }
+}

webapp/frontend/src/app/core/models/measurements/smart-temperature-model.ts

@@ -0,0 +1,6 @@
+// maps to webapp/backend/pkg/models/measurements/smart_temperature.go
+export interface SmartTemperatureModel {
+    date: string;
+    temp: number;
+}
+

webapp/frontend/src/app/core/models/thresholds/attribute-metadata-model.ts

@@ -0,0 +1,13 @@
+// map to webapp/backend/pkg/thresholds/ata_attribute_metadata.go
+// map to webapp/backend/pkg/thresholds/nvme_attribute_metadata.go
+// map to webapp/backend/pkg/thresholds/scsi_attribute_metadata.go
+export interface AttributeMetadataModel {
+    display_name: string
+    ideal: string
+    critical: boolean
+    description: string
+
+    transform_value_unit?: string
+    observed_thresholds?: any[]
+    display_type: string
+}

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component.spec.ts

@@ -1,25 +1,64 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+import {async, ComponentFixture, TestBed} from '@angular/core/testing';
+
+import {DashboardDeviceDeleteDialogComponent} from './dashboard-device-delete-dialog.component';
+import {HttpClientModule} from '@angular/common/http';
+import {MAT_DIALOG_DATA, MatDialogModule, MatDialogRef} from '@angular/material/dialog';
+import {MatButtonModule} from '@angular/material/button';
+import {MatIconModule} from '@angular/material/icon';
+import {SharedModule} from '../../../shared/shared.module';
+import {DashboardDeviceDeleteDialogService} from './dashboard-device-delete-dialog.service';
+import {of} from 'rxjs';
 
-import { DashboardDeviceDeleteDialogComponent } from './dashboard-device-delete-dialog.component';
 
 describe('DashboardDeviceDeleteDialogComponent', () => {
-  let component: DashboardDeviceDeleteDialogComponent;
-  let fixture: ComponentFixture<DashboardDeviceDeleteDialogComponent>;
+    let component: DashboardDeviceDeleteDialogComponent;
+    let fixture: ComponentFixture<DashboardDeviceDeleteDialogComponent>;
 
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ DashboardDeviceDeleteDialogComponent ]
-    })
-    .compileComponents();
-  }));
+    const matDialogRefSpy = jasmine.createSpyObj('MatDialogRef', ['closeDialog', 'close']);
+    const dashboardDeviceDeleteDialogServiceSpy = jasmine.createSpyObj('DashboardDeviceDeleteDialogService', ['deleteDevice']);
 
-  beforeEach(() => {
-    fixture = TestBed.createComponent(DashboardDeviceDeleteDialogComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
+    beforeEach(async(() => {
+        TestBed.configureTestingModule({
+            imports: [
+                HttpClientModule,
+                MatDialogModule,
+                MatButtonModule,
+                MatIconModule,
+                SharedModule,
+            ],
+            providers: [
+                {provide: MatDialogRef, useValue: matDialogRefSpy},
+                {provide: MAT_DIALOG_DATA, useValue: {wwn: 'test-wwn', title: 'my-test-device-title'}},
+                {provide: DashboardDeviceDeleteDialogService, useValue: dashboardDeviceDeleteDialogServiceSpy}
+            ],
+            declarations: [DashboardDeviceDeleteDialogComponent]
+        })
+            .compileComponents()
+    }));
 
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
+    beforeEach(() => {
+        fixture = TestBed.createComponent(DashboardDeviceDeleteDialogComponent);
+        component = fixture.componentInstance;
+        fixture.detectChanges();
+    });
+
+    it('should create', () => {
+        expect(component).toBeTruthy();
+    });
+
+    it('should close the component if cancel is clicked', () => {
+        matDialogRefSpy.closeDialog.calls.reset();
+        matDialogRefSpy.closeDialog()
+        expect(matDialogRefSpy.closeDialog).toHaveBeenCalled();
+    });
+
+    it('should attempt to delete device if delete is clicked', () => {
+        dashboardDeviceDeleteDialogServiceSpy.deleteDevice.and.returnValue(of({'success': true}));
+
+        component.onDeleteClick()
+        expect(dashboardDeviceDeleteDialogServiceSpy.deleteDevice).toHaveBeenCalledWith('test-wwn');
+        expect(dashboardDeviceDeleteDialogServiceSpy.deleteDevice.calls.count())
+            .withContext('one call')
+            .toBe(1);
+    });
 });

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component.ts

@@ -1,7 +1,6 @@
-import { Component, OnInit, Inject } from '@angular/core';
+import {Component, Inject, OnInit} from '@angular/core';
 import {MAT_DIALOG_DATA, MatDialogRef} from '@angular/material/dialog';
 import {DashboardDeviceDeleteDialogService} from 'app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.service';
-import {Subject} from 'rxjs';
 
 @Component({
   selector: 'app-dashboard-device-delete-dialog',

webapp/frontend/src/app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.module.ts

@@ -1,44 +1,21 @@
-import { NgModule } from '@angular/core';
-import { RouterModule } from '@angular/router';
-import { Overlay } from '@angular/cdk/overlay';
-import { MAT_AUTOCOMPLETE_SCROLL_STRATEGY, MatAutocompleteModule } from '@angular/material/autocomplete';
-import { MatButtonModule } from '@angular/material/button';
-import { MatSelectModule } from '@angular/material/select';
-import { MatFormFieldModule } from '@angular/material/form-field';
-import { MatIconModule } from '@angular/material/icon';
-import { MatInputModule } from '@angular/material/input';
-import { SharedModule } from 'app/shared/shared.module';
+import {NgModule} from '@angular/core';
+import {RouterModule} from '@angular/router';
+import {MatButtonModule} from '@angular/material/button';
+import {MatIconModule} from '@angular/material/icon';
+import {SharedModule} from 'app/shared/shared.module';
 import {DashboardDeviceDeleteDialogComponent} from 'app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component'
-import { MatButtonToggleModule} from '@angular/material/button-toggle';
-import {MatTabsModule} from '@angular/material/tabs';
-import {MatSliderModule} from '@angular/material/slider';
-import {MatSlideToggleModule} from '@angular/material/slide-toggle';
-import {MatTooltipModule} from '@angular/material/tooltip';
 import {dashboardRoutes} from 'app/modules/dashboard/dashboard.routing';
-import {MatDividerModule} from '@angular/material/divider';
-import {MatMenuModule} from '@angular/material/menu';
-import {MatProgressBarModule} from '@angular/material/progress-bar';
-import {MatSortModule} from '@angular/material/sort';
-import {MatTableModule} from '@angular/material/table';
-import {NgApexchartsModule} from 'ng-apexcharts';
-import { MatDialogModule } from '@angular/material/dialog';
+import {MatDialogModule} from '@angular/material/dialog';
 
 @NgModule({
     declarations: [
         DashboardDeviceDeleteDialogComponent
     ],
-    imports     : [
+    imports: [
         RouterModule.forChild([]),
         RouterModule.forChild(dashboardRoutes),
         MatButtonModule,
-        MatDividerModule,
-        MatTooltipModule,
         MatIconModule,
-        MatMenuModule,
-        MatProgressBarModule,
-        MatSortModule,
-        MatTableModule,
-        NgApexchartsModule,
         SharedModule,
         MatDialogModule
     ],

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html

@@ -1,21 +1,21 @@
-<div [ngClass]="{ 'border-green': deviceSummary.device.device_status == 0 && deviceSummary.smart,
-                                'border-red': deviceSummary.device.device_status != 0 }"
+<div [ngClass]="{ 'border-green': deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) == 'passed',
+                                'border-red': deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) == 'failed' }"
      class="relative flex flex-col flex-auto p-6 pr-3 pb-3 bg-card rounded border-l-4 shadow-md overflow-hidden">
     <div class="absolute bottom-0 right-0 w-24 h-24 -m-6">
         <mat-icon class="icon-size-96 opacity-12 text-green"
-                  *ngIf="deviceSummary.device.device_status == 0 && deviceSummary.smart"
+                  *ngIf="deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) == 'passed'"
                   [svgIcon]="'heroicons_outline:check-circle'"></mat-icon>
         <mat-icon class="icon-size-96 opacity-12 text-red"
-                  *ngIf="deviceSummary.device.device_status != 0"
+                  *ngIf="deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) == 'failed'"
                   [svgIcon]="'heroicons_outline:exclamation-circle'"></mat-icon>
         <mat-icon class="icon-size-96 opacity-12 text-yellow"
-                  *ngIf="!deviceSummary.smart"
+                  *ngIf="deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) == 'unknown'"
                   [svgIcon]="'heroicons_outline:question-mark-circle'"></mat-icon>
     </div>
     <div class="flex items-center">
         <div class="flex flex-col">
             <a [routerLink]="'/device/'+ deviceSummary.device.wwn"
-               class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceSummary.device | deviceTitle:config.dashboardDisplay}}</a>
+               class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceSummary.device | deviceTitle:config.dashboard_display}}</a>
             <div [ngClass]="classDeviceLastUpdatedOn(deviceSummary)" class="font-medium text-sm" *ngIf="deviceSummary.smart">
                 Last Updated on {{deviceSummary.smart.collector_date | date:'MMMM dd, yyyy - HH:mm' }}
             </div>
@@ -46,17 +46,20 @@
     <div class="flex flex-row flex-wrap mt-4 -mx-6">
         <div class="flex flex-col mx-6 my-3 xs:w-full">
             <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Status</div>
-            <div class="mt-2 font-medium text-3xl leading-none" *ngIf="deviceSummary.smart?.collector_date; else unknownStatus">{{ deviceStatusString(deviceSummary.device.device_status) | titlecase}}</div>
+            <div class="mt-2 font-medium text-3xl leading-none"
+                 *ngIf="deviceSummary.smart?.collector_date; else unknownStatus">{{ deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) | titlecase}}</div>
             <ng-template #unknownStatus><div class="mt-2 font-medium text-3xl leading-none">No Data</div></ng-template>
         </div>
         <div class="flex flex-col mx-6 my-3 xs:w-full">
             <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Temperature</div>
-            <div class="mt-2 font-medium text-3xl leading-none" *ngIf="deviceSummary.smart?.collector_date; else unknownTemp">{{ deviceSummary.smart?.temp | temperature:config.temperatureUnit:true }}</div>
+            <div class="mt-2 font-medium text-3xl leading-none"
+                 *ngIf="deviceSummary.smart?.collector_date; else unknownTemp">{{ deviceSummary.smart?.temp | temperature:config.temperature_unit:true }}</div>
             <ng-template #unknownTemp><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
         </div>
         <div class="flex flex-col mx-6 my-3 xs:w-full">
             <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Capacity</div>
-            <div class="mt-2 font-medium text-3xl leading-none">{{ deviceSummary.device.capacity | fileSize}}</div>
+            <div
+                class="mt-2 font-medium text-3xl leading-none">{{ deviceSummary.device.capacity | fileSize:config.file_size_si_units}}</div>
         </div>
         <div class="flex flex-col mx-6 my-3 xs:w-full">
             <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Powered On</div>

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts

@@ -1,25 +1,158 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+import {async, ComponentFixture, TestBed} from '@angular/core/testing';
 
-import { DashboardDeviceComponent } from './dashboard-device.component';
+import {DashboardDeviceComponent} from './dashboard-device.component';
+import {MatDialog} from '@angular/material/dialog';
+import {MatButtonModule} from '@angular/material/button';
+import {MatIconModule} from '@angular/material/icon';
+import {SharedModule} from 'app/shared/shared.module';
+import {MatMenuModule} from '@angular/material/menu';
+import {TREO_APP_CONFIG} from '@treo/services/config/config.constants';
+import {DeviceSummaryModel} from 'app/core/models/device-summary-model';
+import * as moment from 'moment';
+import {HttpClientTestingModule} from '@angular/common/http/testing';
+import {HttpClient} from '@angular/common/http';
+import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service';
+import {of} from 'rxjs';
+import {MetricsStatusThreshold} from 'app/core/config/app.config';
 
 describe('DashboardDeviceComponent', () => {
-  let component: DashboardDeviceComponent;
-  let fixture: ComponentFixture<DashboardDeviceComponent>;
+    let component: DashboardDeviceComponent;
+    let fixture: ComponentFixture<DashboardDeviceComponent>;
 
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ DashboardDeviceComponent ]
+    const matDialogSpy = jasmine.createSpyObj('MatDialog', ['open']);
+    // const configServiceSpy = jasmine.createSpyObj('ScrutinyConfigService', ['config$']);
+    let configService: ScrutinyConfigService;
+    let httpClientSpy: jasmine.SpyObj<HttpClient>;
+
+    beforeEach(async(() => {
+
+        httpClientSpy = jasmine.createSpyObj('HttpClient', ['get']);
+        configService = new ScrutinyConfigService(httpClientSpy, {});
+
+        TestBed.configureTestingModule({
+            imports: [
+                MatButtonModule,
+                MatIconModule,
+                MatMenuModule,
+                SharedModule,
+                HttpClientTestingModule,
+            ],
+            providers: [
+                {provide: MatDialog, useValue: matDialogSpy},
+                {provide: TREO_APP_CONFIG, useValue: {dashboard_display: 'name', metrics: {status_threshold: 3}}},
+                {provide: ScrutinyConfigService, useValue: configService}
+            ],
+            declarations: [DashboardDeviceComponent]
+        })
+            .compileComponents();
+    }));
+
+    beforeEach(() => {
+        // configServiceSpy.config$.and.returnValue(of({'success': true}));
+        fixture = TestBed.createComponent(DashboardDeviceComponent);
+        component = fixture.componentInstance;
+    });
+
+    it('should create', () => {
+        expect(component).toBeTruthy();
+    });
+
+    describe('#classDeviceLastUpdatedOn()', () => {
+
+        it('if non-zero device status, should be red', () => {
+            httpClientSpy.get.and.returnValue(of({
+                settings: {
+                    metrics: {
+                        status_threshold: MetricsStatusThreshold.Both,
+                    }
+                }
+            }));
+            component.ngOnInit()
+            // component.deviceSummary = summary.data.summary['0x5000c500673e6b5f'] as DeviceSummaryModel
+            expect(component.classDeviceLastUpdatedOn({
+                device: {
+                    device_status: 2,
+                },
+                smart: {
+                    collector_date: moment().subtract(13, 'days').toISOString()
+                },
+            } as DeviceSummaryModel)).toBe('text-red')
+        });
+
+        it('if non-zero device status, should be red', () => {
+            httpClientSpy.get.and.returnValue(of({
+                settings: {
+                    metrics: {
+                        status_threshold: MetricsStatusThreshold.Both,
+                    }
+                }
+            }));
+            component.ngOnInit()
+            expect(component.classDeviceLastUpdatedOn({
+                device: {
+                    device_status: 2
+                },
+                smart: {
+                    collector_date: moment().subtract(13, 'days').toISOString()
+                },
+            } as DeviceSummaryModel)).toBe('text-red')
+        });
+
+        it('if healthy device status and updated in the last two weeks, should be green', () => {
+            httpClientSpy.get.and.returnValue(of({
+                settings: {
+                    metrics: {
+                        status_threshold: MetricsStatusThreshold.Both,
+                    }
+                }
+            }));
+            component.ngOnInit()
+            expect(component.classDeviceLastUpdatedOn({
+                device: {
+                    device_status: 0
+                },
+                smart: {
+                    collector_date: moment().subtract(13, 'days').toISOString()
+                }
+            } as DeviceSummaryModel)).toBe('text-green')
+        });
+
+        it('if healthy device status and updated more than two weeks ago, but less than 1 month, should be yellow', () => {
+            httpClientSpy.get.and.returnValue(of({
+                settings: {
+                    metrics: {
+                        status_threshold: MetricsStatusThreshold.Both,
+                    }
+                }
+            }));
+            component.ngOnInit()
+            expect(component.classDeviceLastUpdatedOn({
+                device: {
+                    device_status: 0
+                },
+                smart: {
+                    collector_date: moment().subtract(3, 'weeks').toISOString()
+                }
+            } as DeviceSummaryModel)).toBe('text-yellow')
+        });
+
+        it('if healthy device status and updated more 1 month ago, should be red', () => {
+            httpClientSpy.get.and.returnValue(of({
+                settings: {
+                    metrics: {
+                        status_threshold: MetricsStatusThreshold.Both,
+                    }
+                }
+            }));
+            component.ngOnInit()
+            expect(component.classDeviceLastUpdatedOn({
+                device: {
+                    device_status: 0
+                },
+                smart: {
+                    collector_date: moment().subtract(5, 'weeks').toISOString()
+                }
+            } as DeviceSummaryModel)).toBe('text-red')
+        });
     })
-    .compileComponents();
-  }));
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(DashboardDeviceComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
 });

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts

@@ -1,29 +1,32 @@
-import { Component, Input, Output, OnInit, EventEmitter} from '@angular/core';
+import {Component, EventEmitter, Input, OnInit, Output} from '@angular/core';
 import * as moment from 'moment';
 import {takeUntil} from 'rxjs/operators';
 import {AppConfig} from 'app/core/config/app.config';
-import {TreoConfigService} from '@treo/services/config';
+import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service';
 import {Subject} from 'rxjs';
-import  humanizeDuration from 'humanize-duration'
+import humanizeDuration from 'humanize-duration'
 import {MatDialog} from '@angular/material/dialog';
 import {DashboardDeviceDeleteDialogComponent} from 'app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component';
 import {DeviceTitlePipe} from 'app/shared/device-title.pipe';
+import {DeviceSummaryModel} from 'app/core/models/device-summary-model';
+import {DeviceStatusPipe} from 'app/shared/device-status.pipe';
 
 @Component({
-  selector: 'app-dashboard-device',
-  templateUrl: './dashboard-device.component.html',
-  styleUrls: ['./dashboard-device.component.scss']
+    selector: 'app-dashboard-device',
+    templateUrl: './dashboard-device.component.html',
+    styleUrls: ['./dashboard-device.component.scss']
 })
 export class DashboardDeviceComponent implements OnInit {
 
     constructor(
-        private _configService: TreoConfigService,
+        private _configService: ScrutinyConfigService,
         public dialog: MatDialog,
     ) {
         // Set the private defaults
         this._unsubscribeAll = new Subject();
     }
-    @Input() deviceSummary: any;
+
+    @Input() deviceSummary: DeviceSummaryModel;
     @Input() deviceWWN: string;
     @Output() deviceDeleted = new EventEmitter<string>();
 
@@ -33,6 +36,8 @@ export class DashboardDeviceComponent implements OnInit {
 
     readonly humanizeDuration = humanizeDuration;
 
+    deviceStatusForModelWithThreshold = DeviceStatusPipe.deviceStatusForModelWithThreshold
+
     ngOnInit(): void {
         // Subscribe to config changes
         this._configService.config$
@@ -47,45 +52,38 @@ export class DashboardDeviceComponent implements OnInit {
     // @ Public methods
     // -----------------------------------------------------------------------------------------------------
 
-    classDeviceLastUpdatedOn(deviceSummary): string {
-        if (deviceSummary.device.device_status !== 0) {
+    classDeviceLastUpdatedOn(deviceSummary: DeviceSummaryModel): string {
+        const deviceStatus = DeviceStatusPipe.deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, this.config.metrics.status_threshold)
+        if (deviceStatus === 'failed') {
             return 'text-red' // if the device has failed, always highlight in red
-        } else if(deviceSummary.device.device_status === 0 && deviceSummary.smart){
-            if(moment().subtract(14, 'd').isBefore(deviceSummary.smart.collector_date)){
+        } else if (deviceStatus === 'passed') {
+            if (moment().subtract(14, 'days').isBefore(deviceSummary.smart.collector_date)) {
                 // this device was updated in the last 2 weeks.
                 return 'text-green'
-            } else if(moment().subtract(1, 'm').isBefore(deviceSummary.smart.collector_date)){
+            } else if (moment().subtract(1, 'months').isBefore(deviceSummary.smart.collector_date)) {
                 // this device was updated in the last month
                 return 'text-yellow'
-            } else{
+            } else {
                 // last updated more than a month ago.
                 return 'text-red'
             }
-
         } else {
             return ''
         }
     }
 
-    deviceStatusString(deviceStatus): string {
-        if(deviceStatus === 0){
-            return 'passed'
-        } else {
-            return 'failed'
-        }
-    }
-
-
-
     openDeleteDialog(): void {
         const dialogRef = this.dialog.open(DashboardDeviceDeleteDialogComponent, {
             // width: '250px',
-            data: {wwn: this.deviceWWN, title: DeviceTitlePipe.deviceTitleWithFallback(this.deviceSummary.device, this.config.dashboardDisplay)}
+            data: {
+                wwn: this.deviceWWN,
+                title: DeviceTitlePipe.deviceTitleWithFallback(this.deviceSummary.device, this.config.dashboard_display)
+            }
         });
 
         dialogRef.afterClosed().subscribe(result => {
             console.log('The dialog was closed', result);
-            if(result.success){
+            if (result.success) {
                 this.deviceDeleted.emit(this.deviceWWN)
             }
         });

webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.module.ts

@@ -1,53 +1,30 @@
-import { NgModule } from '@angular/core';
-import { RouterModule } from '@angular/router';
-import { Overlay } from '@angular/cdk/overlay';
-import { MAT_AUTOCOMPLETE_SCROLL_STRATEGY, MatAutocompleteModule } from '@angular/material/autocomplete';
-import { MatButtonModule } from '@angular/material/button';
-import { MatSelectModule } from '@angular/material/select';
-import { MatFormFieldModule } from '@angular/material/form-field';
-import { MatIconModule } from '@angular/material/icon';
-import { MatInputModule } from '@angular/material/input';
-import { SharedModule } from 'app/shared/shared.module';
+import {NgModule} from '@angular/core';
+import {RouterModule} from '@angular/router';
+import {MatButtonModule} from '@angular/material/button';
+import {MatIconModule} from '@angular/material/icon';
+import {SharedModule} from 'app/shared/shared.module';
 import {DashboardDeviceComponent} from 'app/layout/common/dashboard-device/dashboard-device.component'
-import { MatDialogModule } from '@angular/material/dialog';
-import { MatButtonToggleModule} from '@angular/material/button-toggle';
-import {MatTabsModule} from '@angular/material/tabs';
-import {MatSliderModule} from '@angular/material/slider';
-import {MatSlideToggleModule} from '@angular/material/slide-toggle';
-import {MatTooltipModule} from '@angular/material/tooltip';
 import {dashboardRoutes} from '../../../modules/dashboard/dashboard.routing';
-import {MatDividerModule} from '@angular/material/divider';
 import {MatMenuModule} from '@angular/material/menu';
-import {MatProgressBarModule} from '@angular/material/progress-bar';
-import {MatSortModule} from '@angular/material/sort';
-import {MatTableModule} from '@angular/material/table';
-import {NgApexchartsModule} from 'ng-apexcharts';
 import {DashboardDeviceDeleteDialogModule} from 'app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.module';
 
 @NgModule({
     declarations: [
         DashboardDeviceComponent
     ],
-    imports     : [
+    imports: [
         RouterModule.forChild([]),
         RouterModule.forChild(dashboardRoutes),
         MatButtonModule,
-        MatDividerModule,
-        MatTooltipModule,
         MatIconModule,
         MatMenuModule,
-        MatProgressBarModule,
-        MatSortModule,
-        MatTableModule,
-        NgApexchartsModule,
         SharedModule,
         DashboardDeviceDeleteDialogModule
     ],
-    exports     : [
+    exports: [
         DashboardDeviceComponent,
     ],
-    providers   : []
+    providers: []
 })
-export class DashboardDeviceModule
-{
+export class DashboardDeviceModule {
 }

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html

@@ -37,69 +37,41 @@
 
         <div class="flex flex-col mt-5 gt-md:flex-row">
             <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
-                <mat-label>Temperature Display Unit</mat-label>
+                <mat-label>Temperature</mat-label>
                 <mat-select [(ngModel)]="temperatureUnit">
                     <mat-option value="celsius">Celsius</mat-option>
                     <mat-option value="fahrenheit">Fahrenheit</mat-option>
                 </mat-select>
             </mat-form-field>
 
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>File Size</mat-label>
+                <mat-select [(ngModel)]="fileSizeSIUnits">
+                    <mat-option [value]=true>SI Units (GB)</mat-option>
+                    <mat-option [value]=false>Binary Units (GiB)</mat-option>
+                </mat-select>
+            </mat-form-field>
         </div>
 
-        <div class="flex">
-            <mat-tab-group mat-align-tabs="start">
-                <mat-tab label="Ata">
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Device Status - Thresholds</mat-label>
+                <mat-select [(ngModel)]=statusThreshold>
+                    <mat-option [value]=1>Smart</mat-option>
+                    <mat-option [value]=2>Scrutiny</mat-option>
+                    <mat-option [value]=3>Both</mat-option>
+                </mat-select>
+            </mat-form-field>
+        </div>
 
-                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
-                        <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
-                            <input disabled matInput [value]="'10%'">
-                        </mat-form-field>
-                        <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
-                            <input disabled matInput>
-                        </mat-form-field>
-                    </div>
-
-                    <div matTooltip="not yet implemented" class="flex flex-col gt-md:flex-row">
-                        <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label class="text-hint">Error Threshold</mat-label>
-                            <input disabled matInput [value]="'20%'">
-                        </mat-form-field>
-                        <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label class="text-hint">Warning Threshold</mat-label>
-                            <input disabled matInput [value]="'10%'">
-                        </mat-form-field>
-                    </div>
-
-                </mat-tab>
-                <mat-tab label="NVMe">
-
-                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
-                        <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
-                            <input disabled matInput [value]="'enabled'">
-                        </mat-form-field>
-                        <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
-                            <input disabled matInput>
-                        </mat-form-field>
-                    </div>
-
-                </mat-tab>
-                <mat-tab label="SCSI">
-                    <div matTooltip="not yet implemented" class="flex flex-col mt-5 gt-md:flex-row">
-                        <mat-form-field class="flex-auto gt-md:pr-3">
-                            <mat-label class="text-hint">Critical Error Threshold</mat-label>
-                            <input disabled matInput [value]="'enabled'">
-                        </mat-form-field>
-                        <mat-form-field class="flex-auto gt-md:pl-3">
-                            <mat-label class="text-hint">Critical Warning Threshold</mat-label>
-                            <input disabled matInput>
-                        </mat-form-field>
-                    </div>
-                </mat-tab>
-            </mat-tab-group>
+        <div class="flex flex-col mt-5 gt-md:flex-row">
+            <mat-form-field class="flex-auto gt-xs:pr-3 gt-md:pr-3">
+                <mat-label>Notify - Filter Attributes</mat-label>
+                <mat-select [(ngModel)]=statusFilterAttributes>
+                    <mat-option [value]=0>All</mat-option>
+                    <mat-option [value]=1>Critical</mat-option>
+                </mat-select>
+            </mat-form-field>
         </div>
     </div>
 

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.spec.ts

@@ -1,25 +0,0 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { DashboardSettingsComponent } from './dashboard-settings.component';
-
-describe('DashboardSettingsComponent', () => {
-  let component: DashboardSettingsComponent;
-  let fixture: ComponentFixture<DashboardSettingsComponent>;
-
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ DashboardSettingsComponent ]
-    })
-    .compileComponents();
-  }));
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(DashboardSettingsComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});

webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts

@@ -1,59 +1,77 @@
-import { Component, OnInit } from '@angular/core';
-import {AppConfig} from 'app/core/config/app.config';
-import { TreoConfigService } from '@treo/services/config';
+import {Component, OnInit} from '@angular/core';
+import {
+    AppConfig,
+    DashboardDisplay,
+    DashboardSort,
+    MetricsStatusFilterAttributes,
+    MetricsStatusThreshold,
+    TemperatureUnit,
+    Theme
+} from 'app/core/config/app.config';
+import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service';
 import {Subject} from 'rxjs';
 import {takeUntil} from 'rxjs/operators';
 
 @Component({
-  selector: 'app-dashboard-settings',
-  templateUrl: './dashboard-settings.component.html',
-  styleUrls: ['./dashboard-settings.component.scss']
+    selector: 'app-dashboard-settings',
+    templateUrl: './dashboard-settings.component.html',
+    styleUrls: ['./dashboard-settings.component.scss']
 })
 export class DashboardSettingsComponent implements OnInit {
 
     dashboardDisplay: string;
     dashboardSort: string;
     temperatureUnit: string;
+    fileSizeSIUnits: boolean;
     theme: string;
+    statusThreshold: number;
+    statusFilterAttributes: number;
 
     // Private
     private _unsubscribeAll: Subject<any>;
 
     constructor(
-        private _configService: TreoConfigService,
+        private _configService: ScrutinyConfigService,
     ) {
         // Set the private defaults
         this._unsubscribeAll = new Subject();
     }
 
-  ngOnInit(): void {
-      // Subscribe to config changes
-      this._configService.config$
-          .pipe(takeUntil(this._unsubscribeAll))
-          .subscribe((config: AppConfig) => {
+    ngOnInit(): void {
+        // Subscribe to config changes
+        this._configService.config$
+            .pipe(takeUntil(this._unsubscribeAll))
+            .subscribe((config: AppConfig) => {
 
-              // Store the config
-              this.dashboardDisplay = config.dashboardDisplay;
-              this.dashboardSort = config.dashboardSort;
-              this.temperatureUnit = config.temperatureUnit;
-              this.theme = config.theme;
+                // Store the config
+                this.dashboardDisplay = config.dashboard_display;
+                this.dashboardSort = config.dashboard_sort;
+                this.temperatureUnit = config.temperature_unit;
+                this.fileSizeSIUnits = config.file_size_si_units;
+                this.theme = config.theme;
 
-          });
+                this.statusFilterAttributes = config.metrics.status_filter_attributes;
+                this.statusThreshold = config.metrics.status_threshold;
 
-  }
+            });
 
-  saveSettings(): void {
+    }
 
-
-        const newSettings = {
-            dashboardDisplay: this.dashboardDisplay,
-            dashboardSort: this.dashboardSort,
-            temperatureUnit: this.temperatureUnit,
-            theme: this.theme
+    saveSettings(): void {
+        const newSettings: AppConfig = {
+            dashboard_display: this.dashboardDisplay as DashboardDisplay,
+            dashboard_sort: this.dashboardSort as DashboardSort,
+            temperature_unit: this.temperatureUnit as TemperatureUnit,
+            file_size_si_units: this.fileSizeSIUnits,
+            theme: this.theme as Theme,
+            metrics: {
+                status_filter_attributes: this.statusFilterAttributes as MetricsStatusFilterAttributes,
+                status_threshold: this.statusThreshold as MetricsStatusThreshold
+            }
         }
         this._configService.config = newSettings
         console.log(`Saved Settings: ${JSON.stringify(newSettings)}`)
-  }
+    }
 
     formatLabel(value: number): number {
         return value;

webapp/frontend/src/app/layout/common/detail-settings/detail-settings.module.ts

@@ -1,16 +1,15 @@
-import { NgModule } from '@angular/core';
-import { RouterModule } from '@angular/router';
-import { Overlay } from '@angular/cdk/overlay';
-import { MAT_AUTOCOMPLETE_SCROLL_STRATEGY, MatAutocompleteModule } from '@angular/material/autocomplete';
-import { MatButtonModule } from '@angular/material/button';
-import { MatSelectModule } from '@angular/material/select';
-import { MatFormFieldModule } from '@angular/material/form-field';
-import { MatIconModule } from '@angular/material/icon';
-import { MatInputModule } from '@angular/material/input';
-import { SharedModule } from 'app/shared/shared.module';
+import {NgModule} from '@angular/core';
+import {RouterModule} from '@angular/router';
+import {MatAutocompleteModule} from '@angular/material/autocomplete';
+import {MatButtonModule} from '@angular/material/button';
+import {MatSelectModule} from '@angular/material/select';
+import {MatFormFieldModule} from '@angular/material/form-field';
+import {MatIconModule} from '@angular/material/icon';
+import {MatInputModule} from '@angular/material/input';
+import {SharedModule} from 'app/shared/shared.module';
 import {DetailSettingsComponent} from 'app/layout/common/detail-settings/detail-settings.component'
-import { MatDialogModule } from '@angular/material/dialog';
-import { MatButtonToggleModule} from '@angular/material/button-toggle';
+import {MatDialogModule} from '@angular/material/dialog';
+import {MatButtonToggleModule} from '@angular/material/button-toggle';
 import {MatTabsModule} from '@angular/material/tabs';
 import {MatSliderModule} from '@angular/material/slider';
 import {MatSlideToggleModule} from '@angular/material/slide-toggle';
@@ -20,7 +19,7 @@ import {MatTooltipModule} from '@angular/material/tooltip';
     declarations: [
         DetailSettingsComponent
     ],
-    imports     : [
+    imports: [
         RouterModule.forChild([]),
         MatAutocompleteModule,
         MatDialogModule,
@@ -36,11 +35,10 @@ import {MatTooltipModule} from '@angular/material/tooltip';
         MatSlideToggleModule,
         SharedModule
     ],
-    exports     : [
+    exports: [
         DetailSettingsComponent
     ],
-    providers   : []
+    providers: []
 })
-export class DetailSettingsModule
-{
+export class DetailSettingsModule {
 }

webapp/frontend/src/app/layout/layout.component.ts

@@ -1,22 +1,21 @@
-import { Component, Inject, OnDestroy, OnInit, ViewEncapsulation } from '@angular/core';
-import { DOCUMENT } from '@angular/common';
-import { ActivatedRoute, NavigationEnd, Router } from '@angular/router';
-import { MatSlideToggleChange } from '@angular/material/slide-toggle';
-import { Subject } from 'rxjs';
-import { filter, takeUntil } from 'rxjs/operators';
-import { TreoConfigService } from '@treo/services/config';
-import { TreoDrawerService } from '@treo/components/drawer';
-import { Layout } from 'app/layout/layout.types';
-import { AppConfig, Theme } from 'app/core/config/app.config';
+import {Component, Inject, OnDestroy, OnInit, ViewEncapsulation} from '@angular/core';
+import {DOCUMENT} from '@angular/common';
+import {ActivatedRoute, NavigationEnd, Router} from '@angular/router';
+import {MatSlideToggleChange} from '@angular/material/slide-toggle';
+import {Subject} from 'rxjs';
+import {filter, takeUntil} from 'rxjs/operators';
+import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service';
+import {TreoDrawerService} from '@treo/components/drawer';
+import {Layout} from 'app/layout/layout.types';
+import {AppConfig, Theme} from 'app/core/config/app.config';
 
 @Component({
-    selector     : 'layout',
-    templateUrl  : './layout.component.html',
-    styleUrls    : ['./layout.component.scss'],
+    selector: 'layout',
+    templateUrl: './layout.component.html',
+    styleUrls: ['./layout.component.scss'],
     encapsulation: ViewEncapsulation.None
 })
-export class LayoutComponent implements OnInit, OnDestroy
-{
+export class LayoutComponent implements OnInit, OnDestroy {
     config: AppConfig;
     layout: Layout;
     theme: Theme;
@@ -29,14 +28,14 @@ export class LayoutComponent implements OnInit, OnDestroy
      * Constructor
      *
      * @param {ActivatedRoute} _activatedRoute
-     * @param {TreoConfigService} _treoConfigService
+     * @param {ScrutinyConfigService} _scrutinyConfigService
      * @param {TreoDrawerService} _treoDrawerService
      * @param {DOCUMENT} _document
      * @param {Router} _router
      */
     constructor(
         private _activatedRoute: ActivatedRoute,
-        private _treoConfigService: TreoConfigService,
+        private _scrutinyConfigService: ScrutinyConfigService,
         private _treoDrawerService: TreoDrawerService,
         @Inject(DOCUMENT) private _document: any,
         private _router: Router
@@ -59,7 +58,7 @@ export class LayoutComponent implements OnInit, OnDestroy
     ngOnInit(): void
     {
         // Subscribe to config changes
-        this._treoConfigService.config$
+        this._scrutinyConfigService.config$
             .pipe(takeUntil(this._unsubscribeAll))
             .subscribe((config: AppConfig) => {
 
@@ -180,18 +179,17 @@ export class LayoutComponent implements OnInit, OnDestroy
      *
      * @param layout
      */
-    setLayout(layout: string): void
-    {
+    setLayout(layout: Layout): void {
         // Clear the 'layout' query param to allow layout changes
         this._router.navigate([], {
-            queryParams        : {
+            queryParams: {
                 layout: null
             },
             queryParamsHandling: 'merge'
         }).then(() => {
 
             // Set the config
-            this._treoConfigService.config = {layout};
+            this._scrutinyConfigService.config = {layout};
         });
     }
 
@@ -202,6 +200,6 @@ export class LayoutComponent implements OnInit, OnDestroy
      */
     setTheme(change: MatSlideToggleChange): void
     {
-        this._treoConfigService.config = {theme: change.checked ? 'dark' : 'light'};
+        this._scrutinyConfigService.config = {theme: change.checked ? 'dark' : 'light'};
     }
 }

webapp/frontend/src/app/modules/dashboard/dashboard.component.html

@@ -1,5 +1,5 @@
 
-<div *ngIf="data && data.data && data.data.summary; else emptyDashboard">
+<div *ngIf="summaryData; else emptyDashboard">
     <div class="flex flex-col flex-auto w-full p-8 xs:p-2">
 
         <div class="flex flex-wrap w-full">
@@ -51,7 +51,11 @@
             <div class="flex flex-wrap w-full" *ngFor="let hostId of hostGroups | keyvalue">
                 <h3 class="ml-4" *ngIf="hostId.key">{{hostId.key}}</h3>
                 <div class="flex flex-wrap w-full">
-                    <app-dashboard-device (deviceDeleted)="onDeviceDeleted($event)" class="flex gt-sm:w-1/2 min-w-80 p-4" *ngFor="let deviceSummary of (deviceSummariesForHostGroup(hostId.value) | deviceSort:config.dashboardSort:config.dashboardDisplay )" [deviceWWN]="deviceSummary.device.wwn" [deviceSummary]="deviceSummary"></app-dashboard-device>
+                    <app-dashboard-device (deviceDeleted)="onDeviceDeleted($event)"
+                                          class="flex gt-sm:w-1/2 min-w-80 p-4"
+                                          *ngFor="let deviceSummary of (deviceSummariesForHostGroup(hostId.value) | deviceSort:config.dashboard_sort:config.dashboard_display )"
+                                          [deviceWWN]="deviceSummary.device.wwn"
+                                          [deviceSummary]="deviceSummary"></app-dashboard-device>
                 </div>
             </div>
 

webapp/frontend/src/app/modules/dashboard/dashboard.component.ts

@@ -1,17 +1,24 @@
-import { AfterViewInit, ChangeDetectionStrategy, Component, OnDestroy, OnInit, ViewChild, ViewEncapsulation } from '@angular/core';
-import { MatSort } from '@angular/material/sort';
-import { MatTableDataSource } from '@angular/material/table';
-import { Subject } from 'rxjs';
-import { takeUntil } from 'rxjs/operators';
+import {
+    AfterViewInit,
+    ChangeDetectionStrategy,
+    Component,
+    OnDestroy,
+    OnInit,
+    ViewChild,
+    ViewEncapsulation
+} from '@angular/core';
+import {Subject} from 'rxjs';
+import {takeUntil} from 'rxjs/operators';
 import {ApexOptions, ChartComponent} from 'ng-apexcharts';
-import { DashboardService } from 'app/modules/dashboard/dashboard.service';
+import {DashboardService} from 'app/modules/dashboard/dashboard.service';
 import {MatDialog} from '@angular/material/dialog';
-import { DashboardSettingsComponent } from 'app/layout/common/dashboard-settings/dashboard-settings.component';
+import {DashboardSettingsComponent} from 'app/layout/common/dashboard-settings/dashboard-settings.component';
 import {AppConfig} from 'app/core/config/app.config';
-import {TreoConfigService} from '@treo/services/config';
+import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service';
 import {Router} from '@angular/router';
 import {TemperaturePipe} from 'app/shared/temperature.pipe';
 import {DeviceTitlePipe} from 'app/shared/device-title.pipe';
+import {DeviceSummaryModel} from 'app/core/models/device-summary-model';
 
 @Component({
     selector       : 'example',
@@ -22,7 +29,7 @@ import {DeviceTitlePipe} from 'app/shared/device-title.pipe';
 })
 export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
 {
-    data: any;
+    summaryData: { [key: string]: DeviceSummaryModel };
     hostGroups: { [hostId: string]: string[] } = {}
     temperatureOptions: ApexOptions;
     tempDurationKey = 'forever'
@@ -35,11 +42,14 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
     /**
      * Constructor
      *
-     * @param {SmartService} _smartService
+     * @param {DashboardService} _dashboardService
+     * @param {ScrutinyConfigService} _configService
+     * @param {MatDialog} dialog
+     * @param {Router} router
      */
     constructor(
-        private _smartService: DashboardService,
-        private _configService: TreoConfigService,
+        private _dashboardService: DashboardService,
+        private _configService: ScrutinyConfigService,
         public dialog: MatDialog,
         private router: Router,
     )
@@ -81,16 +91,16 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
             });
 
         // Get the data
-        this._smartService.data$
+        this._dashboardService.data$
             .pipe(takeUntil(this._unsubscribeAll))
             .subscribe((data) => {
 
                 // Store the data
-                this.data = data;
+                this.summaryData = data;
 
                 // generate group data.
-                for(const wwn in this.data.data.summary){
-                    const hostid = this.data.data.summary[wwn].device.host_id
+                for (const wwn in this.summaryData) {
+                    const hostid = this.summaryData[wwn].device.host_id
                     const hostDeviceList = this.hostGroups[hostid] || []
                     hostDeviceList.push(wwn)
                     this.hostGroups[hostid] = hostDeviceList
@@ -132,15 +142,15 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
     private _deviceDataTemperatureSeries(): any[] {
         const deviceTemperatureSeries = []
 
-        console.log('DEVICE DATA SUMMARY', this.data)
+        console.log('DEVICE DATA SUMMARY', this.summaryData)
 
-        for(const wwn in this.data.data.summary){
-            const deviceSummary = this.data.data.summary[wwn]
-            if (!deviceSummary.temp_history){
+        for (const wwn in this.summaryData) {
+            const deviceSummary = this.summaryData[wwn]
+            if (!deviceSummary.temp_history) {
                 continue
             }
 
-            const deviceName = DeviceTitlePipe.deviceTitleWithFallback(deviceSummary.device, this.config.dashboardDisplay)
+            const deviceName = DeviceTitlePipe.deviceTitleWithFallback(deviceSummary.device, this.config.dashboard_display)
 
             const deviceSeriesMetadata = {
                 name: deviceName,
@@ -151,7 +161,7 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
                 const newDate = new Date(tempHistory.date);
                 deviceSeriesMetadata.data.push({
                     x: newDate,
-                    y: TemperaturePipe.formatTemperature(tempHistory.temp, this.config.temperatureUnit, false)
+                    y: TemperaturePipe.formatTemperature(tempHistory.temp, this.config.temperature_unit, false)
                 })
             }
             deviceTemperatureSeries.push(deviceSeriesMetadata)
@@ -202,11 +212,11 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
                 y    : {
 
                     formatter: (value) => {
-                        return TemperaturePipe.formatTemperature(value, this.config.temperatureUnit, true) as string;
+                        return TemperaturePipe.formatTemperature(value, this.config.temperature_unit, true) as string;
                     }
                 }
             },
-            xaxis  : {
+            xaxis: {
                 type: 'datetime'
             }
         };
@@ -216,18 +226,18 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
     // @ Public methods
     // -----------------------------------------------------------------------------------------------------
 
-    deviceSummariesForHostGroup(hostGroupWWNs: string[]): any[] {
-        const deviceSummaries = []
-        for(const wwn of hostGroupWWNs){
-            if(this.data.data.summary[wwn]){
-                deviceSummaries.push(this.data.data.summary[wwn])
+    deviceSummariesForHostGroup(hostGroupWWNs: string[]): DeviceSummaryModel[] {
+        const deviceSummaries: DeviceSummaryModel[] = []
+        for (const wwn of hostGroupWWNs) {
+            if (this.summaryData[wwn]) {
+                deviceSummaries.push(this.summaryData[wwn])
             }
         }
         return deviceSummaries
     }
 
     openDialog(): void {
-        const dialogRef = this.dialog.open(DashboardSettingsComponent);
+        const dialogRef = this.dialog.open(DashboardSettingsComponent, {width: '600px',});
 
         dialogRef.afterClosed().subscribe(result => {
             console.log(`Dialog result: ${result}`);
@@ -235,7 +245,7 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
     }
 
     onDeviceDeleted(wwn: string): void {
-        delete this.data.data.summary[wwn] // remove the device from the summary list.
+        delete this.summaryData[wwn] // remove the device from the summary list.
     }
 
     /*
@@ -246,16 +256,16 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
 	DURATION_KEY_FOREVER = "forever"
      */
 
-    changeSummaryTempDuration(durationKey: string){
+    changeSummaryTempDuration(durationKey: string): void {
         this.tempDurationKey = durationKey
 
-        this._smartService.getSummaryTempData(durationKey)
-            .subscribe((data) => {
+        this._dashboardService.getSummaryTempData(durationKey)
+            .subscribe((tempHistoryData) => {
 
                 // given a list of device temp history, override the data in the "summary" object.
-                for(const wwn in this.data.data.summary) {
+                for (const wwn in this.summaryData) {
                     // console.log(`Updating ${wwn}, length: ${this.data.data.summary[wwn].temp_history.length}`)
-                    this.data.data.summary[wwn].temp_history = data.data.temp_history[wwn] || []
+                    this.summaryData[wwn].temp_history = tempHistoryData[wwn] || []
                 }
 
                 // Prepare the chart series data

webapp/frontend/src/app/modules/dashboard/dashboard.resolvers.ts

@@ -1,13 +1,13 @@
-import { Injectable } from '@angular/core';
-import { ActivatedRouteSnapshot, Resolve, RouterStateSnapshot } from '@angular/router';
-import { Observable } from 'rxjs';
-import { DashboardService } from 'app/modules/dashboard/dashboard.service';
+import {Injectable} from '@angular/core';
+import {ActivatedRouteSnapshot, Resolve, RouterStateSnapshot} from '@angular/router';
+import {Observable} from 'rxjs';
+import {DashboardService} from 'app/modules/dashboard/dashboard.service';
+import {DeviceSummaryModel} from 'app/core/models/device-summary-model';
 
 @Injectable({
     providedIn: 'root'
 })
-export class DashboardResolver implements Resolve<any>
-{
+export class DashboardResolver implements Resolve<any> {
     /**
      * Constructor
      *
@@ -29,8 +29,7 @@ export class DashboardResolver implements Resolve<any>
      * @param route
      * @param state
      */
-    resolve(route: ActivatedRouteSnapshot, state: RouterStateSnapshot): Observable<any>
-    {
+    resolve(route: ActivatedRouteSnapshot, state: RouterStateSnapshot): Observable<{ [p: string]: DeviceSummaryModel }> {
         return this._dashboardService.getSummaryData();
     }
 }

webapp/frontend/src/app/modules/dashboard/dashboard.service.spec.ts

@@ -0,0 +1,44 @@
+import {HttpClient} from '@angular/common/http';
+import {DashboardService} from './dashboard.service';
+import {of} from 'rxjs';
+import {summary} from 'app/data/mock/summary/data'
+import {temp_history} from 'app/data/mock/summary/temp_history'
+import {DeviceSummaryModel} from 'app/core/models/device-summary-model';
+import {SmartTemperatureModel} from 'app/core/models/measurements/smart-temperature-model';
+
+describe('DashboardService', () => {
+    let service: DashboardService;
+    let httpClientSpy: jasmine.SpyObj<HttpClient>;
+
+    beforeEach(() => {
+        httpClientSpy = jasmine.createSpyObj('HttpClient', ['get']);
+        service = new DashboardService(httpClientSpy);
+    });
+
+    it('should unwrap and return getSummaryData() (HttpClient called once)', (done: DoneFn) => {
+        httpClientSpy.get.and.returnValue(of(summary));
+
+        service.getSummaryData().subscribe(value => {
+            expect(value).toBe(summary.data.summary as { [key: string]: DeviceSummaryModel });
+            done();
+        });
+        expect(httpClientSpy.get.calls.count())
+            .withContext('one call')
+            .toBe(1);
+    });
+
+    it('should unwrap and return getSummaryTempData() (HttpClient called once)', (done: DoneFn) => {
+        // const expectedHeroes: any[] =
+        //     [{ id: 1, name: 'A' }, { id: 2, name: 'B' }];
+
+        httpClientSpy.get.and.returnValue(of(temp_history));
+
+        service.getSummaryTempData('weekly').subscribe(value => {
+            expect(value).toBe(temp_history.data.temp_history as { [key: string]: SmartTemperatureModel[] });
+            done();
+        });
+        expect(httpClientSpy.get.calls.count())
+            .withContext('one call')
+            .toBe(1);
+    });
+});

webapp/frontend/src/app/modules/dashboard/dashboard.service.ts

@@ -1,16 +1,19 @@
-import { Injectable } from '@angular/core';
-import { HttpClient } from '@angular/common/http';
-import { BehaviorSubject, Observable } from 'rxjs';
-import { tap } from 'rxjs/operators';
-import { getBasePath } from 'app/app.routing';
+import {Injectable} from '@angular/core';
+import {HttpClient} from '@angular/common/http';
+import {BehaviorSubject, Observable} from 'rxjs';
+import {map, tap} from 'rxjs/operators';
+import {getBasePath} from 'app/app.routing';
+import {DeviceSummaryResponseWrapper} from 'app/core/models/device-summary-response-wrapper';
+import {DeviceSummaryModel} from 'app/core/models/device-summary-model';
+import {SmartTemperatureModel} from 'app/core/models/measurements/smart-temperature-model';
+import {DeviceSummaryTempResponseWrapper} from 'app/core/models/device-summary-temp-response-wrapper';
 
 @Injectable({
     providedIn: 'root'
 })
-export class DashboardService
-{
+export class DashboardService {
     // Observables
-    private _data: BehaviorSubject<any>;
+    private _data: BehaviorSubject<{ [p: string]: DeviceSummaryModel }>;
 
     /**
      * Constructor
@@ -32,8 +35,7 @@ export class DashboardService
     /**
      * Getter for data
      */
-    get data$(): Observable<any>
-    {
+    get data$(): Observable<{ [p: string]: DeviceSummaryModel }> {
         return this._data.asObservable();
     }
 
@@ -44,22 +46,28 @@ export class DashboardService
     /**
      * Get data
      */
-    getSummaryData(): Observable<any>
-    {
+    getSummaryData(): Observable<{ [key: string]: DeviceSummaryModel }> {
         return this._httpClient.get(getBasePath() + '/api/summary').pipe(
-            tap((response: any) => {
+            map((response: DeviceSummaryResponseWrapper) => {
+                // console.log("FILTERING=----", response.data.summary)
+                return response.data.summary
+            }),
+            tap((response: { [key: string]: DeviceSummaryModel }) => {
                 this._data.next(response);
             })
         );
     }
 
-    getSummaryTempData(durationKey: string): Observable<any>
-    {
+    getSummaryTempData(durationKey: string): Observable<{ [key: string]: SmartTemperatureModel[] }> {
         const params = {}
-        if(durationKey){
+        if (durationKey) {
             params['duration_key'] = durationKey
         }
 
-        return this._httpClient.get(getBasePath() + '/api/summary/temp', {params: params});
+        return this._httpClient.get(getBasePath() + '/api/summary/temp', {params: params}).pipe(
+            map((response: DeviceSummaryTempResponseWrapper) => {
+                return response.data.temp_history
+            })
+        );
     }
 }

webapp/frontend/src/app/modules/detail/detail.component.html

@@ -4,7 +4,7 @@
 
         <div class="flex items-center justify-between w-full my-4 px-4 xs:pr-0">
             <div class="mr-6">
-                <h2 class="m-0">Drive Details - {{device | deviceTitle:config.dashboardDisplay}} </h2>
+                <h2 class="m-0">Drive Details - {{device | deviceTitle:config.dashboard_display}} </h2>
                 <div class="text-secondary tracking-tight">Dive into S.M.A.R.T data</div>
             </div>
             <!-- Action buttons -->
@@ -17,7 +17,7 @@
                     <span class="ml-2">Export</span>
                 </button>
                 <button class="ml-2 xs:hidden"
-                        (click)="openDialog()"
+                        matTooltip="not yet implemented"
                         mat-stroked-button>
                     <mat-icon class="icon-size-20 rotate-90 mirror"
                               [svgIcon]="'tune'"></mat-icon>
@@ -56,12 +56,13 @@
                     <div *ngIf="device" class="my-2 col-span-2 lt-md:col-span-1">
                         <div>
                             <span class="inline-flex items-center font-bold text-xs px-2 py-2px rounded-full tracking-wide uppercase"
-                                   [ngClass]="{'red-200': device?.device_status != 0,
+                                  [ngClass]="{'red-200': deviceStatusForModelWithThreshold(device, !!smart_results, config.metrics.status_threshold) == 'failed',
                                   'green-200': device?.device_status == 0}">
                                 <span class="w-2 h-2 rounded-full mr-2"
                                       [ngClass]="{'bg-red': device?.device_status != 0,
                                   'bg-green': device?.device_status == 0}"></span>
-                                <span class="pr-2px leading-relaxed whitespace-no-wrap">{{device?.device_status | deviceStatus}}</span>
+                                <span
+                                    class="pr-2px leading-relaxed whitespace-no-wrap">{{device | deviceStatus:!!smart_results:config.metrics.status_threshold:true}}</span>
                             </span>
                         </div>
                         <div class="text-secondary text-md">Status</div>
@@ -106,7 +107,7 @@
                         <div class="text-secondary text-md">Firmware Version</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{device?.capacity | fileSize}}</div>
+                        <div>{{device?.capacity | fileSize:config.file_size_si_units}}</div>
                         <div class="text-secondary text-md">Capacity</div>
                     </div>
                     <div *ngIf="device?.rotational_speed" class="my-2 col-span-2 lt-md:col-span-1">
@@ -126,7 +127,7 @@
                         <div class="text-secondary text-md">Powered On</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{smart_results[0]?.temp | temperature:config.temperatureUnit:true}}</div>
+                        <div>{{smart_results[0]?.temp | temperature:config.temperature_unit:true}}</div>
                         <div class="text-secondary text-md">Temperature</div>
                     </div>
                 </div>

webapp/frontend/src/app/modules/detail/detail.component.spec.ts

@@ -1,25 +0,0 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { DetailComponent } from './detail.component';
-
-describe('DetailComponent', () => {
-  let component: DetailComponent;
-  let fixture: ComponentFixture<DetailComponent>;
-
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ DetailComponent ]
-    })
-    .compileComponents();
-  }));
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(DetailComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});