(v0.4.11) Automated packaging of release by Packagr

fixes #283

.dockerignore

@@ -0,0 +1,5 @@
+/vendor
+/.idea
+/.github
+/.git
+/webapp/frontend/node_modules

.gitattributes

@@ -0,0 +1,4 @@
+*.css linguist-detectable=false
+*.scss linguist-detectable=false
+*.js linguist-detectable=false
+*.ts linguist-detectable=false

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,24 +18,25 @@ If applicable, add screenshots to help explain your problem.
 
 **Log Files**
 If related to missing devices or SMART data, please run the `collector` in DEBUG mode, and attach the log file.
+See [/docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md) for other troubleshooting tips. 
 
 ```
 docker run -it --rm -p 8080:8080 \
+-v `pwd`/config:/opt/scrutiny/config \
 -v /run/udev:/run/udev:ro \
 --cap-add SYS_RAWIO \
 --device=/dev/sda \
 --device=/dev/sdb \
 -e DEBUG=true \
--e COLLECTOR_LOG_FILE=/tmp/collector.log \
--e SCRUTINY_LOG_FILE=/tmp/web.log \
+-e COLLECTOR_LOG_FILE=/opt/scrutiny/config/collector.log \
+-e SCRUTINY_LOG_FILE=/opt/scrutiny/config/web.log \
 --name scrutiny \
-analogj/scrutiny
+ghcr.io/analogj/scrutiny:master-omnibus
 
 # in another terminal trigger the collector
 docker exec scrutiny scrutiny-collector-metrics run
-
-# then use docker cp to copy the log files out of the container.
-docker cp scrutiny:/tmp/collector.log collector.log
-docker cp scrutiny:/tmp/web.log web.log
-
 ```
+
+The log files will be available on your host in the `config` directory. Please attach them to this issue. 
+
+Please also provide the output of `docker info`

.github/workflows/build.yaml

@@ -1,60 +1,85 @@
 name: CI
-# This workflow is triggered on pushes to the repository.
-on: [push]
+# This workflow is triggered on pushes & pull requests
+on: [pull_request]
 
 jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container: golang:1.13
+    container: techknowlogick/xgo:go-1.17.x
+
+    # Service containers to run with `build` (Required for end-to-end testing)
+    services:
+      influxdb:
+        image: influxdb:2.2
+        env:
+          DOCKER_INFLUXDB_INIT_MODE: setup
+          DOCKER_INFLUXDB_INIT_USERNAME: admin
+          DOCKER_INFLUXDB_INIT_PASSWORD: password12345
+          DOCKER_INFLUXDB_INIT_ORG: scrutiny
+          DOCKER_INFLUXDB_INIT_BUCKET: metrics
+          DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token
+        ports:
+          - 8086:8086
     env:
       PROJECT_PATH: /go/src/github.com/analogj/scrutiny
+      CGO_ENABLED: 1
     steps:
+      - name: Git
+        run: |
+          apt-get update && apt-get install -y software-properties-common
+          add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git
+          git --version
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Build
-        env:
-          GOOS: linux
-          GOARCH: amd64
+      - name: Test
         run: |
-          mkdir -p $PROJECT_PATH
-          cp -a $GITHUB_WORKSPACE/* $PROJECT_PATH/
+          mkdir -p $(dirname "$PROJECT_PATH")
+          cp -a $GITHUB_WORKSPACE $PROJECT_PATH
           cd $PROJECT_PATH
 
           go mod vendor
-
           go test -race -coverprofile=coverage.txt -covermode=atomic -v -tags "static" $(go list ./... | grep -v /vendor/)
+      - name: Generate coverage report
+        uses: codecov/codecov-action@v2
+        with:
+          files: ${{ env.PROJECT_PATH }}/coverage.txt
+          flags: unittests
+          fail_ci_if_error: true
+          verbose: true
+      - name: Build Binaries
+        run: |
 
-          go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go
-          go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go
+          cd $PROJECT_PATH
+          make all
 
-          chmod +x scrutiny-web-linux-amd64
-          chmod +x scrutiny-collector-metrics-linux-amd64
       - name: Archive
         uses: actions/upload-artifact@v2
         with:
-          name: binaries
+          name: binaries.zip
           path: |
-            ${{ env.PROJECT_PATH }}/scrutiny-web-linux-amd64
-            ${{ env.PROJECT_PATH }}/scrutiny-collector-metrics-linux-amd64
-      - uses: codecov/codecov-action@v1
+            /build/scrutiny-web-linux-amd64
+            /build/scrutiny-collector-metrics-linux-amd64
+            /build/scrutiny-web-linux-arm64
+            /build/scrutiny-collector-metrics-linux-arm64
+            /build/scrutiny-web-linux-arm-5
+            /build/scrutiny-collector-metrics-linux-arm-5
+            /build/scrutiny-web-linux-arm-6
+            /build/scrutiny-collector-metrics-linux-arm-6
+            /build/scrutiny-web-linux-arm-7
+            /build/scrutiny-collector-metrics-linux-arm-7
+            /build/scrutiny-web-windows-4.0-amd64.exe
+            /build/scrutiny-collector-metrics-windows-4.0-amd64.exe
+#            /build/scrutiny-web-darwin-arm64
+#            /build/scrutiny-collector-metrics-darwin-arm64
+#            /build/scrutiny-web-darwin-amd64
+#            /build/scrutiny-collector-metrics-darwin-amd64
+#            /build/scrutiny-web-freebsd-amd64
+#            /build/scrutiny-collector-metrics-freebsd-amd64
+      - uses: codecov/codecov-action@v2
         with:
           file: ${{ env.PROJECT_PATH }}/coverage.txt
           flags: unittests
-          fail_ci_if_error: true
-  build-docker:
-    name: Build Docker
-    runs-on: ubuntu-latest
-    container: docker:19.03.2
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Build Docker
-        run: |
-          docker build -t analogj/scrutiny -f docker/Dockerfile .
-          docker save -o docker-analogj-scrutiny-latest.tar analogj/scrutiny:latest
-      - name: Archive Docker
-        uses: actions/upload-artifact@v2
-        with:
-          name: docker
-          path: docker-analogj-scrutiny-latest.tar
+          fail_ci_if_error: false
+
+

.github/workflows/docker-build.yaml

@@ -0,0 +1,186 @@
+name: Docker
+on:
+  schedule:
+    - cron: '36 12 * * *'
+  push:
+    branches: [ master, beta ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  collector:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          flavor: |
+            latest=false
+          tags: |
+            type=ref,enable=true,event=branch,suffix=-collector
+            type=ref,enable=true,event=tag,suffix=-collector
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          context: .
+          file: docker/Dockerfile.collector
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+#          cache-from: type=gha
+#          cache-to: type=gha,mode=max
+
+  web:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: "Populate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
+      - name: "Generate frontend"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: node:lts
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            cd /work
+            make frontend && echo "print contents of /work/dist" && ls -alt /work/dist
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          flavor: |
+            latest=false
+          tags: |
+            type=ref,enable=true,event=branch,suffix=-web
+            type=ref,enable=true,event=tag,suffix=-web
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          context: .
+          file: docker/Dockerfile.web
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+#          cache-from: type=gha
+#          cache-to: type=gha,mode=max
+  omnibus:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: "Populate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
+      - name: "Generate frontend & version information"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: node:lts
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            cd /work
+            make frontend && echo "print contents of /work/dist" && ls -alt /work/dist
+
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          tags: |
+            type=ref,enable=true,event=branch,suffix=-omnibus
+            type=ref,enable=true,event=tag,suffix=-omnibus
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+          context: .
+          file: docker/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+#          cache-from: type=gha
+#          cache-to: type=gha,mode=max

.github/workflows/release-freebsd.yaml

@@ -0,0 +1,83 @@
+# compiles FreeBSD artifacts and attaches them to build
+name: Release FreeBSD
+
+on:
+  release:
+    # Only use the types keyword to narrow down the activity types that will trigger your workflow.
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'tag to build artifacts for'
+        required: true
+        default: 'v0.0.0'
+jobs:
+
+  release-freebsd:
+    name: Release FreeBSD
+    runs-on: macos-10.15
+    env:
+      PROJECT_PATH: /go/src/github.com/analogj/scrutiny
+      GOPATH: /go
+      GOOS: freebsd
+      GOARCH: amd64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{github.event.release.tag_name || github.event.inputs.tag_name }}
+      - name: Build Binaries
+        uses: vmactions/freebsd-vm@v0.1.5
+        with:
+          envs: 'PROJECT_PATH GOPATH GOOS GOARCH'
+          usesh: true
+          #TODO: lock go version using https://www.jeremymorgan.com/tutorials/golang/how-to-install-go-freebsd/
+          prepare: pkg install -y curl go gmake
+          run: |
+            pwd
+            ls -lah
+            whoami
+            freebsd-version
+
+            mkdir -p $(dirname "$PROJECT_PATH")
+            cp -R $GITHUB_WORKSPACE $PROJECT_PATH
+            cd $PROJECT_PATH
+
+            mkdir -p $GITHUB_WORKSPACE/dist
+
+            echo "building web binary (OS = ${GOOS}, ARCH = ${GOARCH})"
+            go build -ldflags "-extldflags=-static -X main.goos=${GOOS} -X main.goarch=${GOARCH}" -o $GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH} -tags "static netgo sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go
+
+            chmod +x "$GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH}"
+            file "$GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH}" || true
+            ldd "$GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH}" || true
+
+            echo "building collector binary (OS = ${GOOS}, ARCH = ${GOARCH})"
+            go build -ldflags "-extldflags=-static -X main.goos=${GOOS} -X main.goarch=${GOARCH}" -o $GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH} -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go
+
+            chmod +x "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}"
+            file "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}" || true
+            ldd "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}" || true
+
+      - name: Release Asset - Collector - freebsd-amd64
+        id: upload-release-asset2
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: './dist/scrutiny-collector-metrics-freebsd-amd64'
+          asset_name: scrutiny-collector-metrics-freebsd-amd64
+          asset_content_type: application/octet-stream
+
+      - name: Release Asset - Web - freebsd-amd64
+        id: upload-release-asset1
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: './dist/scrutiny-web-freebsd-amd64'
+          asset_name: scrutiny-web-freebsd-amd64
+          asset_content_type: application/octet-stream
+

.github/workflows/release-frontend.yaml

@@ -15,13 +15,15 @@ jobs:
         uses: actions/checkout@v2
         with:
           ref: ${{github.event.release.tag_name}}
+      - name: "Generate frontend version information"
+        run: "cd webapp/frontend && ./git.version.sh"
       - name: Build Frontend
         run: |
           cd webapp/frontend
           npm install -g @angular/cli@9.1.4
           npm install
           mkdir -p dist
-          ng build --output-path=dist --deploy-url="/web/" --base-href="/web/" --prod
+          npm run build:prod -- --output-path=dist
           tar -czf scrutiny-web-frontend.tar.gz dist
       - name: Upload Frontend Asset
         id: upload-release-asset3
@@ -30,6 +32,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
         with:
           upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: scrutiny-web-frontend.tar.gz
+          asset_path: './webapp/frontend/scrutiny-web-frontend.tar.gz'
           asset_name: scrutiny-web-frontend.tar.gz
-          asset_content_type: application/octet-stream
+          asset_content_type: application/gzip

.github/workflows/release.yaml

@@ -16,78 +16,88 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container: golang:1.13
+    container: techknowlogick/xgo:go-1.17.x
+    # Service containers to run with `build` (Required for end-to-end testing)
+    services:
+      influxdb:
+        image: influxdb:2.2
+        env:
+          DOCKER_INFLUXDB_INIT_MODE: setup
+          DOCKER_INFLUXDB_INIT_USERNAME: admin
+          DOCKER_INFLUXDB_INIT_PASSWORD: password12345
+          DOCKER_INFLUXDB_INIT_ORG: scrutiny
+          DOCKER_INFLUXDB_INIT_BUCKET: metrics
+          DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token
+        ports:
+          - 8086:8086
     env:
       PROJECT_PATH: /go/src/github.com/analogj/scrutiny
+      CGO_ENABLED: 1
     steps:
+      - name: Git
+        run: |
+          apt-get update && apt-get install -y software-properties-common
+          add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git
+          git --version
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Bump version
         id: bump_version
         uses: packagrio/action-bumpr-go@master
         with:
           version_bump_type: ${{ github.event.inputs.version_bump_type }}
           version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
-          github_token: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-      - name: Build
         env:
-          GOOS: linux
-          GOARCH: amd64
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }} # Leave this line unchanged
+      - name: Test
         run: |
-          mkdir -p $PROJECT_PATH
-          cp -a $GITHUB_WORKSPACE/. $PROJECT_PATH/
+          mkdir -p $(dirname "$PROJECT_PATH")
+          cp -a $GITHUB_WORKSPACE $PROJECT_PATH
           cd $PROJECT_PATH
 
           go mod vendor
-
           go test -v -tags "static" $(go list ./... | grep -v /vendor/)
 
-          go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go
-          go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go
+      - name: Build Binaries
+        run: |
 
-          chmod +x scrutiny-web-linux-amd64
-          chmod +x scrutiny-collector-metrics-linux-amd64
-      - name: Commit
-        uses: EndBug/add-and-commit@v4 # You can change this to use a specific version
-        with:
+          cd $PROJECT_PATH
+          make all
 
-          author_name: Jason Kulatunga
-          author_email: jason@thesparktree.com
-          cwd: ${{ env.PROJECT_PATH }}
-          force: false
-          signoff: true
-          message: '(${{steps.bump_version.outputs.release_version}}) Automated packaging of release by Packagr'
-          tag: ${{steps.bump_version.outputs.release_version}}
+          # restore modified dir to GH workspace.
+          cp -arf $PROJECT_PATH/. $GITHUB_WORKSPACE/
+
+          # copy all the build artifacts to the GH workspace
+          cp -arf /build/. $GITHUB_WORKSPACE/
+
+      - name: Commit Changes
+        id: commit
+        uses: packagrio/action-releasr-go@master
         env:
+          # This is necessary in order to push a commit to the repo
           GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }} # Leave this line unchanged
-
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
         with:
-          tag_name: ${{ steps.bump_version.outputs.release_version }}
-          release_name: Release ${{ steps.bump_version.outputs.release_version }}
-          draft: false
-          prerelease: false
-      - name: Upload Web Backend Release Asset
-        id: upload-release-asset1
-        uses: actions/upload-release-asset@v1
+          version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
+      - name: Publish Release
+        id: publish
+        uses: packagrio/action-publishr-go@master
         env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+          # This is necessary in order to push a commit to the repo
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }} # Leave this line unchanged
         with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: ${{ env.PROJECT_PATH }}/scrutiny-web-linux-amd64
-          asset_name: scrutiny-web-linux-amd64
-          asset_content_type: application/octet-stream
-      - name: Upload Collector Release Asset
-        id: upload-release-asset2
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: ${{ env.PROJECT_PATH }}/scrutiny-collector-metrics-linux-amd64
-          asset_name: scrutiny-collector-metrics-linux-amd64
-          asset_content_type: application/octet-stream
+          version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
+          upload_assets:
+            scrutiny-web-linux-amd64
+            scrutiny-collector-metrics-linux-amd64
+            scrutiny-web-linux-arm64
+            scrutiny-collector-metrics-linux-arm64
+            scrutiny-web-linux-arm-5
+            scrutiny-collector-metrics-linux-arm-5
+            scrutiny-web-linux-arm-6
+            scrutiny-collector-metrics-linux-arm-6
+            scrutiny-web-linux-arm-7
+            scrutiny-collector-metrics-linux-arm-7
+            scrutiny-web-windows-4.0-amd64.exe
+            scrutiny-collector-metrics-windows-4.0-amd64.exe

.github/workflows/sponsors.yaml

@@ -0,0 +1,15 @@
+name: Label sponsors
+on:
+  pull_request:
+    types: [opened]
+  issues:
+    types: [opened]
+jobs:
+  build:
+    name: is-sponsor-label
+    runs-on: ubuntu-latest
+    if: ${{ false }}
+    steps:
+      - uses: JasonEtco/is-sponsor-label-action@v1.2.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -58,9 +58,11 @@ scrutiny.db
 /dist/
 vendor
 /scrutiny
-/scrutiny-collector-metrics-linux-amd64
-/scrutiny-web-linux-amd64
+/scrutiny-collector-metrics-*
+/scrutiny-web-*
 scrutiny-*.db
 scrutiny_test.db
 scrutiny.yaml
 coverage.txt
+/config
+/influxdb

CONTRIBUTING.md

@@ -1,82 +1,143 @@
 # Contributing
 
-There are multiple ways to develop on the scrutiny codebase locally. The two most popular are:
-- Docker Development Container - only requires docker
-- Run Components Locally - requires smartmontools, golang & nodejs installed locally
+The Scrutiny repository is a [monorepo](https://en.wikipedia.org/wiki/Monorepo) containing source code for:
+- Scrutiny Backend Server (API)
+- Scrutiny Frontend Angular SPA
+- S.M.A.R.T Collector
 
-## Docker Development
-```
-docker build -f docker/Dockerfile . -t analogj/scrutiny
-docker run -it --rm -p 8080:8080 \
--v /run/udev:/run/udev:ro \
---cap-add SYS_RAWIO \
---device=/dev/sda \
---device=/dev/sdb \
-analogj/scrutiny
-/scrutiny/bin/scrutiny-collector-metrics run
-```
+Depending on the functionality you are adding, you may need to setup a development environment for 1 or more projects. 
+
+# Modifying the Scrutiny Backend Server (API)
+
+1. install the [Go runtime](https://go.dev/doc/install) (v1.17+)
+2. download the `scrutiny-web-frontend.tar.gz` for the [latest release](https://github.com/AnalogJ/scrutiny/releases/latest). Extract to a folder named `dist`
+3. create a `scrutiny.yaml` config file
+    ```yaml
+    # config file for local development. store as scrutiny.yaml
+    version: 1
+    
+    web:
+      listen:
+        port: 8080
+        host: 0.0.0.0
+      database:
+        # can also set absolute path here
+        location: ./scrutiny.db
+      src:
+        frontend:
+          path: ./dist
+      influxdb:
+        retention_policy: false
+    
+    log:
+      file: 'web.log' #absolute or relative paths allowed, eg. web.log
+      level: DEBUG
+
+    ```
+4. start a InfluxDB docker container. 
+    ```bash
+    docker run -p 8086:8086 --rm influxdb:2.2
+    ```
+5. start the scrutiny web server
+    ```bash
+    go mod vendor
+    go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
+    ```
+6. open your browser to [http://localhost:8080/web](http://localhost:8080/web)
+
+# Modifying the Scrutiny Frontend Angular SPA
+
+The frontend is written in Angular. If you're working on the frontend and can use mocked data rather than a real backend, you can follow the instructions below:
+
+1. install [NodeJS](https://nodejs.org/en/download/)
+2. start the Angular Frontend Application
+    ```bash
+    cd webapp/frontend
+    npm install
+    npm run start -- --deploy-url="/web/" --base-href="/web/" --port 4200
+    ```
+3. open your browser and visit [http://localhost:4200/web](http://localhost:4200/web)
+
+# Modifying both Scrutiny Backend and Frontend Applications
+If you're developing a feature that requires changes to the backend and the frontend, or a frontend feature that requires real data, 
+you'll need to follow the steps below:
+
+1. install the [Go runtime](https://go.dev/doc/install) (v1.17+)
+2. install [NodeJS](https://nodejs.org/en/download/)
+3. create a `scrutiny.yaml` config file
+    ```yaml
+    # config file for local development. store as scrutiny.yaml
+    version: 1
+    
+    web:
+      listen:
+        port: 8080
+        host: 0.0.0.0
+      database:
+        # can also set absolute path here
+        location: ./scrutiny.db
+      src:
+        frontend:
+          path: ./dist
+      influxdb:
+        retention_policy: false
+    
+    log:
+      file: 'web.log' #absolute or relative paths allowed, eg. web.log
+      level: DEBUG
+
+    ```
+4. start a InfluxDB docker container.
+    ```bash
+    docker run -p 8086:8086 --rm influxdb:2.2
+    ```
+5. build the Angular Frontend Application
+    ```bash
+    cd webapp/frontend
+    npm install
+    npm run build:prod -- --watch --output-path=../../dist
+    # Note: if you do not add `--prod` flag, app will display mocked data for api calls.
+    ```
+6. start the scrutiny web server
+    ```bash
+    go mod vendor
+    go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
+    ```
+7. open your browser to [http://localhost:8080/web](http://localhost:8080/web)
 
 
-## Local Development
+If you'd like to populate the database with some test data,  you can run the following commands:
 
-### Frontend
-The frontend is written in Angular.
-If you're working on the frontend and can use mocked data rather than a real backend, you can use
-```
-cd webapp/frontend && ng serve
-```
-
-However, if you need to also run the backend, and use real data, you'll need to run the following command:
-```
-cd webapp/frontend && ng build --watch --output-path=../../dist --deploy-url="/web/" --base-href="/web/" --prod
-```
-
-> Note: if you do not add `--prod` flag, app will display mocked data for api calls.
-
-### Backend
-
-If you're using the `ng build` command above to generate your frontend, you'll need to create a custom config file and
-override the `web.src.frontend.path` value.
+> NOTE: you may need to update the `local_time` key within the JSON file, any timestamps older than ~3 weeks will be automatically ignored
+> (since the downsampling & retention policy takes effect at 2 weeks)
+> This is done automatically by the `webapp/backend/pkg/models/testdata/helper.go` script
 
 ```
-# config file for local development. store as scrutiny.yaml
-version: 1
-
-web:
-  listen:
-    port: 8080
-    host: 0.0.0.0
-  database:
-    # can also set absolute path here
-    location: ./scrutiny.db
-  src:
-    frontend:
-      path: ./dist
+docker run -p 8086:8086 --rm influxdb:2.2
 
 
-log:
-  file: 'web.log' #absolute or relative paths allowed, eg. web.log
-  level: DEBUG
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/web/testdata/register-devices-req.json localhost:8080/api/devices/register
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata-date.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata-date2.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-fail2.json localhost:8080/api/device/0x5000cca264ec3183/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-nvme.json localhost:8080/api/device/0x5002538e40a22954/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-scsi.json localhost:8080/api/device/0x5000cca252c859cc/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-scsi2.json localhost:8080/api/device/0x5000cca264ebc248/smart
+go run webapp/backend/pkg/models/testdata/helper.go
+
+curl localhost:8080/api/summary
 
 ```
 
-Once you've created a config file, you can pass it to the scrutiny binary during startup.
-
-```
-go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
-```
-
-Now visit http://localhost:8080
-
-
-### Collector
+# Modifying the Collector
 ```
 brew install smartmontools
 go run collector/cmd/collector-metrics/collector-metrics.go run --debug
 ```
 
 
-## Debugging
+# Debugging
 
 If you need more verbose logs for debugging, you can use the following environmental variables:
 
@@ -95,3 +156,32 @@ Finally, you can copy the files from the scrutiny container to your host using t
 docker cp scrutiny:/tmp/collector.log collector.log
 docker cp scrutiny:/tmp/web.log web.log
 ```
+
+# Docker Development
+
+```
+docker build -f docker/Dockerfile . -t chcr.io/analogj/scrutiny:master-omnibus
+docker run -it --rm -p 8080:8080 \
+-v /run/udev:/run/udev:ro \
+--cap-add SYS_RAWIO \
+--device=/dev/sda \
+--device=/dev/sdb \
+ghcr.io/analogj/scrutiny:master-omnibus
+/opt/scrutiny/bin/scrutiny-collector-metrics run
+```
+
+
+# Running Tests
+
+```bash
+docker run -p 8086:8086 -d --rm \
+-e DOCKER_INFLUXDB_INIT_MODE=setup \
+-e DOCKER_INFLUXDB_INIT_USERNAME=admin \
+-e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
+-e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
+-e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
+-e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \
+influxdb:2.2
+go test ./...
+
+```

Makefile

@@ -0,0 +1,66 @@
+export CGO_ENABLED = 1
+
+GO_WORKSPACE ?= /go/src/github.com/analogj/scrutiny
+
+BINARY=\
+  linux/amd64   \
+  linux/arm-5   \
+  linux/arm-6   \
+  linux/arm-7   \
+  linux/arm64   \
+
+.ONESHELL: # Applies to every targets in the file! .ONESHELL instructs make to invoke a single instance of the shell and provide it with the entire recipe, regardless of how many lines it contains.
+.PHONY: all $(BINARY)
+all: $(BINARY) windows/amd64
+
+$(BINARY): OS = $(word 1,$(subst /, ,$*))
+$(BINARY): ARCH = $(word 2,$(subst /, ,$*))
+$(BINARY): build/scrutiny-web-%:
+	@echo "building web binary (OS = $(OS), ARCH = $(ARCH))"
+	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-web -tags "static netgo sqlite_omit_load_extension" ${GO_WORKSPACE}/webapp/backend/cmd/scrutiny/
+
+	chmod +x "/build/scrutiny-web-$(OS)-$(ARCH)"
+	file "/build/scrutiny-web-$(OS)-$(ARCH)" || true
+	ldd "/build/scrutiny-web-$(OS)-$(ARCH)" || true
+
+	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
+	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
+
+	chmod +x "/build/scrutiny-collector-metrics-$(OS)-$(ARCH)"
+	file "/build/scrutiny-collector-metrics-$(OS)-$(ARCH)" || true
+	ldd "/build/scrutiny-collector-metrics-$(OS)-$(ARCH)" || true
+
+windows/amd64: export OS = windows
+windows/amd64: export ARCH = amd64
+windows/amd64:
+	@echo "building web binary (OS = $(OS), ARCH = $(ARCH))"
+	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-web -tags "static netgo sqlite_omit_load_extension" ${GO_WORKSPACE}/webapp/backend/cmd/scrutiny/
+
+	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
+	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
+
+
+docker-collector:
+	@echo "building collector docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.collector -t analogj/scrutiny-dev:collector .
+
+docker-web:
+	@echo "building web docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile.web -t analogj/scrutiny-dev:web .
+
+docker-omnibus:
+	@echo "building omnibus docker image"
+	docker build --build-arg TARGETARCH=amd64 -f docker/Dockerfile -t analogj/scrutiny-dev:omnibus .
+
+# reduce logging, disable angular-cli analytics for ci environment
+frontend: export NPM_CONFIG_LOGLEVEL = warn
+frontend: export NG_CLI_ANALYTICS = false
+frontend:
+	cd webapp/frontend
+	npm install -g @angular/cli@9.1.4
+	mkdir -p $(CURDIR)/dist
+	npm ci
+	npm run build:prod -- --output-path=$(CURDIR)/dist
+
+# clean:
+# 	rm scrutiny-collector-metrics-* scrutiny-web-*

README.md

@@ -14,7 +14,6 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/AnalogJ/scrutiny?style=flat-square)](https://goreportcard.com/report/github.com/AnalogJ/scrutiny)
 [![GitHub release](http://img.shields.io/github/release/AnalogJ/scrutiny.svg?style=flat-square)](https://github.com/AnalogJ/scrutiny/releases)
 
-
 WebUI for smartd S.M.A.R.T monitoring
 
 > NOTE: Scrutiny is a Work-in-Progress and still has some rough edges.
@@ -47,70 +46,205 @@ Scrutiny is a simple but focused application, with a couple of core features:
 - Customized thresholds using real world failure rates
 - Temperature tracking
 - Provided as an all-in-one Docker image (but can be installed manually)
-- (Future) Configurable Alerting/Notifications via Webhooks
+- Future Configurable Alerting/Notifications via Webhooks
 - (Future) Hard Drive performance testing & tracking
 
 # Getting Started
 
+## RAID/Virtual Drives
+
+Scrutiny uses `smartctl --scan` to detect devices/drives.
+
+- All RAID controllers supported by `smartctl` are automatically supported by Scrutiny.
+    - While some RAID controllers support passing through the underlying SMART data to `smartctl` others do not.
+    - In some cases `--scan` does not correctly detect the device type, returning [incomplete SMART data](https://github.com/AnalogJ/scrutiny/issues/45).
+    Scrutiny supports overriding detected device type via the config file: see [example.collector.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.collector.yaml)
+- If you use docker, you **must** pass though the RAID virtual disk to the container using `--device` (see below)
+    - This device may be in `/dev/*` or `/dev/bus/*`.
+    - If you're unsure, run `smartctl --scan` on your host, and pass all listed devices to the container.
+
+See [docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](./docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md) for help
+
 ## Docker
 
 If you're using Docker, getting started is as simple as running the following command:
 
+> See [docker/example.omnibus.docker-compose.yml](./docker/example.omnibus.docker-compose.yml) for a docker-compose file. 
+
 ```bash
-docker run -it --rm -p 8080:8080 \
--v /run/udev:/run/udev:ro \
---cap-add SYS_RAWIO \
---device=/dev/sda \
---device=/dev/sdb \
---name scrutiny \
-analogj/scrutiny
+docker run -it --rm -p 8080:8080 -p 8086:8086 \
+  -v `pwd`/scrutiny:/opt/scrutiny/config \
+  -v `pwd`/influxdb2:/opt/scrutiny/influxdb \
+  -v /run/udev:/run/udev:ro \
+  --cap-add SYS_RAWIO \
+  --device=/dev/sda \
+  --device=/dev/sdb \
+  --name scrutiny \
+  ghcr.io/analogj/scrutiny:master-omnibus
 ```
 
 - `/run/udev` is necessary to provide the Scrutiny collector with access to your device metadata
 - `--cap-add SYS_RAWIO` is necessary to allow `smartctl` permission to query your device SMART data
-    - NOTE: If you have NVMe drives, you must use `--cap-add SYS_ADMIN` instead. See issue [#26](https://github.com/AnalogJ/scrutiny/issues/26#issuecomment-696817130)
-- `--device` entries are required to ensure that your hard disk devices are accessible within the container
-- `analogj/scrutiny` is a omnibus image, containing both the webapp server (frontend & api) as well as the S.M.A.R.T metric collector. (see below)
+    - NOTE: If you have **NVMe** drives, you must add `--cap-add SYS_ADMIN` as well. See issue [#26](https://github.com/AnalogJ/scrutiny/issues/26#issuecomment-696817130)
+- `--device` entries are required to ensure that your hard disk devices are accessible within the container.
+- `ghcr.io/analogj/scrutiny:master-omnibus` is a omnibus image, containing both the webapp server (frontend & api) as well as the S.M.A.R.T metric collector. (see below)
 
 ### Hub/Spoke Deployment
 
 In addition to the Omnibus image (available under the `latest` tag) there are 2 other Docker images available:
 
-- `analogj/scrutiny:collector` - Contains the Scrutiny data collector, `smartctl` binary and cron-like scheduler. You can run one collector on each server.
-- `analogj/scrutiny:web` - Contains the Web UI, API and Database. Only one container necessary
+- `ghcr.io/analogj/scrutiny:master-collector` - Contains the Scrutiny data collector, `smartctl` binary and cron-like scheduler. You can run one collector on each server.
+- `ghcr.io/analogj/scrutiny:master-web` - Contains the Web UI, API and Database. Only one container necessary
+
+> See [docker/example.hubspoke.docker-compose.yml](./docker/example.hubspoke.docker-compose.yml) for a docker-compose file.
 
 ```bash
-docker run -it --rm -p 8080:8080 \
---name scrutiny-web \
-analogj/scrutiny:web
+docker run --rm -p 8086:8086 \
+  -v `pwd`/influxdb2:/var/lib/influxdb2 \
+  --name scrutiny-influxdb \
+  influxdb:2.2
 
-docker run -it --rm \
--v /run/udev:/run/udev:ro \
---cap-add SYS_RAWIO \
---device=/dev/sda \
---device=/dev/sdb \
--e SCRUTINY_API_ENDPOINT=http://SCRUTINY_WEB_IPADDRESS:8080 \
---name scrutiny-collector \
-analogj/scrutiny:collector
+docker run --rm -p 8080:8080 \
+  -v `pwd`/scrutiny:/opt/scrutiny/config \
+  --name scrutiny-web \
+  ghcr.io/analogj/scrutiny:master-web
+
+docker run --rm \
+  -v /run/udev:/run/udev:ro \
+  --cap-add SYS_RAWIO \
+  --device=/dev/sda \
+  --device=/dev/sdb \
+  -e COLLECTOR_API_ENDPOINT=http://SCRUTINY_WEB_IPADDRESS:8080 \
+  --name scrutiny-collector \
+  ghcr.io/analogj/scrutiny:master-collector
 ```
 
+## Manual Installation (without-Docker)
+
+While the easiest way to get started with [Scrutiny is using Docker](https://github.com/AnalogJ/scrutiny#docker),
+it is possible to run it manually without much work. You can even mix and match, using Docker for one component and
+a manual installation for the other.
+
+See [docs/INSTALL_MANUAL.md](docs/INSTALL_MANUAL.md) for instructions.
 
 ## Usage
 
 Once scrutiny is running, you can open your browser to `http://localhost:8080` and take a look at the dashboard.
 
-Initially it will be empty, however after the first collector run, you'll be greeted with a list of all your hard drives and their current smart status.
+If you're using the omnibus image, the collector should already have run, and your dashboard should be populate with every
+drive that Scrutiny detected. The collector is configured to run once a day, but you can trigger it manually by running the command below.
 
-The collector is configured to run once a day, but you can trigger it manually by running the following command
+For users of the docker Hub/Spoke deployment or manual install: initially the dashboard will be empty.
+After the first collector run, you'll be greeted with a list of all your hard drives and their current smart status.
 
-```
-docker exec scrutiny /scrutiny/bin/scrutiny-collector-metrics run
+```bash
+docker exec scrutiny /opt/scrutiny/bin/scrutiny-collector-metrics run
 ```
 
 # Configuration
-We support a global YAML configuration file that must be located at /scrutiny/config/scrutiny.yaml
+By default Scrutiny looks for its YAML configuration files in `/opt/scrutiny/config`
+
+There are two configuration files available:
+
+- Webapp/API config via `scrutiny.yaml` - [example.scrutiny.yaml](example.scrutiny.yaml).
+- Collector config via `collector.yaml` - [example.collector.yaml](example.collector.yaml).
+
+Neither file is required, however if provided, it allows you to configure how Scrutiny functions.
+
+## Cron Schedule
+Unfortunately the Cron schedule cannot be configured via the `collector.yaml` (as the collector binary needs to be trigged by a scheduler/cron).
+However, if you are using the official `ghcr.io/analogj/scrutiny:master-collector` or `ghcr.io/analogj/scrutiny:master-omnibus` docker images, 
+you can use the `COLLECTOR_CRON_SCHEDULE` environmental variable to override the default cron schedule (daily @ midnight - `0 0 * * *`).
+
+`docker run -e COLLECTOR_CRON_SCHEDULE="0 0 * * *" ...`
+
+## Notifications
+
+Scrutiny supports sending SMART device failure notifications via the following services:
+- Custom Script (data provided via environmental variables)
+- Email
+- Webhooks
+- Discord
+- Gotify
+- Hangouts
+- IFTTT
+- Join
+- Mattermost
+- Pushbullet
+- Pushover
+- Slack
+- Teams
+- Telegram
+- Tulip
+
+Check the `notify.urls` section of [example.scrutiny.yml](example.scrutiny.yaml) for examples.
+
+For more information and troubleshooting, see the [TROUBLESHOOTING_NOTIFICATIONS.md](./docs/TROUBLESHOOTING_NOTIFICATIONS.md) file
+
+### Testing Notifications
+
+You can test that your notifications are configured correctly by posting an empty payload to the notifications health check API.
+
+```bash
+curl -X POST http://localhost:8080/api/health/notify
+```
+
+# Debug mode & Log Files
+Scrutiny provides various methods to change the log level to debug and generate log files.
+
+## Web Server/API
+
+You can use environmental variables to enable debug logging and/or log files for the web server:
+
+```bash
+DEBUG=true
+SCRUTINY_LOG_FILE=/tmp/web.log
+```
+
+You can configure the log level and log file in the config file:
+
+```yml
+log:
+  file: '/tmp/web.log'
+  level: DEBUG
+```
+
+Or if you're not using docker, you can pass CLI arguments to the web server during startup:
+
+```bash
+scrutiny start --debug --log-file /tmp/web.log
+```
+
+## Collector
+
+You can use environmental variables to enable debug logging and/or log files for the collector:
+
+```bash
+DEBUG=true
+COLLECTOR_LOG_FILE=/tmp/collector.log
+```
+
+Or if you're not using docker, you can pass CLI arguments to the collector during startup:
+
+```bash
+scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
+```
+
+# Supported Architectures
+
+
+| Architecture Name | Binaries | Docker |
+| --- | --- | --- |
+| amd64 | :white_check_mark: | :white_check_mark: |
+| arm-5 | :white_check_mark: |  |
+| arm-6 | :white_check_mark: |  |
+| arm-7 | :white_check_mark: | web/collector only. see [#236](https://github.com/AnalogJ/scrutiny/issues/236)  |
+| arm64 | :white_check_mark: | :white_check_mark: |
+| freebsd | collector only. see [#238](https://github.com/AnalogJ/scrutiny/issues/238) |  | 
+| macos-amd64 |  | :white_check_mark: |
+| macos-arm64 |  | :white_check_mark: |
+| windows-amd64 | :white_check_mark: |  |
 
-Check the [example.scrutiny.yml](example.scrutiny.yaml) file for a fully commented version.
 
 # Contributing
 

collector/cmd/collector-metrics/collector-metrics.go

@@ -3,11 +3,14 @@ package main
 import (
 	"fmt"
 	"github.com/analogj/scrutiny/collector/pkg/collector"
+	"github.com/analogj/scrutiny/collector/pkg/config"
+	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/version"
 	"github.com/sirupsen/logrus"
 	"io"
 	"log"
 	"os"
+	"strings"
 	"time"
 
 	utils "github.com/analogj/go-util/utils"
@@ -20,6 +23,20 @@ var goarch string
 
 func main() {
 
+	config, err := config.Create()
+	if err != nil {
+		fmt.Printf("FATAL: %+v\n", err)
+		os.Exit(1)
+	}
+
+	//we're going to load the config file manually, since we need to validate it.
+	err = config.ReadConfig("/opt/scrutiny/config/collector.yaml") // Find and read the config file
+	if _, ok := err.(errors.ConfigFileMissingError); ok {          // Handle errors reading the config file
+		//ignore "could not find config file"
+	} else if err != nil {
+		os.Exit(1)
+	}
+
 	cli.CommandHelpTemplate = `NAME:
    {{.HelpName}} - {{.Usage}}
 USAGE:
@@ -75,21 +92,48 @@ OPTIONS:
 				Name:  "run",
 				Usage: "Run the scrutiny smartctl metrics collector",
 				Action: func(c *cli.Context) error {
+					if c.IsSet("config") {
+						err = config.ReadConfig(c.String("config")) // Find and read the config file
+						if err != nil {                             // Handle errors reading the config file
+							//ignore "could not find config file"
+							fmt.Printf("Could not find config file at specified path: %s", c.String("config"))
+							return err
+						}
+					}
+					//override config with flags if set
+					if c.IsSet("host-id") {
+						config.Set("host.id", c.String("host-id")) // set/override the host-id using CLI.
+					}
+
+					if c.Bool("debug") {
+						config.Set("log.level", "DEBUG")
+					}
+
+					if c.IsSet("log-file") {
+						config.Set("log.file", c.String("log-file"))
+					}
+
+					if c.IsSet("api-endpoint") {
+						//if the user is providing an api-endpoint with a basepath (eg. http://localhost:8080/scrutiny),
+						//we need to ensure the basepath has a trailing slash, otherwise the url.Parse() path concatenation doesnt work.
+						apiEndpoint := strings.TrimSuffix(c.String("api-endpoint"), "/") + "/"
+						config.Set("api.endpoint", apiEndpoint)
+					}
 
 					collectorLogger := logrus.WithFields(logrus.Fields{
 						"type": "metrics",
 					})
 
-					if c.Bool("debug") {
-						logrus.SetLevel(logrus.DebugLevel)
+					if level, err := logrus.ParseLevel(config.GetString("log.level")); err == nil {
+						logrus.SetLevel(level)
 					} else {
 						logrus.SetLevel(logrus.InfoLevel)
 					}
 
-					if c.IsSet("log-file") {
-						logFile, err := os.OpenFile(c.String("log-file"), os.O_CREATE|os.O_WRONLY, 0644)
+					if config.IsSet("log.file") && len(config.GetString("log.file")) > 0 {
+						logFile, err := os.OpenFile(config.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
 						if err != nil {
-							logrus.Errorf("Failed to open log file %s for output: %s", c.String("log-file"), err)
+							logrus.Errorf("Failed to open log file %s for output: %s", config.GetString("log.file"), err)
 							return err
 						}
 						defer logFile.Close()
@@ -97,8 +141,9 @@ OPTIONS:
 					}
 
 					metricCollector, err := collector.CreateMetricsCollector(
+						config,
 						collectorLogger,
-						c.String("api-endpoint"),
+						config.GetString("api.endpoint"),
 					)
 
 					if err != nil {
@@ -109,17 +154,20 @@ OPTIONS:
 				},
 
 				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:  "config",
+						Usage: "Specify the path to the devices file",
+					},
 					&cli.StringFlag{
 						Name:    "api-endpoint",
 						Usage:   "The api server endpoint",
-						Value:   "http://localhost:8080",
-						EnvVars: []string{"SCRUTINY_API_ENDPOINT"},
+						EnvVars: []string{"COLLECTOR_API_ENDPOINT", "SCRUTINY_API_ENDPOINT"},
+						//SCRUTINY_API_ENDPOINT is deprecated, but kept for backwards compatibility
 					},
 
 					&cli.StringFlag{
 						Name:    "log-file",
 						Usage:   "Path to file for logging. Leave empty to use STDOUT",
-						Value:   "",
 						EnvVars: []string{"COLLECTOR_LOG_FILE"},
 					},
 
@@ -128,12 +176,19 @@ OPTIONS:
 						Usage:   "Enable debug logging",
 						EnvVars: []string{"COLLECTOR_DEBUG", "DEBUG"},
 					},
+
+					&cli.StringFlag{
+						Name:    "host-id",
+						Usage:   "Host identifier/label, used for grouping devices",
+						Value:   "",
+						EnvVars: []string{"COLLECTOR_HOST_ID"},
+					},
 				},
 			},
 		},
 	}
 
-	err := app.Run(os.Args)
+	err = app.Run(os.Args)
 	if err != nil {
 		log.Fatal(color.HiRedString("ERROR: %v", err))
 	}

collector/cmd/collector-selftest/collector-selftest.go

@@ -96,6 +96,7 @@ OPTIONS:
 						logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
 					}
 
+					//TODO: pass in the collector, use configuration from collector-metrics
 					stCollector, err := collector.CreateSelfTestCollector(
 						collectorLogger,
 						c.String("api-endpoint"),
@@ -113,7 +114,7 @@ OPTIONS:
 						Name:    "api-endpoint",
 						Usage:   "The api server endpoint",
 						Value:   "http://localhost:8080",
-						EnvVars: []string{"SCRUTINY_API_ENDPOINT"},
+						EnvVars: []string{"COLLECTOR_API_ENDPOINT"},
 					},
 
 					&cli.StringFlag{

collector/pkg/collector/base.go

@@ -40,6 +40,7 @@ func (c *BaseCollector) postJson(url string, body interface{}, target interface{
 	return json.NewDecoder(r.Body).Decode(target)
 }
 
+// http://www.linuxguide.it/command_line/linux-manpage/do.php?file=smartctl#sect7
 func (c *BaseCollector) LogSmartctlExitCode(exitCode int) {
 	if exitCode&0x01 != 0 {
 		c.logger.Errorln("smartctl could not parse commandline")

collector/pkg/collector/metrics.go

@@ -4,7 +4,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
+	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/collector/pkg/models"
@@ -16,21 +17,25 @@ import (
 )
 
 type MetricsCollector struct {
+	config config.Interface
 	BaseCollector
 	apiEndpoint *url.URL
+	shell       shell.Interface
 }
 
-func CreateMetricsCollector(logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
+func CreateMetricsCollector(appConfig config.Interface, logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
 	apiEndpointUrl, err := url.Parse(apiEndpoint)
 	if err != nil {
 		return MetricsCollector{}, err
 	}
 
 	sc := MetricsCollector{
+		config:      appConfig,
 		apiEndpoint: apiEndpointUrl,
 		BaseCollector: BaseCollector{
 			logger: logger,
 		},
+		shell: shell.Create(),
 	}
 
 	return sc, nil
@@ -43,12 +48,13 @@ func (mc *MetricsCollector) Run() error {
 	}
 
 	apiEndpoint, _ := url.Parse(mc.apiEndpoint.String())
-	apiEndpoint.Path = "/api/devices/register"
+	apiEndpoint, _ = apiEndpoint.Parse("api/devices/register") //this acts like filepath.Join()
 
 	deviceRespWrapper := new(models.DeviceWrapper)
 
 	deviceDetector := detect.Detect{
 		Logger: mc.logger,
+		Config: mc.config,
 	}
 	detectedStorageDevices, err := deviceDetector.Start()
 	if err != nil {
@@ -67,6 +73,7 @@ func (mc *MetricsCollector) Run() error {
 
 	if !deviceRespWrapper.Success {
 		mc.logger.Errorln("An error occurred while retrieving filtered devices")
+		mc.logger.Debugln(deviceRespWrapper)
 		return errors.ApiServerCommunicationError("An error occurred while retrieving filtered devices")
 	} else {
 		mc.logger.Debugln(deviceRespWrapper)
@@ -103,16 +110,21 @@ func (mc *MetricsCollector) Validate() error {
 //func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
 func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceType string) {
 	//defer wg.Done()
+	if len(deviceWWN) == 0 {
+		mc.logger.Errorf("no device WWN detected for %s. Skipping collection for this device (no data association possible).\n", deviceName)
+		return
+	}
 	mc.logger.Infof("Collecting smartctl results for %s\n", deviceName)
 
-	args := []string{"-x", "-j"}
+	fullDeviceName := fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName)
+	args := strings.Split(mc.config.GetCommandMetricsSmartArgs(fullDeviceName), " ")
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
 	if len(deviceType) > 0 && deviceType != "scsi" && deviceType != "ata" {
-		args = append(args, "-d", deviceType)
+		args = append(args, "--device", deviceType)
 	}
-	args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName))
+	args = append(args, fullDeviceName)
 
-	result, err := common.ExecCmd(mc.logger, "smartctl", args, "", os.Environ())
+	result, err := mc.shell.Command(mc.logger, "smartctl", args, "", os.Environ())
 	resultBytes := []byte(result)
 	if err != nil {
 		if exitError, ok := err.(*exec.ExitError); ok {
@@ -136,7 +148,7 @@ func (mc *MetricsCollector) Publish(deviceWWN string, payload []byte) error {
 	mc.logger.Infof("Publishing smartctl results for %s\n", deviceWWN)
 
 	apiEndpoint, _ := url.Parse(mc.apiEndpoint.String())
-	apiEndpoint.Path = fmt.Sprintf("/api/device/%s/smart", strings.ToLower(deviceWWN))
+	apiEndpoint, _ = apiEndpoint.Parse(fmt.Sprintf("api/device/%s/smart", strings.ToLower(deviceWWN)))
 
 	resp, err := httpClient.Post(apiEndpoint.String(), "application/json", bytes.NewBuffer(payload))
 	if err != nil {

collector/pkg/collector/metrics_test.go

@@ -0,0 +1,38 @@
+package collector
+
+import (
+	"github.com/stretchr/testify/require"
+	"net/url"
+	"testing"
+)
+
+func TestApiEndpointParse(t *testing.T) {
+	baseURL, _ := url.Parse("http://localhost:8080/")
+
+	url1, _ := baseURL.Parse("d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url1.String())
+
+	url2, _ := baseURL.Parse("/d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url2.String())
+}
+
+func TestApiEndpointParse_WithBasepathWithoutTrailingSlash(t *testing.T) {
+	baseURL, _ := url.Parse("http://localhost:8080/scrutiny")
+
+	//This testcase is unexpected and can cause issues. We need to ensure the apiEndpoint always has a trailing slash.
+	url1, _ := baseURL.Parse("d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url1.String())
+
+	url2, _ := baseURL.Parse("/d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url2.String())
+}
+
+func TestApiEndpointParse_WithBasepathWithTrailingSlash(t *testing.T) {
+	baseURL, _ := url.Parse("http://localhost:8080/scrutiny/")
+
+	url1, _ := baseURL.Parse("d/e")
+	require.Equal(t, "http://localhost:8080/scrutiny/d/e", url1.String())
+
+	url2, _ := baseURL.Parse("/d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url2.String())
+}

collector/pkg/common/shell/factory.go

@@ -0,0 +1,5 @@
+package shell
+
+func Create() Interface {
+	return new(localShell)
+}

collector/pkg/common/shell/interface.go

@@ -0,0 +1,11 @@
+package shell
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+// Create mock using:
+// mockgen -source=collector/pkg/common/shell/interface.go -destination=collector/pkg/common/shell/mock/mock_shell.go
+type Interface interface {
+	Command(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error)
+}

collector/pkg/common/shell/local_shell.go

@@ -1,4 +1,4 @@
-package common
+package shell
 
 import (
 	"bytes"
@@ -10,12 +10,22 @@ import (
 	"strings"
 )
 
-func ExecCmd(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
+type localShell struct{}
+
+func (s *localShell) Command(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
 	logger.Infof("Executing command: %s %s", cmdName, strings.Join(cmdArgs, " "))
 
 	cmd := exec.Command(cmdName, cmdArgs...)
 	var stdBuffer bytes.Buffer
-	mw := io.MultiWriter(logger.Logger.Out, &stdBuffer)
+
+	logWriters := []io.Writer{
+		&stdBuffer,
+	}
+	if logger.Logger.Level == logrus.DebugLevel {
+		logWriters = append(logWriters, logger.Logger.Out)
+	}
+
+	mw := io.MultiWriter(logWriters...)
 
 	cmd.Stdout = mw
 	cmd.Stderr = mw

collector/pkg/common/shell/local_shell_test.go

@@ -1,33 +1,33 @@
-package common_test
+package shell
 
 import (
-	"github.com/analogj/scrutiny/collector/pkg/common"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"os/exec"
 	"testing"
 )
 
-func TestExecCmd(t *testing.T) {
+func TestLocalShellCommand(t *testing.T) {
 	t.Parallel()
 
 	//setup
-
+	testShell := localShell{}
 	//test
-	result, err := common.ExecCmd(logrus.WithField("exec", "test"), "echo", []string{"hello world"}, "", nil)
+	result, err := testShell.Command(logrus.WithField("exec", "test"), "echo", []string{"hello world"}, "", nil)
 
 	//assert
 	require.NoError(t, err)
 	require.Equal(t, "hello world\n", result)
 }
 
-func TestExecCmd_Date(t *testing.T) {
+func TestLocalShellCommand_Date(t *testing.T) {
 	t.Parallel()
 
 	//setup
+	testShell := localShell{}
 
 	//test
-	_, err := common.ExecCmd(logrus.WithField("exec", "test"), "date", []string{}, "", nil)
+	_, err := testShell.Command(logrus.WithField("exec", "test"), "date", []string{}, "", nil)
 
 	//assert
 	require.NoError(t, err)
@@ -51,13 +51,14 @@ func TestExecCmd_Date(t *testing.T) {
 //}
 //
 
-func TestExecCmd_InvalidCommand(t *testing.T) {
+func TestLocalShellCommand_InvalidCommand(t *testing.T) {
 	t.Parallel()
 
 	//setup
+	testShell := localShell{}
 
 	//test
-	_, err := common.ExecCmd(logrus.WithField("exec", "test"), "invalid_binary", []string{}, "", nil)
+	_, err := testShell.Command(logrus.WithField("exec", "test"), "invalid_binary", []string{}, "", nil)
 
 	//assert
 	_, castOk := err.(*exec.ExitError)

collector/pkg/common/shell/mock/mock_shell.go

@@ -0,0 +1,50 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: collector/pkg/common/shell/interface.go
+
+// Package mock_shell is a generated GoMock package.
+package mock_shell
+
+import (
+	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
+	logrus "github.com/sirupsen/logrus"
+)
+
+// MockInterface is a mock of Interface interface.
+type MockInterface struct {
+	ctrl     *gomock.Controller
+	recorder *MockInterfaceMockRecorder
+}
+
+// MockInterfaceMockRecorder is the mock recorder for MockInterface.
+type MockInterfaceMockRecorder struct {
+	mock *MockInterface
+}
+
+// NewMockInterface creates a new mock instance.
+func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
+	mock := &MockInterface{ctrl: ctrl}
+	mock.recorder = &MockInterfaceMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
+	return m.recorder
+}
+
+// Command mocks base method.
+func (m *MockInterface) Command(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Command", logger, cmdName, cmdArgs, workingDir, environ)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Command indicates an expected call of Command.
+func (mr *MockInterfaceMockRecorder) Command(logger, cmdName, cmdArgs, workingDir, environ interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Command", reflect.TypeOf((*MockInterface)(nil).Command), logger, cmdName, cmdArgs, workingDir, environ)
+}

collector/pkg/config/config.go

@@ -0,0 +1,187 @@
+package config
+
+import (
+	"fmt"
+	"github.com/analogj/go-util/utils"
+	"github.com/analogj/scrutiny/collector/pkg/errors"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+	"log"
+	"os"
+	"sort"
+	"strings"
+)
+
+// When initializing this class the following methods must be called:
+// Config.New
+// Config.Init
+// This is done automatically when created via the Factory.
+type configuration struct {
+	*viper.Viper
+
+	deviceOverrides []models.ScanOverride
+}
+
+//Viper uses the following precedence order. Each item takes precedence over the item below it:
+// explicit call to Set
+// flag
+// env
+// config
+// key/value store
+// default
+
+func (c *configuration) Init() error {
+	c.Viper = viper.New()
+	//set defaults
+	c.SetDefault("host.id", "")
+
+	c.SetDefault("devices", []string{})
+
+	c.SetDefault("log.level", "INFO")
+	c.SetDefault("log.file", "")
+
+	c.SetDefault("api.endpoint", "http://localhost:8080")
+
+	c.SetDefault("commands.metrics_scan_args", "--scan --json")
+	c.SetDefault("commands.metrics_info_args", "--info --json")
+	c.SetDefault("commands.metrics_smart_args", "--xall --json")
+
+	//c.SetDefault("collect.short.command", "-a -o on -S on")
+
+	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
+	c.SetConfigType("yaml")
+	//c.SetConfigName("drawbridge")
+	//c.AddConfigPath("$HOME/")
+
+	//CLI options will be added via the `Set()` function
+	return nil
+}
+
+func (c *configuration) ReadConfig(configFilePath string) error {
+	configFilePath, err := utils.ExpandPath(configFilePath)
+	if err != nil {
+		return err
+	}
+
+	if !utils.FileExists(configFilePath) {
+		log.Printf("No configuration file found at %v. Using Defaults.", configFilePath)
+		return errors.ConfigFileMissingError("The configuration file could not be found.")
+	}
+
+	//validate config file contents
+	//err = c.ValidateConfigFile(configFilePath)
+	//if err != nil {
+	//	log.Printf("Config file at `%v` is invalid: %s", configFilePath, err)
+	//	return err
+	//}
+
+	log.Printf("Loading configuration file: %s", configFilePath)
+
+	config_data, err := os.Open(configFilePath)
+	if err != nil {
+		log.Printf("Error reading configuration file: %s", err)
+		return err
+	}
+
+	err = c.MergeConfig(config_data)
+	if err != nil {
+		return err
+	}
+
+	return c.ValidateConfig()
+}
+
+// This function ensures that the merged config works correctly.
+func (c *configuration) ValidateConfig() error {
+
+	//TODO:
+	// check that device prefix matches OS
+	// check that schema of config file is valid
+
+	// check that the collector commands are valid
+	commandArgStrings := map[string]string{
+		"commands.metrics_scan_args":  c.GetString("commands.metrics_scan_args"),
+		"commands.metrics_info_args":  c.GetString("commands.metrics_info_args"),
+		"commands.metrics_smart_args": c.GetString("commands.metrics_smart_args"),
+	}
+
+	errorStrings := []string{}
+	for configKey, commandArgString := range commandArgStrings {
+		args := strings.Split(commandArgString, " ")
+		//ensure that the args string contains `--json` or `-j` flag
+		containsJsonFlag := false
+		containsDeviceFlag := false
+		for _, flag := range args {
+			if strings.HasPrefix(flag, "--json") || strings.HasPrefix(flag, "-j") {
+				containsJsonFlag = true
+			}
+			if strings.HasPrefix(flag, "--device") || strings.HasPrefix(flag, "-d") {
+				containsDeviceFlag = true
+			}
+		}
+
+		if !containsJsonFlag {
+			errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' is missing '--json' flag", configKey))
+		}
+
+		if containsDeviceFlag {
+			errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' must not contain '--device' or '-d' flag", configKey))
+		}
+	}
+	//sort(errorStrings)
+	sort.Strings(errorStrings)
+
+	if len(errorStrings) == 0 {
+		return nil
+	} else {
+		return errors.ConfigValidationError(strings.Join(errorStrings, ", "))
+	}
+}
+
+func (c *configuration) GetDeviceOverrides() []models.ScanOverride {
+	// we have to support 2 types of device types.
+	// - simple device type (device_type: 'sat')
+	// and list of device types (type: \n- 3ware,0 \n- 3ware,1 \n- 3ware,2)
+	// GetString will return "" if this is a list of device types.
+
+	if c.deviceOverrides == nil {
+		overrides := []models.ScanOverride{}
+		c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
+		c.deviceOverrides = overrides
+	}
+
+	return c.deviceOverrides
+}
+
+func (c *configuration) GetCommandMetricsInfoArgs(deviceName string) string {
+	overrides := c.GetDeviceOverrides()
+
+	for _, deviceOverrides := range overrides {
+		if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) {
+			//found matching device
+			if len(deviceOverrides.Commands.MetricsInfoArgs) > 0 {
+				return deviceOverrides.Commands.MetricsInfoArgs
+			} else {
+				return c.GetString("commands.metrics_info_args")
+			}
+		}
+	}
+	return c.GetString("commands.metrics_info_args")
+}
+
+func (c *configuration) GetCommandMetricsSmartArgs(deviceName string) string {
+	overrides := c.GetDeviceOverrides()
+
+	for _, deviceOverrides := range overrides {
+		if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) {
+			//found matching device
+			if len(deviceOverrides.Commands.MetricsSmartArgs) > 0 {
+				return deviceOverrides.Commands.MetricsSmartArgs
+			} else {
+				return c.GetString("commands.metrics_smart_args")
+			}
+		}
+	}
+	return c.GetString("commands.metrics_smart_args")
+}

collector/pkg/config/config_test.go

@@ -0,0 +1,127 @@
+package config_test
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/config"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/stretchr/testify/require"
+	"path"
+	"testing"
+)
+
+func TestConfiguration_InvalidConfigPath(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig("does_not_exist.yaml")
+
+	//assert
+	require.Error(t, err, "should return an error")
+}
+
+func TestConfiguration_GetScanOverrides_Simple(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "simple_device.yaml"))
+	require.NoError(t, err, "should correctly load simple device config")
+	scanOverrides := testConfig.GetDeviceOverrides()
+
+	//assert
+	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat"}, Ignore: false}}, scanOverrides)
+}
+
+func TestConfiguration_GetScanOverrides_Ignore(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "ignore_device.yaml"))
+	require.NoError(t, err, "should correctly load ignore device config")
+	scanOverrides := testConfig.GetDeviceOverrides()
+
+	//assert
+	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}, scanOverrides)
+}
+
+func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "raid_device.yaml"))
+	require.NoError(t, err, "should correctly load ignore device config")
+	scanOverrides := testConfig.GetDeviceOverrides()
+
+	//assert
+	require.Equal(t, []models.ScanOverride{
+		{
+			Device:     "/dev/bus/0",
+			DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
+			Ignore:     false,
+		},
+		{
+			Device:     "/dev/twa0",
+			DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
+			Ignore:     false,
+		}}, scanOverrides)
+}
+
+func TestConfiguration_InvalidCommands_MissingJson(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_missing_json.yaml"))
+	require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_scan_args' is missing '--json' flag"`, "should throw an error because json flag is missing")
+}
+
+func TestConfiguration_InvalidCommands_IncludesDevice(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_includes_device.yaml"))
+	require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_info_args' must not contain '--device' or '-d' flag, configuration key 'commands.metrics_smart_args' must not contain '--device' or '-d' flag"`, "should throw an error because device flags detected")
+}
+
+func TestConfiguration_OverrideCommands(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "override_commands.yaml"))
+	require.NoError(t, err, "should not throw an error")
+	require.Equal(t, "--xall --json -T permissive", testConfig.GetString("commands.metrics_smart_args"))
+}
+
+func TestConfiguration_OverrideDeviceCommands_MetricsInfoArgs(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "override_device_commands.yaml"))
+	require.NoError(t, err, "should correctly override device command")
+
+	//assert
+	require.Equal(t, "--info --json -T permissive", testConfig.GetCommandMetricsInfoArgs("/dev/sda"))
+	require.Equal(t, "--info --json", testConfig.GetCommandMetricsInfoArgs("/dev/sdb"))
+	//require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Commands: {MetricsInfoArgs: "--info --json -T "}}}, scanOverrides)
+}

collector/pkg/config/factory.go

@@ -0,0 +1,9 @@
+package config
+
+func Create() (Interface, error) {
+	config := new(configuration)
+	if err := config.Init(); err != nil {
+		return nil, err
+	}
+	return config, nil
+}

collector/pkg/config/interface.go

@@ -0,0 +1,28 @@
+package config
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/spf13/viper"
+)
+
+// Create mock using:
+// mockgen -source=collector/pkg/config/interface.go -destination=collector/pkg/config/mock/mock_config.go
+type Interface interface {
+	Init() error
+	ReadConfig(configFilePath string) error
+	Set(key string, value interface{})
+	SetDefault(key string, value interface{})
+
+	AllSettings() map[string]interface{}
+	IsSet(key string) bool
+	Get(key string) interface{}
+	GetBool(key string) bool
+	GetInt(key string) int
+	GetString(key string) string
+	GetStringSlice(key string) []string
+	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
+
+	GetDeviceOverrides() []models.ScanOverride
+	GetCommandMetricsInfoArgs(deviceName string) string
+	GetCommandMetricsSmartArgs(deviceName string) string
+}

collector/pkg/config/mock/mock_config.go

@@ -0,0 +1,247 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: collector/pkg/config/interface.go
+
+// Package mock_config is a generated GoMock package.
+package mock_config
+
+import (
+	reflect "reflect"
+
+	models "github.com/analogj/scrutiny/collector/pkg/models"
+	gomock "github.com/golang/mock/gomock"
+	viper "github.com/spf13/viper"
+)
+
+// MockInterface is a mock of Interface interface.
+type MockInterface struct {
+	ctrl     *gomock.Controller
+	recorder *MockInterfaceMockRecorder
+}
+
+// MockInterfaceMockRecorder is the mock recorder for MockInterface.
+type MockInterfaceMockRecorder struct {
+	mock *MockInterface
+}
+
+// NewMockInterface creates a new mock instance.
+func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
+	mock := &MockInterface{ctrl: ctrl}
+	mock.recorder = &MockInterfaceMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
+	return m.recorder
+}
+
+// AllSettings mocks base method.
+func (m *MockInterface) AllSettings() map[string]interface{} {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AllSettings")
+	ret0, _ := ret[0].(map[string]interface{})
+	return ret0
+}
+
+// AllSettings indicates an expected call of AllSettings.
+func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings))
+}
+
+// Get mocks base method.
+func (m *MockInterface) Get(key string) interface{} {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", key)
+	ret0, _ := ret[0].(interface{})
+	return ret0
+}
+
+// Get indicates an expected call of Get.
+func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key)
+}
+
+// GetBool mocks base method.
+func (m *MockInterface) GetBool(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetBool", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// GetBool indicates an expected call of GetBool.
+func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key)
+}
+
+// GetCommandMetricsInfoArgs mocks base method.
+func (m *MockInterface) GetCommandMetricsInfoArgs(deviceName string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCommandMetricsInfoArgs", deviceName)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetCommandMetricsInfoArgs indicates an expected call of GetCommandMetricsInfoArgs.
+func (mr *MockInterfaceMockRecorder) GetCommandMetricsInfoArgs(deviceName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsInfoArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsInfoArgs), deviceName)
+}
+
+// GetCommandMetricsSmartArgs mocks base method.
+func (m *MockInterface) GetCommandMetricsSmartArgs(deviceName string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCommandMetricsSmartArgs", deviceName)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetCommandMetricsSmartArgs indicates an expected call of GetCommandMetricsSmartArgs.
+func (mr *MockInterfaceMockRecorder) GetCommandMetricsSmartArgs(deviceName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsSmartArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsSmartArgs), deviceName)
+}
+
+// GetDeviceOverrides mocks base method.
+func (m *MockInterface) GetDeviceOverrides() []models.ScanOverride {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDeviceOverrides")
+	ret0, _ := ret[0].([]models.ScanOverride)
+	return ret0
+}
+
+// GetDeviceOverrides indicates an expected call of GetDeviceOverrides.
+func (mr *MockInterfaceMockRecorder) GetDeviceOverrides() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceOverrides", reflect.TypeOf((*MockInterface)(nil).GetDeviceOverrides))
+}
+
+// GetInt mocks base method.
+func (m *MockInterface) GetInt(key string) int {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetInt", key)
+	ret0, _ := ret[0].(int)
+	return ret0
+}
+
+// GetInt indicates an expected call of GetInt.
+func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
+}
+
+// GetString mocks base method.
+func (m *MockInterface) GetString(key string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetString", key)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// GetString indicates an expected call of GetString.
+func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key)
+}
+
+// GetStringSlice mocks base method.
+func (m *MockInterface) GetStringSlice(key string) []string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetStringSlice", key)
+	ret0, _ := ret[0].([]string)
+	return ret0
+}
+
+// GetStringSlice indicates an expected call of GetStringSlice.
+func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key)
+}
+
+// Init mocks base method.
+func (m *MockInterface) Init() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Init")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Init indicates an expected call of Init.
+func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
+}
+
+// IsSet mocks base method.
+func (m *MockInterface) IsSet(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsSet", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsSet indicates an expected call of IsSet.
+func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
+}
+
+// ReadConfig mocks base method.
+func (m *MockInterface) ReadConfig(configFilePath string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReadConfig indicates an expected call of ReadConfig.
+func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
+}
+
+// Set mocks base method.
+func (m *MockInterface) Set(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Set", key, value)
+}
+
+// Set indicates an expected call of Set.
+func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
+}
+
+// SetDefault mocks base method.
+func (m *MockInterface) SetDefault(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetDefault", key, value)
+}
+
+// SetDefault indicates an expected call of SetDefault.
+func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
+}
+
+// UnmarshalKey mocks base method.
+func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{key, rawVal}
+	for _, a := range decoderOpts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "UnmarshalKey", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UnmarshalKey indicates an expected call of UnmarshalKey.
+func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{key, rawVal}, decoderOpts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...)
+}

collector/pkg/config/testdata/ignore_device.yaml

@@ -0,0 +1,4 @@
+version: 1
+devices:
+  - device: /dev/sda
+    ignore: true

collector/pkg/config/testdata/invalid_commands_includes_device.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan --json' # used to detect devices
+  metrics_info_args: '--info --json --device=sat' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json -d sat' # used to retrieve smart data for each device.

collector/pkg/config/testdata/invalid_commands_missing_json.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan' # used to detect devices
+  metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json' # used to retrieve smart data for each device.

collector/pkg/config/testdata/override_commands.yaml

@@ -0,0 +1,4 @@
+commands:
+  metrics_scan_args: '--scan --json' # used to detect devices
+  metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny
+  metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device.

collector/pkg/config/testdata/override_device_commands.yaml

@@ -0,0 +1,5 @@
+version: 1
+devices:
+  - device: /dev/sda
+    commands:
+      metrics_info_args: "--info --json -T permissive"

collector/pkg/config/testdata/raid_device.yaml

@@ -0,0 +1,19 @@
+version: 1
+devices:
+  - device: /dev/bus/0
+    type:
+      - megaraid,14
+      - megaraid,15
+      - megaraid,18
+      - megaraid,19
+      - megaraid,20
+      - megaraid,21
+
+  - device: /dev/twa0
+    type:
+      - 3ware,0
+      - 3ware,1
+      - 3ware,2
+      - 3ware,3
+      - 3ware,4
+      - 3ware,5

collector/pkg/config/testdata/simple_device.yaml

@@ -0,0 +1,27 @@
+version: 1
+devices:
+  - device: /dev/sda
+    type: 'sat'
+#
+#  # example to show how to ignore a specific disk/device.
+#  - device: /dev/sda
+#    ignore: true
+#
+#  # examples showing how to force smartctl to detect disks inside a raid array/virtual disk
+#  - device: /dev/bus/0
+#    type:
+#      - megaraid,14
+#      - megaraid,15
+#      - megaraid,18
+#      - megaraid,19
+#      - megaraid,20
+#      - megaraid,21
+#
+#  - device: /dev/twa0
+#    type:
+#      - 3ware,0
+#      - 3ware,1
+#      - 3ware,2
+#      - 3ware,3
+#      - 3ware,4
+#      - 3ware,5

collector/pkg/detect/detect.go

@@ -3,10 +3,10 @@ package detect
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
+	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"github.com/denisbrodbeck/machineid"
 	"github.com/sirupsen/logrus"
 	"os"
 	"strings"
@@ -14,6 +14,8 @@ import (
 
 type Detect struct {
 	Logger *logrus.Entry
+	Config config.Interface
+	Shell  shell.Interface
 }
 
 //private/common functions
@@ -24,9 +26,10 @@ type Detect struct {
 //
 // To handle these issues, we have OS specific wrapper functions that update/modify these detected devices.
 // models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
-func (d *Detect) smartctlScan() ([]models.Device, error) {
+func (d *Detect) SmartctlScan() ([]models.Device, error) {
 	//we use smartctl to detect all the drives available.
-	detectedDeviceConnJson, err := common.ExecCmd(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
+	args := strings.Split(d.Config.GetString("commands.metrics_scan_args"), " ")
+	detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ())
 	if err != nil {
 		d.Logger.Errorf("Error scanning for devices: %v", err)
 		return nil, err
@@ -39,14 +42,7 @@ func (d *Detect) smartctlScan() ([]models.Device, error) {
 		return nil, err
 	}
 
-	detectedDevices := []models.Device{}
-
-	for _, detectedDevice := range detectedDeviceConns.Devices {
-		detectedDevices = append(detectedDevices, models.Device{
-			DeviceType: detectedDevice.Type,
-			DeviceName: strings.TrimPrefix(detectedDevice.Name, DevicePrefix()),
-		})
-	}
+	detectedDevices := d.TransformDetectedDevices(detectedDeviceConns)
 
 	return detectedDevices, nil
 }
@@ -55,16 +51,16 @@ func (d *Detect) smartctlScan() ([]models.Device, error) {
 // It has a couple of issues however:
 // - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
 // - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
-func (d *Detect) smartCtlInfo(device *models.Device) error {
-
-	args := []string{"--info", "-j"}
+func (d *Detect) SmartCtlInfo(device *models.Device) error {
+	fullDeviceName := fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName)
+	args := strings.Split(d.Config.GetCommandMetricsInfoArgs(fullDeviceName), " ")
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
 	if len(device.DeviceType) > 0 && device.DeviceType != "scsi" && device.DeviceType != "ata" {
-		args = append(args, "-d", device.DeviceType)
+		args = append(args, "--device", device.DeviceType)
 	}
-	args = append(args, fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName))
+	args = append(args, fullDeviceName)
 
-	availableDeviceInfoJson, err := common.ExecCmd(d.Logger, "smartctl", args, "", os.Environ())
+	availableDeviceInfoJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ())
 	if err != nil {
 		d.Logger.Errorf("Could not retrieve device information for %s: %v", device.DeviceName, err)
 		return err
@@ -77,8 +73,8 @@ func (d *Detect) smartCtlInfo(device *models.Device) error {
 		return err
 	}
 
-	//DeviceType and DeviceName are already populated.
-	//WWN
+	//WWN: this is a serial number/world-wide number that will not change.
+	//DeviceType and DeviceName are already populated, however may change between collector runs (eg. config/host restart)
 	//InterfaceType:
 	device.ModelName = availableDeviceInfo.ModelName
 	device.InterfaceSpeed = availableDeviceInfo.InterfaceSpeed.Current.String
@@ -106,11 +102,70 @@ func (d *Detect) smartCtlInfo(device *models.Device) error {
 		d.Logger.Info("Using WWN Fallback")
 		d.wwnFallback(device)
 	}
+	if len(device.WWN) == 0 {
+		// no WWN populated after WWN lookup and fallback. we need to throw an error
+		errMsg := fmt.Sprintf("no WWN (or fallback) populated for device: %s. Device will be registered, but no data will be published for this device. ", device.DeviceName)
+		d.Logger.Errorf(errMsg)
+		return fmt.Errorf(errMsg)
+	}
 
 	return nil
 }
 
-//uses https://github.com/denisbrodbeck/machineid to get a OS specific unique machine ID.
-func (d *Detect) getMachineId() (string, error) {
-	return machineid.ProtectedID("scrutiny")
+// function will remove devices that are marked for "ignore" in config file
+// will also add devices that are specified in config file, but "missing" from smartctl --scan
+// this function will also update the deviceType to the option specified in config.
+func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []models.Device {
+	groupedDevices := map[string][]models.Device{}
+
+	for _, scannedDevice := range detectedDeviceConns.Devices {
+
+		deviceFile := strings.ToLower(scannedDevice.Name)
+
+		detectedDevice := models.Device{
+			HostId:     d.Config.GetString("host.id"),
+			DeviceType: scannedDevice.Type,
+			DeviceName: strings.TrimPrefix(deviceFile, DevicePrefix()),
+		}
+
+		//find (or create) a slice to contain the devices in this group
+		if groupedDevices[deviceFile] == nil {
+			groupedDevices[deviceFile] = []models.Device{}
+		}
+
+		// add this scanned device to the group
+		groupedDevices[deviceFile] = append(groupedDevices[deviceFile], detectedDevice)
+	}
+
+	//now tha we've "grouped" all the devices, lets override any groups specified in the config file.
+
+	for _, overrideDevice := range d.Config.GetDeviceOverrides() {
+		overrideDeviceFile := strings.ToLower(overrideDevice.Device)
+
+		if overrideDevice.Ignore {
+			// this device file should be deleted if it exists
+			delete(groupedDevices, overrideDeviceFile)
+		} else {
+			//create a new device group, and replace the one generated by smartctl --scan
+			overrideDeviceGroup := []models.Device{}
+
+			for _, overrideDeviceType := range overrideDevice.DeviceType {
+				overrideDeviceGroup = append(overrideDeviceGroup, models.Device{
+					HostId:     d.Config.GetString("host.id"),
+					DeviceType: overrideDeviceType,
+					DeviceName: strings.TrimPrefix(overrideDeviceFile, DevicePrefix()),
+				})
+			}
+
+			groupedDevices[overrideDeviceFile] = overrideDeviceGroup
+		}
+	}
+
+	//flatten map
+	detectedDevices := []models.Device{}
+	for _, group := range groupedDevices {
+		detectedDevices = append(detectedDevices, group...)
+	}
+
+	return detectedDevices
 }

collector/pkg/detect/detect_test.go

@@ -0,0 +1,236 @@
+package detect_test
+
+import (
+	mock_shell "github.com/analogj/scrutiny/collector/pkg/common/shell/mock"
+	mock_config "github.com/analogj/scrutiny/collector/pkg/config/mock"
+	"github.com/analogj/scrutiny/collector/pkg/detect"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/golang/mock/gomock"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+	"io/ioutil"
+	"testing"
+)
+
+func TestDetect_SmartctlScan(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
+	fakeShell := mock_shell.NewMockInterface(mockCtrl)
+	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_simple.json")
+	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
+
+	d := detect.Detect{
+		Logger: logrus.WithFields(logrus.Fields{}),
+		Shell:  fakeShell,
+		Config: fakeConfig,
+	}
+
+	//test
+	scannedDevices, err := d.SmartctlScan()
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, 7, len(scannedDevices))
+	require.Equal(t, "scsi", scannedDevices[0].DeviceType)
+}
+
+func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
+	fakeShell := mock_shell.NewMockInterface(mockCtrl)
+	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_megaraid.json")
+	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
+
+	d := detect.Detect{
+		Logger: logrus.WithFields(logrus.Fields{}),
+		Shell:  fakeShell,
+		Config: fakeConfig,
+	}
+
+	//test
+	scannedDevices, err := d.SmartctlScan()
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, 2, len(scannedDevices))
+	require.Equal(t, []models.Device{
+		models.Device{DeviceName: "bus/0", DeviceType: "megaraid,0"},
+		models.Device{DeviceName: "bus/0", DeviceType: "megaraid,1"},
+	}, scannedDevices)
+}
+
+func TestDetect_SmartctlScan_Nvme(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
+	fakeShell := mock_shell.NewMockInterface(mockCtrl)
+	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_nvme.json")
+	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
+
+	d := detect.Detect{
+		Logger: logrus.WithFields(logrus.Fields{}),
+		Shell:  fakeShell,
+		Config: fakeConfig,
+	}
+
+	//test
+	scannedDevices, err := d.SmartctlScan()
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, 1, len(scannedDevices))
+	require.Equal(t, []models.Device{
+		models.Device{DeviceName: "nvme0", DeviceType: "nvme"},
+	}, scannedDevices)
+}
+
+func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
+	detectedDevices := models.Scan{
+		Devices: []models.ScanDevice{
+			{
+				Name:     "/dev/sda",
+				InfoName: "/dev/sda",
+				Protocol: "scsi",
+				Type:     "scsi",
+			},
+		},
+	}
+
+	d := detect.Detect{
+		Config: fakeConfig,
+	}
+
+	//test
+	transformedDevices := d.TransformDetectedDevices(detectedDevices)
+
+	//assert
+	require.Equal(t, "sda", transformedDevices[0].DeviceName)
+	require.Equal(t, "scsi", transformedDevices[0].DeviceType)
+}
+
+func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+
+	detectedDevices := models.Scan{
+		Devices: []models.ScanDevice{
+			{
+				Name:     "/dev/sda",
+				InfoName: "/dev/sda",
+				Protocol: "scsi",
+				Type:     "scsi",
+			},
+		},
+	}
+
+	d := detect.Detect{
+		Config: fakeConfig,
+	}
+
+	//test
+	transformedDevices := d.TransformDetectedDevices(detectedDevices)
+
+	//assert
+	require.Equal(t, []models.Device{}, transformedDevices)
+}
+
+func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{
+		{
+			Device:     "/dev/bus/0",
+			DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
+			Ignore:     false,
+		},
+		{
+			Device:     "/dev/twa0",
+			DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
+			Ignore:     false,
+		}})
+	detectedDevices := models.Scan{
+		Devices: []models.ScanDevice{
+			{
+				Name:     "/dev/bus/0",
+				InfoName: "/dev/bus/0",
+				Protocol: "scsi",
+				Type:     "scsi",
+			},
+		},
+	}
+
+	d := detect.Detect{
+		Config: fakeConfig,
+	}
+
+	//test
+	transformedDevices := d.TransformDetectedDevices(detectedDevices)
+
+	//assert
+	require.Equal(t, 12, len(transformedDevices))
+}
+
+func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
+	fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
+	detectedDevices := models.Scan{
+		Devices: []models.ScanDevice{
+			{
+				Name:     "/dev/sda",
+				InfoName: "/dev/sda",
+				Protocol: "ata",
+				Type:     "ata",
+			},
+		},
+	}
+
+	d := detect.Detect{
+		Config: fakeConfig,
+	}
+
+	//test
+	transformedDevices := d.TransformDetectedDevices(detectedDevices)
+
+	//assert
+	require.Equal(t, 1, len(transformedDevices))
+	require.Equal(t, "sat+megaraid", transformedDevices[0].DeviceType)
+}

collector/pkg/detect/devices_darwin.go

@@ -1,6 +1,7 @@
 package detect
 
 import (
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/jaypipes/ghw"
 	"strings"
@@ -11,8 +12,9 @@ func DevicePrefix() string {
 }
 
 func (d *Detect) Start() ([]models.Device, error) {
-	// call the base/common functionality to get a list of devicess
-	detectedDevices, err := d.smartctlScan()
+	d.Shell = shell.Create()
+	// call the base/common functionality to get a list of devices
+	detectedDevices, err := d.SmartctlScan()
 	if err != nil {
 		return nil, err
 	}
@@ -25,7 +27,7 @@ func (d *Detect) Start() ([]models.Device, error) {
 
 	//inflate device info for detected devices.
 	for ndx, _ := range detectedDevices {
-		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+		d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
 	}
 
 	return detectedDevices, nil
@@ -92,7 +94,7 @@ func (d *Detect) wwnFallback(detectedDevice *models.Device) {
 	block, err := ghw.Block()
 	if err == nil {
 		for _, disk := range block.Disks {
-			if disk.Name == detectedDevice.DeviceName {
+			if disk.Name == detectedDevice.DeviceName && strings.ToLower(disk.WWN) != "unknown" {
 				d.Logger.Debugf("Found matching block device. WWN: %s", disk.WWN)
 				detectedDevice.WWN = disk.WWN
 				break

collector/pkg/detect/devices_freebsd.go

@@ -0,0 +1,51 @@
+package detect
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/jaypipes/ghw"
+	"strings"
+)
+
+func DevicePrefix() string {
+	return "/dev/"
+}
+
+func (d *Detect) Start() ([]models.Device, error) {
+	d.Shell = shell.Create()
+	// call the base/common functionality to get a list of devices
+	detectedDevices, err := d.SmartctlScan()
+	if err != nil {
+		return nil, err
+	}
+
+	//inflate device info for detected devices.
+	for ndx, _ := range detectedDevices {
+		d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+	}
+
+	return detectedDevices, nil
+}
+
+//WWN values NVMe and SCSI
+func (d *Detect) wwnFallback(detectedDevice *models.Device) {
+	block, err := ghw.Block()
+	if err == nil {
+		for _, disk := range block.Disks {
+			if disk.Name == detectedDevice.DeviceName && strings.ToLower(disk.WWN) != "unknown" {
+				d.Logger.Debugf("Found matching block device. WWN: %s", disk.WWN)
+				detectedDevice.WWN = disk.WWN
+				break
+			}
+		}
+	}
+
+	//no WWN found, or could not open Block devices. Either way, fallback to serial number
+	if len(detectedDevice.WWN) == 0 {
+		d.Logger.Debugf("WWN is empty, falling back to serial number: %s", detectedDevice.SerialNumber)
+		detectedDevice.WWN = detectedDevice.SerialNumber
+	}
+
+	//wwn must always be lowercase.
+	detectedDevice.WWN = strings.ToLower(detectedDevice.WWN)
+}

collector/pkg/detect/devices_linux.go

@@ -1,8 +1,12 @@
 package detect
 
 import (
+	"fmt"
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/jaypipes/ghw"
+	"io/ioutil"
+	"path/filepath"
 	"strings"
 )
 
@@ -11,15 +15,17 @@ func DevicePrefix() string {
 }
 
 func (d *Detect) Start() ([]models.Device, error) {
+	d.Shell = shell.Create()
 	// call the base/common functionality to get a list of devices
-	detectedDevices, err := d.smartctlScan()
+	detectedDevices, err := d.SmartctlScan()
 	if err != nil {
 		return nil, err
 	}
 
 	//inflate device info for detected devices.
 	for ndx, _ := range detectedDevices {
-		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+		d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+		populateUdevInfo(&detectedDevices[ndx]) //ignore errors.
 	}
 
 	return detectedDevices, nil
@@ -30,7 +36,7 @@ func (d *Detect) wwnFallback(detectedDevice *models.Device) {
 	block, err := ghw.Block()
 	if err == nil {
 		for _, disk := range block.Disks {
-			if disk.Name == detectedDevice.DeviceName {
+			if disk.Name == detectedDevice.DeviceName && strings.ToLower(disk.WWN) != "unknown" {
 				d.Logger.Debugf("Found matching block device. WWN: %s", disk.WWN)
 				detectedDevice.WWN = disk.WWN
 				break
@@ -47,3 +53,51 @@ func (d *Detect) wwnFallback(detectedDevice *models.Device) {
 	//wwn must always be lowercase.
 	detectedDevice.WWN = strings.ToLower(detectedDevice.WWN)
 }
+
+// as discussed in
+// - https://github.com/AnalogJ/scrutiny/issues/225
+// - https://github.com/jaypipes/ghw/issues/59#issue-361915216
+// udev exposes its data in a standardized way under /run/udev/data/....
+func populateUdevInfo(detectedDevice *models.Device) error {
+	// Get device major:minor numbers
+	// `cat /sys/class/block/sda/dev`
+	devNo, err := ioutil.ReadFile(filepath.Join("/sys/class/block/", detectedDevice.DeviceName, "dev"))
+	if err != nil {
+		return err
+	}
+
+	// Look up block device in udev runtime database
+	// `cat /run/udev/data/b8:0`
+	udevID := "b" + strings.TrimSpace(string(devNo))
+	udevBytes, err := ioutil.ReadFile(filepath.Join("/run/udev/data/", udevID))
+	if err != nil {
+		return err
+	}
+
+	deviceMountPaths := []string{}
+	udevInfo := make(map[string]string)
+	for _, udevLine := range strings.Split(string(udevBytes), "\n") {
+		if strings.HasPrefix(udevLine, "E:") {
+			if s := strings.SplitN(udevLine[2:], "=", 2); len(s) == 2 {
+				udevInfo[s[0]] = s[1]
+			}
+		} else if strings.HasPrefix(udevLine, "S:") {
+			deviceMountPaths = append(deviceMountPaths, udevLine[2:])
+		}
+	}
+
+	//Set additional device information.
+	if deviceLabel, exists := udevInfo["ID_FS_LABEL"]; exists {
+		detectedDevice.DeviceLabel = deviceLabel
+	}
+	if deviceUUID, exists := udevInfo["ID_FS_UUID"]; exists {
+		detectedDevice.DeviceUUID = deviceUUID
+	}
+	if deviceSerialID, exists := udevInfo["ID_SERIAL"]; exists {
+		detectedDevice.DeviceSerialID = fmt.Sprintf("%s-%s", udevInfo["ID_BUS"], deviceSerialID)
+	}
+
+
+	return nil
+}
+

collector/pkg/detect/devices_windows.go

@@ -1,8 +1,8 @@
 package detect
 
 import (
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/models"
-	"github.com/jaypipes/ghw"
 	"strings"
 )
 
@@ -11,15 +11,16 @@ func DevicePrefix() string {
 }
 
 func (d *Detect) Start() ([]models.Device, error) {
+	d.Shell = shell.Create()
 	// call the base/common functionality to get a list of devices
-	detectedDevices, err := d.smartctlScan()
+	detectedDevices, err := d.SmartctlScan()
 	if err != nil {
 		return nil, err
 	}
 
 	//inflate device info for detected devices.
 	for ndx, _ := range detectedDevices {
-		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+		d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
 	}
 
 	return detectedDevices, nil

collector/pkg/detect/testdata/smartctl_scan_megaraid.json

@@ -0,0 +1,35 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      1
+    ],
+    "svn_revision": "5022",
+    "platform_info": "x86_64-linux-5.4.0-45-generic",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "--scan"
+    ],
+    "exit_status": 0
+  },
+  "devices": [
+    {
+      "name": "/dev/bus/0",
+      "info_name": "/dev/bus/0 [megaraid_disk_00]",
+      "type": "megaraid,0",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/bus/0",
+      "info_name": "/dev/bus/0 [megaraid_disk_01]",
+      "type": "megaraid,1",
+      "protocol": "SCSI"
+    }
+  ]
+}

collector/pkg/detect/testdata/smartctl_scan_nvme.json

@@ -0,0 +1,29 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-4.19.107-Unraid",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "--scan"
+    ],
+    "exit_status": 0
+  },
+  "devices": [
+    {
+      "name": "/dev/nvme0",
+      "info_name": "/dev/nvme0",
+      "type": "nvme",
+      "protocol": "NVMe"
+    }
+  ]
+}

collector/pkg/detect/testdata/smartctl_scan_simple.json

@@ -0,0 +1,65 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-5.15.32-flatcar",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "--scan",
+      "-j"
+    ],
+    "exit_status": 0
+  },
+  "devices": [
+    {
+      "name": "/dev/sda",
+      "info_name": "/dev/sda",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdb",
+      "info_name": "/dev/sdb",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdc",
+      "info_name": "/dev/sdc",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdd",
+      "info_name": "/dev/sdd",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sde",
+      "info_name": "/dev/sde",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdf",
+      "info_name": "/dev/sdf",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdg",
+      "info_name": "/dev/sdg",
+      "type": "scsi",
+      "protocol": "SCSI"
+    }
+  ]
+}

collector/pkg/models/device.go

@@ -4,6 +4,10 @@ type Device struct {
 	WWN string `json:"wwn"`
 
 	DeviceName     string `json:"device_name"`
+	DeviceUUID	   string `json:"device_uuid"`
+	DeviceSerialID	   string `json:"device_serial_id"`
+	DeviceLabel	   string `json:"device_label"`
+
 	Manufacturer   string `json:"manufacturer"`
 	ModelName      string `json:"model_name"`
 	InterfaceType  string `json:"interface_type"`
@@ -16,6 +20,10 @@ type Device struct {
 	SmartSupport   bool   `json:"smart_support"`
 	DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
 	DeviceType     string `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string `json:"label"`
+	HostId string `json:"host_id"`
 }
 
 type DeviceWrapper struct {

collector/pkg/models/scan.go

@@ -10,10 +10,11 @@ type Scan struct {
 		Argv         []string `json:"argv"`
 		ExitStatus   int      `json:"exit_status"`
 	} `json:"smartctl"`
-	Devices []struct {
-		Name     string `json:"name"`
-		InfoName string `json:"info_name"`
-		Type     string `json:"type"`
-		Protocol string `json:"protocol"`
-	} `json:"devices"`
+	Devices []ScanDevice `json:"devices"`
+}
+type ScanDevice struct {
+	Name     string `json:"name"`
+	InfoName string `json:"info_name"`
+	Type     string `json:"type"`
+	Protocol string `json:"protocol"`
 }

collector/pkg/models/scan_override.go

@@ -0,0 +1,11 @@
+package models
+
+type ScanOverride struct {
+	Device     string   `mapstructure:"device"`
+	DeviceType []string `mapstructure:"type"`
+	Ignore     bool     `mapstructure:"ignore"`
+	Commands   struct {
+		MetricsInfoArgs  string `mapstructure:"metrics_info_args"`
+		MetricsSmartArgs string `mapstructure:"metrics_smart_args"`
+	} `mapstructure:"commands"`
+}

docker/Dockerfile

@@ -1,55 +1,50 @@
 ########
-FROM golang:1.14.4-buster as backendbuild
+FROM golang:1.17-bullseye as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
 COPY . /go/src/github.com/analogj/scrutiny
 
 RUN go mod vendor && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-selftest collector/cmd/collector-selftest/collector-selftest.go && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
+    go build -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go && \
+    go build -o scrutiny-collector-selftest collector/cmd/collector-selftest/collector-selftest.go && \
+    go build -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
 
 ########
-FROM node:lts-slim as frontendbuild
-
-#reduce logging, disable angular-cli analytics for ci environment
-ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
-
-WORKDIR /scrutiny/src
-COPY webapp/frontend /scrutiny/src
-
-RUN npm install -g @angular/cli@9.1.4 && \
-    mkdir -p /scrutiny/dist && \
-    npm install && \
-    ng build --output-path=/scrutiny/dist --deploy-url="/web/" --base-href="/web/" --prod
-
-
-########
-FROM ubuntu:bionic as runtime
+FROM debian:bullseye-slim as runtime
+ARG TARGETARCH
 EXPOSE 8080
-WORKDIR /scrutiny
-ENV PATH="/scrutiny/bin:${PATH}"
+WORKDIR /opt/scrutiny
+ENV PATH="/opt/scrutiny/bin:${PATH}"
+ENV INFLUXD_CONFIG_PATH=/opt/scrutiny/influxdb
 
-ADD https://github.com/dshearer/jobber/releases/download/v1.4.4/jobber_1.4.4-1_amd64.deb /tmp/
-RUN apt install /tmp/jobber_1.4.4-1_amd64.deb
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates curl tzdata \
+    && update-ca-certificates \
+    &&  case ${TARGETARCH} in \
+            "amd64")  S6_ARCH=amd64  ;; \
+            "arm64")  S6_ARCH=aarch64  ;; \
+        esac \
+    && curl https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-${S6_ARCH}.tar.gz -L -s --output /tmp/s6-overlay-${S6_ARCH}.tar.gz \
+    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C / \
+    && rm -rf /tmp/s6-overlay-${S6_ARCH}.tar.gz \
+    && curl -L https://dl.influxdata.com/influxdb/releases/influxdb2-2.2.0-${TARGETARCH}.deb --output /tmp/influxdb2-2.2.0-${TARGETARCH}.deb \
+    && dpkg -i --force-all /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
 
-RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1
-
-ADD https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-amd64.tar.gz /tmp/
-RUN tar xzf /tmp/s6-overlay-amd64.tar.gz -C /
 COPY /rootfs /
 
+COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /opt/scrutiny/bin/
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
+COPY dist /opt/scrutiny/web
+RUN chmod +x /opt/scrutiny/bin/scrutiny && \
+    chmod +x /opt/scrutiny/bin/scrutiny-collector-selftest && \
+    chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \
+    chmod 0644 /etc/cron.d/scrutiny && \
+    rm -f /etc/cron.daily/* && \
+    mkdir -p /opt/scrutiny/web && \
+    mkdir -p /opt/scrutiny/config && \
+    chmod -R ugo+rwx /opt/scrutiny/config
 
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /scrutiny/bin/
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /scrutiny/bin/
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /scrutiny/bin/
-COPY --from=frontendbuild /scrutiny/dist /scrutiny/web
-RUN chmod +x /scrutiny/bin/scrutiny && \
-    chmod +x /scrutiny/bin/scrutiny-collector-selftest && \
-    chmod +x /scrutiny/bin/scrutiny-collector-metrics && \
-    mkdir -p /scrutiny/web && \
-    mkdir -p /scrutiny/config && \
-    mkdir -p /scrutiny/jobber
 
 CMD ["/init"]

docker/Dockerfile.collector

@@ -1,5 +1,5 @@
 ########
-FROM golang:1.14.4-buster as backendbuild
+FROM golang:1.17-bullseye as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
@@ -10,22 +10,20 @@ RUN go mod vendor && \
     go build -ldflags '-w -extldflags "-static"' -o scrutiny-collector-metrics collector/cmd/collector-metrics/collector-metrics.go
 
 ########
-FROM ubuntu:bionic as runtime
-EXPOSE 8080
+FROM debian:bullseye-slim as runtime
 WORKDIR /scrutiny
-ENV PATH="/scrutiny/bin:${PATH}"
+ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-ADD https://github.com/dshearer/jobber/releases/download/v1.4.4/jobber_1.4.4-1_amd64.deb /tmp/
-RUN apt install /tmp/jobber_1.4.4-1_amd64.deb
+RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && update-ca-certificates
 
-RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1
+COPY /docker/entrypoint-collector.sh /entrypoint-collector.sh
+COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /opt/scrutiny/bin/
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
+RUN chmod +x /opt/scrutiny/bin/scrutiny-collector-selftest && \
+    chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \
+    chmod +x /entrypoint-collector.sh && \
+    chmod 0644 /etc/cron.d/scrutiny && \
+    rm -f /etc/cron.daily/apt /etc/cron.daily/dpkg /etc/cron.daily/passwd
 
-COPY /rootfs/scrutiny /scrutiny
-
-
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /scrutiny/bin/
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /scrutiny/bin/
-RUN chmod +x /scrutiny/bin/scrutiny-collector-selftest && \
-    chmod +x /scrutiny/bin/scrutiny-collector-metrics
-
-CMD ["/usr/lib/x86_64-linux-gnu/jobberrunner", "/scrutiny/jobber/jobber.yaml"]
+CMD ["/entrypoint-collector.sh"]

docker/Dockerfile.web

@@ -1,37 +1,25 @@
 ########
-FROM golang:1.14.4-buster as backendbuild
+FROM golang:1.17-bullseye as backendbuild
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 
 COPY . /go/src/github.com/analogj/scrutiny
 
 RUN go mod vendor && \
-    go build -ldflags '-w -extldflags "-static"' -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go
+    go build -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go
 
 ########
-FROM node:lts-slim as frontendbuild
-
-#reduce logging, disable angular-cli analytics for ci environment
-ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
-
-WORKDIR /scrutiny/src
-COPY webapp/frontend /scrutiny/src
-
-RUN npm install -g @angular/cli@9.1.4 && \
-    mkdir -p /scrutiny/dist && \
-    npm install && \
-    ng build --output-path=/scrutiny/dist --deploy-url="/web/" --base-href="/web/" --prod
-
-
-########
-FROM ubuntu:bionic as runtime
+FROM debian:bullseye-slim as runtime
 EXPOSE 8080
-WORKDIR /scrutiny
-ENV PATH="/scrutiny/bin:${PATH}"
+WORKDIR /opt/scrutiny
+ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /scrutiny/bin/
-COPY --from=frontendbuild /scrutiny/dist /scrutiny/web
-RUN chmod +x /scrutiny/bin/scrutiny && \
-    mkdir -p /scrutiny/web && \
-    mkdir -p /scrutiny/config
-CMD ["/scrutiny/bin/scrutiny", "start"]
+RUN apt-get update && apt-get install -y ca-certificates curl tzdata && update-ca-certificates
+
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
+COPY dist /opt/scrutiny/web
+RUN chmod +x /opt/scrutiny/bin/scrutiny && \
+    mkdir -p /opt/scrutiny/web && \
+    mkdir -p /opt/scrutiny/config && \
+    chmod -R ugo+rwx /opt/scrutiny/config
+CMD ["/opt/scrutiny/bin/scrutiny", "start"]

docker/Dockerfile.xgo

@@ -0,0 +1,7 @@
+FROM techknowlogick/xgo:go-1.17.x
+
+WORKDIR /go/src/github.com/analogj/scrutiny
+
+COPY . /go/src/github.com/analogj/scrutiny
+
+RUN make all

docker/README.md

@@ -0,0 +1 @@
+`rootfs`  is only used by Dockerfile and Dockerfile.collector

docker/Vagrantfile

@@ -0,0 +1,18 @@
+# This vagrant file is only used for local development & testing.
+
+Vagrant.configure("2") do |config|
+  config.vm.guest = :freebsd
+  config.vm.synced_folder ".", "/vagrant", id: "vagrant-root", disabled: true
+  config.vm.box = "freebsd/FreeBSD-11.0-CURRENT"
+  config.ssh.shell = "sh"
+  config.vm.base_mac = "080027D14C66"
+
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--memory", "1024"]
+    vb.customize ["modifyvm", :id, "--cpus", "1"]
+    vb.customize ["modifyvm", :id, "--hwvirtex", "on"]
+    vb.customize ["modifyvm", :id, "--audio", "none"]
+    vb.customize ["modifyvm", :id, "--nictype1", "virtio"]
+    vb.customize ["modifyvm", :id, "--nictype2", "virtio"]
+  end
+end

docker/entrypoint-collector.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Cron runs in its own isolated environment (usually using only /etc/environment )
+# So when the container starts up, we will do a dump of the runtime environment into a .env file that we
+# will then source into the crontab file (/etc/cron.d/scrutiny)
+(set -o posix; export -p) > /env.sh
+
+# adding ability to customize the cron schedule.
+COLLECTOR_CRON_SCHEDULE=${COLLECTOR_CRON_SCHEDULE:-"0 0 * * *"}
+
+# if the cron schedule has been overridden via env variable (eg docker-compose) we should make sure to strip quotes
+[[ "${COLLECTOR_CRON_SCHEDULE}" == \"*\" || "${COLLECTOR_CRON_SCHEDULE}" == \'*\' ]] && COLLECTOR_CRON_SCHEDULE="${COLLECTOR_CRON_SCHEDULE:1:-1}"
+
+# replace placeholder with correct value
+sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron.d/scrutiny
+
+# now that we have the env start cron in the foreground
+echo "starting cron"
+su -c "cron -f -L 15" root

docker/example.hubspoke.docker-compose.yml

@@ -0,0 +1,48 @@
+version: '2.4'
+
+services:
+  influxdb:
+    image: influxdb:2.2
+    ports:
+      - '8086:8086'
+    volumes:
+      - './influxdb:/var/lib/influxdb2'
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8086/health"]
+      interval: 5s
+      timeout: 10s
+      retries: 20
+
+
+  web:
+    image: 'ghcr.io/analogj/scrutiny:master-web'
+    ports:
+      - '8080:8080'
+    volumes:
+      - './config:/opt/scrutiny/config'
+    environment:
+      SCRUTINY_WEB_INFLUXDB_HOST: 'influxdb'
+    depends_on:
+      influxdb:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/api/health"]
+      interval: 5s
+      timeout: 10s
+      retries: 20
+      start_period: 10s
+
+  collector:
+    image: 'ghcr.io/analogj/scrutiny:master-collector'
+    cap_add:
+      - SYS_RAWIO
+    volumes:
+      - '/run/udev:/run/udev:ro'
+    environment:
+      COLLECTOR_API_ENDPOINT: 'http://web:8080'
+    depends_on:
+      web:
+        condition: service_healthy
+    devices:
+      - "/dev/sda"
+      - "/dev/sdb"

docker/example.omnibus.docker-compose.yml

@@ -0,0 +1,18 @@
+version: '3.5'
+
+services:
+  scrutiny:
+    container_name: scrutiny
+    image: ghcr.io/analogj/scrutiny:master-omnibus
+    cap_add:
+      - SYS_RAWIO
+    ports:
+      - "8080:8080" # webapp
+      - "8086:8086" # influxDB admin
+    volumes:
+      - /run/udev:/run/udev:ro
+      - ./config:/opt/scrutiny/config
+      - ./influxdb:/opt/scrutiny/influxdb
+    devices:
+      - "/dev/sda"
+      - "/dev/sdb"

docs/DOWNSAMPLING.md

@@ -0,0 +1,43 @@
+# Downsampling
+
+Scrutiny collects alot of data, that can cause the database to grow unbounded. 
+
+- Smart data
+- Smart test data
+- Temperature data
+- Disk metrics (capacity/usage)
+- etc
+
+This data must be accurate in the short term, and is useful for doing trend analysis in the long term.
+However, for trend analysis we only need aggregate data, individual data points are not as useful.
+
+Scrutiny will automatically downsample data on a schedule to ensure that the database size stays reasonable, while still
+ensuring historical data is present for comparisons.
+
+
+| Bucket Name | Retention Period | Downsampling Range | Downsampling Aggregation Window | Downsampling Cron | Comments |
+| --- | --- | --- | --- | --- | --- |
+| `metrics` | 15 days | `-2w -1w` | `1w` | main bucket, weekly on Sunday at 1:00am |
+| `metrics_weekly` | 9 weeks | `-2mo -1mo` | `1mo` | monthly on first day of the month at 1:30am
+| `metrics_monthly` | 25 months | `-2y -1y` | `1y` | yearly on the first day of the year at 2:00am
+| `metrics_yearly` | forever | - | - | - | |
+
+
+After 5 months, here's how may data points should exist in each bucket for one disk
+
+| Bucket Name | Datapoints | Comments |
+| --- | --- | --- |
+| `metrics` | 15 | 7 daily datapoints , up to 7 pending data, 1 buffer data point |
+| `metrics_weekly` | 9 | 4 aggregated weekly data points, 4 pending datapoints, 1 buffer data point |
+| `metrics_monthly` | 3 | 3 aggregated monthly data points | 
+| `metrics_yearly` | 0 | |
+
+After 5 years, here's how may data points should exist in each bucket for one disk
+
+| Bucket Name | Datapoints | Comments |
+| --- | --- | --- |
+| `metrics` | - | - |
+| `metrics_weekly` | - | 
+| `metrics_monthly` | - |
+| `metrics_yearly` | - |
+

docs/INSTALL_ANSIBLE.md

@@ -0,0 +1,17 @@
+# Ansible Install
+
+[Zorlin](https://github.com/Zorlin) has developed and now maintains [an Ansible playbook](https://github.com/Zorlin/scrutiny-playbook) which automates the steps involved in manually setting up Scrutiny.
+
+Using it is simple:
+
+* Grab a copy of the playbook
+* Follow the directions in the playbook repository
+* Run `ansible-playbook site.yml`
+* Visit http://your-machine:8080 to see your new Scrutiny installation.
+
+It will automatically pull metrics from machines once a day, at 1am.
+
+You can see it in action below.
+
+[![asciicast](https://asciinema.org/a/493531.svg)](https://asciinema.org/a/493531)
+

docs/INSTALL_HUB_SPOKE.md

@@ -0,0 +1 @@
+> See [docker/example.hubspoke.docker-compose.yml](https://github.com/AnalogJ/scrutiny/blob/master/docker/example.hubspoke.docker-compose.yml) for a docker-compose file.

docs/INSTALL_MANUAL.md

@@ -1 +1,172 @@
+# Manual Install
 
+While the easiest way to get started with [Scrutiny is using Docker](https://github.com/AnalogJ/scrutiny#docker),
+it is possible to run it manually without much work. You can even mix and match, using Docker for one component and
+a manual installation for the other. There's also [an installer](INSTALL_ANSIBLE.md) which automates this manual installation procedure.
+
+Scrutiny is made up of three components: an influxdb Database, a collector and a webapp/api. Here's how each component can be deployed manually.
+
+> Note: the `/opt/scrutiny` directory is not hardcoded, you can use any directory name/path.
+
+## InfluxDB
+
+Please follow the official InfluxDB installation guide. Note, you'll need to install v2.2.0+. 
+
+https://docs.influxdata.com/influxdb/v2.2/install/
+
+## Webapp/API
+
+### Dependencies
+
+Since the webapp is packaged as a stand alone binary, there isn't really any software you need to install other than `glibc`
+which is included by most linux OS's already.
+
+
+### Directory Structure
+
+Now let's create a directory structure to contain the Scrutiny files & binary.
+
+```
+mkdir -p /opt/scrutiny/config
+mkdir -p /opt/scrutiny/web
+mkdir -p /opt/scrutiny/bin
+```
+
+### Config file
+
+While it is possible to run the webapp/api without a config file, the defaults are designed for use in a container environment,
+and so will need to be overridden. So the first thing you'll need to do is create a config file that looks like the following:
+
+```
+# stored in /opt/scrutiny/config/scrutiny.yaml
+
+version: 1
+
+web:
+  database:
+    # The Scrutiny webapp will create a database for you, however the parent directory must exist.
+    location: /opt/scrutiny/config/scrutiny.db
+  src:
+    frontend:
+      # The path to the Scrutiny frontend files (js, css, images) must be specified.
+      # We'll populate it with files in the next section
+      path: /opt/scrutiny/web
+  
+  # if you're runnning influxdb on a different host (or using a cloud-provider) you'll need to update the host & port below. 
+  # token, org, bucket are unnecessary for a new InfluxDB installation, as Scrutiny will automatically run the InfluxDB setup, 
+  # and store the information in the config file. If you 're re-using an existing influxdb installation, you'll need to provide
+  # the `token`
+  influxdb:
+    host: localhost
+    port: 8086
+#    token: 'my-token'
+#    org: 'my-org'
+#    bucket: 'bucket'
+```
+
+> Note: for a full list of available configuration options, please check the [example.scrutiny.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.scrutiny.yaml) file.
+
+### Download Files
+
+Next, we'll download the Scrutiny API binary and frontend files from the [latest Github release](https://github.com/analogj/scrutiny/releases).
+The files you need to download are named:
+
+- **scrutiny-web-linux-amd64** - save this file to `/opt/scrutiny/bin`
+- **scrutiny-web-frontend.tar.gz** - save this file to `/opt/scrutiny/web`
+
+### Prepare Scrutiny
+
+Now that we have downloaded the required files, let's prepare the filesystem.
+
+```
+# Let's make sure the Scrutiny webapp is executable.
+chmod +x /opt/scrutiny/bin/scrutiny-web-linux-amd64
+
+# Next, lets extract the frontend files.
+# NOTE: after extraction, there **should not** be a `dist` subdirectory in `/opt/scrutiny/web` directory.
+cd /opt/scrutiny/web
+tar xvzf scrutiny-web-frontend.tar.gz --strip-components 1 -C .
+
+
+# Cleanup
+rm -rf scrutiny-web-frontend.tar.gz
+```
+
+### Start Scrutiny Webapp
+
+Finally, we start the Scrutiny webapp:
+
+```
+/opt/scrutiny/bin/scrutiny-web-linux-amd64 start --config /opt/scrutiny/config/scrutiny.yaml
+```
+
+The webapp listens for traffic on `http://0.0.0.0:8080` by default.
+
+
+## Collector
+
+### Dependencies
+
+Unlike the webapp, the collector does have some dependencies:
+
+- `smartctl`, v7+
+- `cron` (or an alternative process scheduler)
+
+Unfortunately the version of `smartmontools` (which contains `smartctl`) available in some of the base OS repositories is ancient.
+So you'll need to install the v7+ version using one of the following commands:
+
+- **Ubuntu (22.04/Jammy/LTS):** `apt-get install -y smartmontools`
+- **Ubuntu (18.04/Bionic):** `apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1`
+- **Centos8:**
+    - `dnf install https://extras.getpagespeed.com/release-el8-latest.rpm`
+    - `dnf install smartmontools`
+- **FreeBSD:** `pkg install smartmontools`
+
+### Directory Structure
+
+Now let's create a directory structure to contain the Scrutiny collector binary.
+
+```
+mkdir -p /opt/scrutiny/bin
+```
+
+
+### Download Files
+
+Next, we'll download the Scrutiny collector binary from the [latest Github release](https://github.com/analogj/scrutiny/releases).
+The file you need to download is named:
+
+- **scrutiny-collector-metrics-linux-amd64** - save this file to `/opt/scrutiny/bin`
+
+
+### Prepare Scrutiny
+
+Now that we have downloaded the required files, let's prepare the filesystem.
+
+```
+# Let's make sure the Scrutiny collector is executable.
+chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics-linux-amd64
+```
+
+### Start Scrutiny Collector, Populate Webapp
+
+Next, we will manually trigger the collector, to populate the Scrutiny dashboard:
+
+> NOTE: if you need to pass a config file to the scrutiny collector, you can provide it using the `--config` flag.
+
+```
+/opt/scrutiny/bin/scrutiny-collector-metrics-linux-amd64 run --api-endpoint "http://localhost:8080"
+```
+
+### Schedule Collector with Cron
+
+Finally you need to schedule the collector to run periodically.
+This may be different depending on your OS/environment, but it may look something like this:
+
+```
+# open crontab
+crontab -e
+
+# add a line for Scrutiny
+*/15 * * * * . /etc/profile; /opt/scrutiny/bin/scrutiny-collector-metrics-linux-amd64 run --api-endpoint "http://localhost:8080"
+```

docs/INSTALL_PFSENSE.md

@@ -0,0 +1,72 @@
+# pfsense Install
+
+This bascially follows the [Manual collector instructions](https://github.com/AnalogJ/scrutiny/blob/master/docs/INSTALL_MANUAL.md#collector) and assumes you are running a hub and spoke deployment and already have the web app setup.
+
+
+### Dependencies
+
+SSH into pfsense, hit `8` for the shell and install the required dependencies.
+
+```
+pkg install smartmontools
+```
+
+Ensure smartmontools is v7+. This won't be a problem in pfsense 2.6.0+
+
+
+### Directory Structure
+
+Now let's create a directory structure to contain the Scrutiny collector binary.
+
+```
+mkdir -p /opt/scrutiny/bin
+```
+
+
+### Download Files
+
+Next, we'll download the Scrutiny collector binary from the [latest Github release](https://github.com/analogj/scrutiny/releases).
+
+> NOTE: Ensure you have the latest version in the below command
+
+```
+fetch -o /opt/scrutiny/bin https://github.com/AnalogJ/scrutiny/releases/download/vX.X.X/scrutiny-collector-metrics-freebsd-amd64
+```
+
+
+### Prepare Scrutiny
+
+Now that we have downloaded the required files, let's prepare the filesystem.
+
+```
+chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64
+```
+
+
+### Start Scrutiny Collector, Populate Webapp
+
+Next, we will manually trigger the collector, to populate the Scrutiny dashboard:
+
+> NOTE: if you need to pass a config file to the scrutiny collector, you can provide it using the `--config` flag.
+
+```
+/opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64 run --api-endpoint "http://localhost:8080"
+```
+> NOTE: change the IP address to that of your web app
+
+### Schedule Collector with Cron
+
+Finally you need to schedule the collector to run periodically.
+
+Login to the pfsense webGUI and head to `Services/Cron` add an entry with the following details:
+
+```
+Minute: */15
+Hour: *
+Day of the Month: *
+Month of the Year: *
+Day of the Week: *
+User: root
+Command: /opt/scrutiny/bin/scrutiny-collector-metrics-freebsd-amd64 run --api-endpoint "http://localhost:8080" >/dev/null 2>&1
+```
+> NOTE: `>/dev/null 2>&1` is used to stop cron confirmation emails being sent.

docs/INSTALL_UNRAID.md

@@ -0,0 +1,32 @@
+# UnRAID Install
+
+Installation of Scrutiny in UnRAID follows the same process as installing any other docker container, utilizing the Community Applications plugin
+
+## Install the 'Community Applications' Plugin
+
+All docker containers in UnRAID are typically installed utilizing the Community Applications plugin. To get started:
+- Navigate to the plugins tab ( <UnRaid_IP_Address>/Plugins )
+- Select the 'Install Plugin' tab, and enter the following address into the input field
+```
+https://raw.githubusercontent.com/Squidly271/community.applications/master/plugins/community.applications.plg
+```
+
+You're all set with the pre-requisites!
+
+## Installing the Scrutiny docker image
+
+To install, simply click 'Install'; the configuration parameters should not need modification as the template within CA already defines the necessary parameters.
+
+As a docker image can be created using various OS bases, the image choice is entirely the users choice. Recommendations of a specific image from a specific maintainer is beyond the scope of this guide. However, to provide some context given the number of questions posed regarding the various versions available:
+
+- **ghcr.io/analogj/scrutiny:master-omnibus**
+    - `Image maintained directly by the application author`
+    - `Debian based docker image`
+- **linuxserver/scrutiny**
+    - `Image maintained by the LinuxServer.io group`
+    - `Alpine based docker image`
+- **hotio/scrutiny**
+    - `Image maintained by hotio`
+    - `DETAILS TBD`
+
+The support for a given image is provided by that images maintainers, while support for the application itself remains with the developer - i.e. LinuxServer.io supports the docker image of Scrutiny which they create, to the extent an issue is specific to that image. If an issue/enhancement pertains directly to the source code, support would still come directly from this repository's contributors. 

docs/SUPPORTED_NAS_OS.md

@@ -0,0 +1,16 @@
+# Officially Supported NAS OS's
+
+These are the officially supported NAS OS's (with documentation and setup guides).
+Once a guide is created (in `docs/guides/`) it will be linked here.
+
+- [ ] freenas/truenas
+- [x] [unraid](./INSTALL_UNRAID.md)
+- [ ] ESXI
+- [ ] Proxmox
+- [ ] Synology
+- [ ] OMV
+- [ ] Amahi
+- [ ] Running in a LXC container
+- [x] [PFSense](./INSTALL_UNRAID.md)
+- [ ] QNAP
+

docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md

@@ -0,0 +1,204 @@
+# Scrutiny <-> SmartMonTools 
+
+Scrutiny uses `smartctl --scan` to detect devices/drives. If your devices are not being detected by Scrutiny, or some 
+data is missing, this is probably due to a `smartctl` issue. 
+The following page will document commonly asked questions and troubleshooting steps for the Scrutiny S.M.A.R.T. data collector.
+
+## WWN vs Device name
+As discussed in [`#117`](https://github.com/AnalogJ/scrutiny/issues/117), `/dev/sd*` device paths are ephemeral. 
+
+> Device paths in Linux aren't guaranteed to be consistent across restarts. Device names consist of major numbers (letters) and minor numbers. When the Linux storage device driver detects a new device, the driver assigns major and minor numbers from the available range to the device. When a device is removed, the device numbers are freed for reuse.
+>
+> The problem occurs because device scanning in Linux is scheduled by the SCSI subsystem to happen asynchronously. As a result, a device path name can vary across restarts.
+>
+> https://docs.microsoft.com/en-us/troubleshoot/azure/virtual-machines/troubleshoot-device-names-problems
+
+While the Docker Scrutiny collector does require devices to attached to the docker container by device name (using `--device=/dev/sd..`), internally 
+Scrutiny stores and references the devices by their `WWN` which is globally unique, and never changes. 
+
+As such, passing devices to the Scrutiny collector container using `/dev/disk/by-id/`, `/dev/disk/by-label/`, `/dev/disk/by-path/` and `/dev/disk/by-uuid/`
+paths are unnecessary, unless you'd like to ensure the docker run command never needs to change.
+
+
+## Device Detection By Smartctl
+
+The first thing you'll want to do is run `smartctl` locally (not in Docker) and make sure the output shows all your drives as expected.
+See the `Drive Types` section below for what this output should look like for `NVMe`/`ATA`/`RAID` drives.
+
+```bash
+smartctl --scan
+
+/dev/sda -d scsi # /dev/sda, SCSI device
+/dev/sdb -d scsi # /dev/sdb, SCSI device
+/dev/sdc -d scsi # /dev/sdc, SCSI device
+/dev/sdd -d scsi # /dev/sdd, SCSI device
+```
+
+Once you've verified that `smartctl` correctly detects your drives, make sure scrutiny is correctly detecting them as well.
+> NOTE: make sure you specify all the devices you'd like scrutiny to process using `--device=` flags.
+
+```bash
+docker run -it --rm \
+  -v /run/udev:/run/udev:ro \
+  --cap-add SYS_RAWIO \
+  --device=/dev/sda \
+  --device=/dev/sdb \
+  ghcr.io/analogj/scrutiny:master-collector smartctl --scan
+```
+
+If the output is the same, your devices will be processed by Scrutiny.
+
+### Collector Config File
+In some cases `--scan` does not correctly detect the device type, returning [incomplete SMART data](https://github.com/AnalogJ/scrutiny/issues/45).
+Scrutiny will supports overriding the detected device type via the config file.
+
+[example.collector.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.collector.yaml)
+
+### RAID Controllers (Megaraid/3ware/HBA/Adaptec/HPE/etc)
+Smartctl has support for a large number of [RAID controllers](https://www.smartmontools.org/wiki/Supported_RAID-Controllers), however this 
+support is not automatic, and may require some additional device type hinting. You can provide this information to the Scrutiny collector
+using a collector config file. See [example.collector.yaml](/example.collector.yaml)
+
+> NOTE: If you use docker, you **must** pass though the RAID virtual disk to the container using `--device` (see below)
+>
+> This device may be in `/dev/*` or `/dev/bus/*`. 
+>
+> If you're unsure, run `smartctl --scan` on your host, and pass all listed devices to the container.
+
+```yaml
+# /opt/scrutiny/config/collector.yaml
+devices:
+  # Dell PERC/Broadcom Megaraid example: https://github.com/AnalogJ/scrutiny/issues/30
+  - device: /dev/bus/0
+    type:
+      - megaraid,14
+      - megaraid,15
+      - megaraid,18
+      - megaraid,19
+      - megaraid,20
+      - megaraid,21
+
+  - device: /dev/twa0
+    type:
+      - 3ware,0
+      - 3ware,1
+      - 3ware,2
+      - 3ware,3
+      - 3ware,4
+      - 3ware,5
+  
+  # Adapec RAID: https://github.com/AnalogJ/scrutiny/issues/189
+  - device: /dev/sdb
+    type:
+      - aacraid,0,0,0
+      - aacraid,0,0,1
+  
+  # HPE Smart Array example:  https://github.com/AnalogJ/scrutiny/issues/213
+  - device: /dev/sda
+    type:
+      - 'cciss,0'
+      - 'cciss,1'
+```
+
+### NVMe Drives
+As mentioned in the [README.md](/README.md), NVMe devices require both `--cap-add SYS_RAWIO` and `--cap-add SYS_ADMIN` 
+to allow smartctl permission to query your NVMe device SMART data [#26](https://github.com/AnalogJ/scrutiny/issues/26)
+
+When attaching NVMe devices using `--device=/dev/nvme..`, make sure to provide the device controller (`/dev/nvme0`) 
+instead of the block device (`/dev/nvme0n1`). See [#209](https://github.com/AnalogJ/scrutiny/issues/209).
+
+> The character device /dev/nvme0 is the NVME device controller, and block devices like /dev/nvme0n1 are the NVME storage namespaces: the devices you use for actual storage, which will behave essentially as disks.
+>
+> In enterprise-grade hardware, there might be support for several namespaces, thin provisioning within namespaces and other features. For now, you could think namespaces as sort of meta-partitions with extra features for enterprise use.
+
+### ATA
+
+### Exit Codes
+
+If you see an error message similar to `smartctl returned an error code (2) while processing /dev/sda`, this means that
+`smartctl` (not Scrutiny) exited with an error code. Scrutiny will attempt to print a helpful error message to help you debug,
+but you can look at the table (and associated links) below to debug `smartctl`. 
+
+> smartctl Return Values
+> The return values of smartctl are defined by a bitmask. If all is well with the disk, the return value (exit status) of 
+> smartctl is 0 (all bits turned off). If a problem occurs, or an error, potential error, or fault is detected, then 
+> a non-zero status is returned. In this case, the eight different bits in the return value have the following meanings 
+> for ATA disks; some of these values may also be returned for SCSI disks.
+> 
+> source: http://www.linuxguide.it/command_line/linux-manpage/do.php?file=smartctl#sect7
+
+
+| Exit Code (Isolated) | Binary | Problem Message |
+| --- | --- | --- |
+| 1 | Bit 0 | Command line did not parse. |
+| 2 | Bit 1 | Device open failed, or device did not return an IDENTIFY DEVICE structure. |
+| 4 | Bit 2 | Some SMART command to the disk failed, or there was a checksum error in a SMART data structure (see В´-bВ´ option above). |
+| 8 | Bit 3 | SMART status check returned “DISK FAILING". |
+| 16 | Bit 4 | We found prefail Attributes <= threshold. |
+| 32 | Bit 5 | SMART status check returned “DISK OK” but we found that some (usage or prefail) Attributes have been <= threshold at some time in the past. |
+| 64 | Bit 6 | The device error log contains records of errors. |
+| 128 | Bit 7 | The device self-test log contains records of errors. |
+
+#### Standby/Sleeping Disks
+
+Disks in Standby/Sleep can also cause `smartctl` to exit abnormally, usually with `exit code: 2`. 
+
+- https://github.com/AnalogJ/scrutiny/issues/221
+- https://github.com/AnalogJ/scrutiny/issues/157
+
+### Volume Mount All Devices (`/dev`) - Privileged
+
+> WARNING: This is an insecure/dangerous workaround. Running Scrutiny (or any Docker image) with `--privileged` is equivalent to running it with root access. 
+
+If you have exhausted all other mechanisms to get your disks working with `smartctl` running within a container, you can try running the docker image with the following additional flags:
+
+- `--privileged` (instead of `--cap-add`) - this gives the docker container full access to your system. Scrutiny does not require this permission, however it can be helpful for `smartctl`
+- `-v /dev:/dev:ro` (instead of `--device`) - this mounts the `/dev` folder (containing all your device files) into the container, allowing `smartctl` to see your disks, exactly as if it were running on your host directly. 
+
+With this workaround your `docker run` command would look similar to the following:
+
+```bash
+docker run -it --rm -p 8080:8080 -p 8086:8086 \
+  -v `pwd`/scrutiny:/opt/scrutiny/config \
+  -v `pwd`/influxdb2:/opt/scrutiny/influxdb \
+  -v /run/udev:/run/udev:ro \
+  --privileged \
+  -v /dev:/dev \
+  --name scrutiny \
+  ghcr.io/analogj/scrutiny:master-omnibus
+```
+
+## Scrutiny detects Failure but SMART Passed?
+
+There's 2 different mechanisms that Scrutiny uses to detect failures.
+
+The first is simple SMART failures. If SMART thinks an attribute is in a failed state, Scrutiny will display it as failed as well.
+
+The second is using BackBlaze failure data: [https://backblaze.com/blog-smart-stats-2014-8.html](https://backblaze.com/blog-smart-stats-2014-8.html) 
+If Scrutiny detects that an attribute corresponds with a high rate of failure using BackBlaze's data, it will also mark that attribute (and disk) as failed (even though SMART may think the device is still healthy).
+
+This can cause some confusion when comparing Scrutiny's dashboard against other SMART analysis tools. 
+If you hover over the "failed" label beside an attribute, Scrutiny will tell you if the failure was due to SMART or Scrutiny/BackBlaze data. 
+
+
+## Hub & Spoke model, with multiple Hosts.
+
+When deploying Scrutiny in a hub & spoke model, it can be difficult to determine exactly which node a set of devices are associated with.
+Thankfully the collector has a special `--host-id` flag (or `COLLECTOR_HOST_ID` env variable) that can be used to associate devices with a friendly host name.
+
+See the [docs/INSTALL_HUB_SPOKE.md](/docs/INSTALL_HUB_SPOKE.md) guide for more information. 
+
+## Collector DEBUG mode
+
+You can use environmental variables to enable debug logging and/or log files for the collector:
+
+```bash
+DEBUG=true
+COLLECTOR_LOG_FILE=/tmp/collector.log
+```
+
+Or if you're not using docker, you can pass CLI arguments to the collector during startup:
+
+```bash
+scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
+```

docs/TROUBLESHOOTING_DOCKER.md

@@ -0,0 +1,25 @@
+# Docker Images `master-omnibus` vs `latest`
+
+> TL;DR; The `master-omnibus` and `latest` tags are almost semantically identical, as I follow a `golden master` 
+development process. However if you want to ensure you're only using the latest release, you can change to `latest`
+
+The CI script used to orchestrate the docker image builds can be found here: https://github.com/AnalogJ/scrutiny/blob/master/.github/workflows/docker-build.yaml#L166-L184
+
+In general Scrutiny follows a `golden master` development process, which means that the `master` branch is not directly updated (unless its for documentation changes), 
+instead development is done in a feature branch, or committed to the `beta` branch. 
+
+As development progresses, and we're satisfied that a feature is complete, and the quality is acceptable, 
+I merge the changes to `master` and trigger the creation of a new release -- ie, when master is updated, a new release
+is almost immediately created (and tagged with `latest`)
+
+So changing from `master-omnibus -> latest` will be the same thing for all intents and purposes. 
+
+Having said that -- the one key difference is the `automated cron builds` that run on the `master` and `beta` branches. 
+They trigger a `nightly` build, even if nothing has changed on the branch. This has a couple of benefits, but one is to 
+ensure that there's no broken external dependencies in our (unchanged) code. 
+
+However, as everyone unfortunately found out recently, I had an error in my CI script, which caused failures to be 
+ignored -- https://github.com/AnalogJ/scrutiny/issues/287. That has since been fixed.
+
+Hope that gives you an understanding for how everything is wired up. 
+

docs/TROUBLESHOOTING_INFLUXDB.md

@@ -0,0 +1,68 @@
+# InfluxDB Troubleshooting
+
+## Installation 
+InfluxDB is a required dependency for Scrutiny v0.4.0+. 
+
+https://docs.influxdata.com/influxdb/v2.2/install/
+
+## Persistence
+
+To ensure that all data is correctly stored, you must also persist the InfluxDB database directory
+
+- If you're using the Official Scrutiny Omnibus image (`ghcr.io/analogj/scrutiny:master-omnibus`), the path is `/opt/scrutiny/influxdb`
+- If you're deploying in Hub/Spoke mode with the InfluxDB maintained image (`influxdb:2.2`), the path is `/var/lib/influxdb2`
+
+If you attempt to restart Scrutiny but you forgot to persist the InfluxDB directory, you will get an error message like follows:
+
+```
+scrutiny    | time="2022-05-12T22:54:12Z" level=info msg="Trying to connect to scrutiny sqlite db: /opt/scrutiny/config/scrutiny.db\n"
+scrutiny    | time="2022-05-12T22:54:12Z" level=info msg="Successfully connected to scrutiny sqlite db: /opt/scrutiny/config/scrutiny.db\n"
+scrutiny    | ts=2022-05-12T22:54:12.240791Z lvl=info msg=Unauthorized log_id=0aQcVlOW000 error="authorization not found"
+scrutiny    | panic: unauthorized: unauthorized access
+```
+
+Unfortunately this may mean that your database is lost, and the previous Scrutiny data is unavailable. 
+You should fix the docker-compose/docker run command that you're using to ensure that your database folder is persisted correctly, 
+then delete the `web.influxdb.token` field in your `scrutiny.yaml` file, and then restart Scrutiny.
+
+
+## First Start
+The web/api service will trigger an InfluxDB onboarding process automatically when it first starts. After that, it will store the newly generated influxdb api token in the Scrutiny config file. 
+
+If this Credential is not correctly stored in the scrutiny config file, Scrutiny will fail to start (with an authentication error)
+
+```
+scrutiny    | time="2022-05-12T22:52:55Z" level=info msg="Successfully connected to scrutiny sqlite db: /opt/scrutiny/config/scrutiny.db\n"
+scrutiny    | ts=2022-05-12T22:52:55.235753Z lvl=error msg="failed to onboard user admin" log_id=0aQcRnc0000 handler=onboard error="onboarding has already been completed" took=0.038ms
+scrutiny    | ts=2022-05-12T22:52:55.235816Z lvl=error msg="api error encountered" log_id=0aQcRnc0000 error="onboarding has already been completed"
+scrutiny    | panic: conflict: onboarding has already been completed
+```
+
+You can fix this issue by authenticating to the InfluxDB admin portal (the default credentials are username: `admin`, password: `password12345`),
+then retrieving the API token, and writing it to your `scrutiny.yaml` config file under the `web.influxdb.token` field:
+
+![influx db admin token](./influxdb-admin-token.png)
+
+## Upgrading from v0.3.x to v0.4.x
+
+When upgrading from v0.3.x to v0.4.x, some users have noticed problems such as:
+
+```
+2022/05/13 14:38:05 Loading configuration file: /opt/scrutiny/config/scrutiny.yaml
+time="2022-05-13T14:38:05Z" level=info msg="Trying to connect to scrutiny sqlite db:"
+time="2022-05-13T14:38:05Z" level=info msg="Successfully connected to scrutiny sqlite db:"
+panic: a username and password is required for a setup
+```
+
+As discussed in [#248](https://github.com/AnalogJ/scrutiny/issues/248) and [#234](https://github.com/AnalogJ/scrutiny/issues/234),
+this usually related to either:
+
+- Upgrading from the LSIO Scrutiny image to the Official Scrutiny image, without removing LSIO specific environmental variables
+  - remove the `SCRUTINY_WEB=true` and `SCRUTINY_COLLECTOR=true` environmental variables. They were used by the LSIO image, but are unnecessary and cause issues with the official Scrutiny image.
+- Updated versions of the [LSIO Scrutiny images are broken](https://github.com/linuxserver/docker-scrutiny/issues/22), as they have not installed InfluxDB which is a required dependency of Scrutiny v0.4.x
+  - You can revert to an earlier version of the LSIO image (`lscr.io/linuxserver/scrutiny:060ac7b8-ls34`), or just change to the official Scrutiny image (`ghcr.io/analogj/scrutiny:master-omnibus`)
+
+Here's a couple of confirmed working docker-compose files that you may want to look at:
+
+- https://github.com/AnalogJ/scrutiny/blob/master/docker/example.hubspoke.docker-compose.yml
+- https://github.com/AnalogJ/scrutiny/blob/master/docker/example.omnibus.docker-compose.yml

docs/TROUBLESHOOTING_NOTIFICATIONS.md

@@ -0,0 +1,25 @@
+# Notifications
+
+As documented in [example.scrutiny.yaml](https://github.com/AnalogJ/scrutiny/blob/master/example.scrutiny.yaml#L59-L75)
+there are multiple ways to configure notifications for Scrutiny.
+
+Under the hood we use a library called [Shoutrrr](https://github.com/containrrr/shoutrrr) to send our notifications, and you should use their documentation if you run into
+any issues: https://containrrr.dev/shoutrrr/services/overview/
+
+
+# Script Notifications
+
+While the Shoutrrr library supports many popular providers for sending notifications Scrutiny also supports a "script" based
+notification system, allowing you to execute a custom script whenever a notification needs to be sent. 
+Data is provided to this script using the following environmental variables:
+
+```
+SCRUTINY_SUBJECT - 	eg. "Scrutiny SMART error (%s) detected on device: %s"
+SCRUTINY_DATE 
+SCRUTINY_FAILURE_TYPE - EmailTest, SmartFail, ScrutinyFail
+SCRUTINY_DEVICE_NAME - eg. /dev/sda
+SCRUTINY_DEVICE_TYPE - ATA/SCSI/NVMe
+SCRUTINY_DEVICE_SERIAL - eg. WDDJ324KSO
+SCRUTINY_MESSAGE - eg. "Scrutiny SMART error notification for device: %s\nFailure Type: %s\nDevice Name: %s\nDevice Serial: %s\nDevice Type: %s\nDate: %s"
+```
+

docs/TROUBLESHOOTING_REVERSE_PROXY.md

@@ -0,0 +1,106 @@
+# Reverse Proxy Support
+
+Scrutiny is designed so that it can be used with a reverse proxy, leveraging `domain`, `port` or `path` based matching to correctly route to the Scrutiny service.
+
+For simple `domain` and/or `port` based routing, this is easy.
+
+If your domain:port pair is similar to `http://scrutiny.example.com` or `http://localhost:54321`, just update your reverse proxy configuration
+to route traffic to the Scrutiny backend, which is listening on `0.0.0.0:8080` by default.
+
+```yaml
+# default config
+web:
+  listen:
+    port: 8080
+    host: 0.0.0.0
+```
+
+However if you're using `path` based routing to differentiate your reverse proxy protected services, things become more complicated.
+
+If you'd like to access Scrutiny using a path like: `http://example.com/scrutiny/`, then we need a way to configure Scrutiny so that it
+understands `http://example.com/scrutiny/api/health` actually means `http://localhost:8080/api/health`.
+
+Thankfully this can be done by changing **two** settings (both are required).
+
+1. The webserver has a `web.listen.basepath` key
+2. The collectors have a `api.endpoint` key.
+
+## Webserver Configuration
+
+When setting the `web.listen.basepath` key in the web config file, make sure the `basepath` key is prefixed with `/`.
+
+```yaml
+# customized webserver config
+web:
+  listen:
+    port: 8080
+    host: 0.0.0.0
+    # if you're using a reverse proxy like apache/nginx, you can override this value to serve scrutiny on a subpath.
+    # eg. http://example.com/custombasepath/* vs http://example.com:8080
+    basepath: '/custombasepath'
+```
+
+## Collector Configuration 
+
+Here's how you can update the collector `api.endpoint` key:
+
+```yaml
+# customized collector config
+api:
+  endpoint: 'http://localhost:8080/custombasepath'
+```
+
+# Environmental Variables.
+
+You may also configure these values using the following environmental variables (both are required).
+
+- `COLLECTOR_API_ENDPOINT=http://localhost:8080/custombasepath`
+- `SCRUTINY_WEB_LISTEN_BASEPATH=/custombasepath`
+
+# Real Examples
+
+## Caddy
+
+1. Create a Caddyfile
+    ```yaml
+    # Caddyfile
+    :9090
+    
+    # The `scrutiny` text in this file must match the service name in the docker-compose file below. 
+    # The `/custom/` text is the custom base path scrutiny will be availble on. 
+    reverse_proxy /custom/* scrutiny:8080
+
+    ```
+2. Create a `docker-compose.yml` file
+
+    ```yaml
+    # docker-compose.yml
+    version: '3.5'
+    
+    services:
+      scrutiny:
+        container_name: scrutiny
+        image: ghcr.io/analogj/scrutiny:master-omnibus
+        cap_add:
+          - SYS_RAWIO
+        ports:
+          - "8086:8086" # influxDB admin
+        volumes:
+          - /run/udev:/run/udev:ro
+          - ./config:/opt/scrutiny/config
+          - ./influxdb:/opt/scrutiny/influxdb
+        devices:
+          - "/dev/sda"
+          - "/dev/sdb"
+        environment:
+          - SCRUTINY_WEB_LISTEN_BASEPATH=/custom
+          - COLLECTOR_API_ENDPOINT=http://localhost:8080/custom
+      caddy:
+        image: caddy
+        volumes:
+          - ./Caddyfile:/etc/caddy/Caddyfile
+        ports:
+          - "9090:9090"
+    ```
+3. run `docker-compose up`
+4. visit [http://localhost:9090/custom/web](http://localhost:9090/custom/web) - access the scrutiny container via caddy reverse proxy

docs/dbdiagram.io.txt

@@ -0,0 +1,62 @@
+
+// SQLite Table(s)
+Table device {
+  created_at timestamp
+
+  wwn varchar [pk]
+
+  //user provided
+  label varchar
+  host_id varchar
+
+  // smartctl provided
+  device_name varchar
+  manufacturer varchar
+  model_name varchar
+  interface_type varchar
+  interface_speed varchar
+  serial_number varchar
+  firmware varchar
+  rotational_speed varchar
+  capacity varchar
+  form_factor varchar
+  smart_support varchar
+  device_protocol varchar
+  device_type varchar
+
+}
+
+
+// InfluxDB Tables
+Table device_temperature {
+  //timestamp
+  created_at timestamp
+
+  //tags (indexed & queryable)
+  device_wwn varchar [pk]
+
+  //fields
+  temp bigint
+ }
+
+
+Table smart_ata_results {
+  //timestamp
+  created_at timestamp
+
+  //tags (indexed & queryable)
+  device_wwn varchar [pk]
+  smart_status varchar
+  scrutiny_status varchar
+
+
+
+  //fields
+  temp bigint
+  power_on_hours bigint
+  power_cycle_count bigint
+
+
+ }
+
+Ref: device.wwn < smart_ata_results.device_wwn

example.collector.yaml

@@ -0,0 +1,97 @@
+# Commented Scrutiny Configuration File
+#
+# The default location for this file is /opt/scrutiny/config/collector.yaml.
+# In some cases to improve clarity default values are specified,
+# uncommented. Other example values are commented out.
+#
+# When this file is parsed by Scrutiny, all configuration file keys are
+# lowercased automatically. As such, Configuration keys are case-insensitive,
+# and should be lowercase in this file to be consistent with usage.
+
+
+######################################################################
+# Version
+#
+# version specifies the version of this configuration file schema, not
+# the scrutiny binary. There is only 1 version available at the moment
+version: 1
+
+# The host id is a label used for identifying groups of disks running on the same host
+# Primiarly used for hub/spoke deployments (can be left empty if using all-in-one image).
+host:
+  id: ""
+
+
+# This block allows you to override/customize the settings for devices detected by
+# Scrutiny via `smartctl --scan`
+# See the "--device=TYPE" section of https://linux.die.net/man/8/smartctl
+# type can be a 'string' or a 'list'
+devices:
+#  # example for forcing device type detection for a single disk
+#  - device: /dev/sda
+#    type: 'sat'
+#
+#  # example to show how to ignore a specific disk/device.
+#  - device: /dev/sda
+#    ignore: true
+#
+#  # examples showing how to force smartctl to detect disks inside a raid array/virtual disk
+#  - device: /dev/bus/0
+#    type:
+#      - megaraid,14
+#      - megaraid,15
+#      - megaraid,18
+#      - megaraid,19
+#      - megaraid,20
+#      - megaraid,21
+#
+#  - device: /dev/twa0
+#    type:
+#      - 3ware,0
+#      - 3ware,1
+#      - 3ware,2
+#      - 3ware,3
+#      - 3ware,4
+#      - 3ware,5
+#
+#  # example to show how to override the smartctl command args (per device), see below for how to override these globally.
+#  - device: /dev/sda
+#    commands:
+#      metrics_info_args: '--info --json -T permissive' # used to determine device unique ID & register device with Scrutiny
+#      metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device.
+
+
+#log:
+#  file: '' #absolute or relative paths allowed, eg. web.log
+#  level: INFO
+#
+#api:
+#  endpoint: 'http://localhost:8080'
+#  endpoint: 'http://localhost:8080/custombasepath'
+# if you need to use a custom base path (for a reverse proxy), you can add a suffix to the endpoint.
+#  See docs/TROUBLESHOOTING_REVERSE_PROXY.md for more info,
+
+# example to show how to override the smartctl command args globally
+#commands:
+#  metrics_scan_args: '--scan --json' # used to detect devices
+#  metrics_info_args: '--info --json' # used to determine device unique ID & register device with Scrutiny
+#  metrics_smart_args: '--xall --json' # used to retrieve smart data for each device.
+
+
+########################################################################################################################
+# FEATURES COMING SOON
+#
+# The following commented out sections are a preview of additional configuration options that will be available soon.
+#
+########################################################################################################################
+
+#collect:
+#  metric:
+#    enable: true
+#    command: '-a -o on -S on'
+#  long:
+#    enable: false
+#    command: ''
+#  short:
+#    enable: false
+#    command: ''

example.scrutiny.yaml

@@ -1,6 +1,6 @@
 # Commented Scrutiny Configuration File
 #
-# The default location for this file is ~/scrutiny.yaml.
+# The default location for this file is /opt/scrutiny/config/scrutiny.yaml.
 # In some cases to improve clarity default values are specified,
 # uncommented. Other example values are commented out.
 #
@@ -20,31 +20,47 @@ web:
   listen:
     port: 8080
     host: 0.0.0.0
+
+    # if you're using a reverse proxy like apache/nginx, you can override this value to serve scrutiny on a subpath.
+    # eg. http://example.com/scrutiny/* vs http://example.com:8080
+    # see docs/TROUBLESHOOTING_REVERSE_PROXY.md
+    # basepath: `/scrutiny`
+    # leave empty unless behind a path prefixed proxy
+    basepath: ''
   database:
     # can also set absolute path here
-    location: /scrutiny/config/scrutiny.db
+    location: /opt/scrutiny/config/scrutiny.db
   src:
+    # the location on the filesystem where scrutiny javascript + css is located
     frontend:
-      path: /scrutiny/web
+      path: /opt/scrutiny/web
 
+  # if you're running influxdb on a different host (or using a cloud-provider) you'll need to update the host & port below.
+  # token, org, bucket are unnecessary for a new InfluxDB installation, as Scrutiny will automatically run the InfluxDB setup,
+  # and store the information in the config file. If you 're re-using an existing influxdb installation, you'll need to provide
+  # the `token`
+  influxdb:
+#    scheme: 'http'
+    host: 0.0.0.0
+    port: 8086
+#    token: 'my-token'
+#    org: 'my-org'
+#    bucket: 'bucket'
+    retention_policy: true
 
 log:
   file: '' #absolute or relative paths allowed, eg. web.log
   level: INFO
 
-# The following commented out sections are a preview of additional configuration options that will be available soon.
 
-#disks:
-#  include:
-#    # - /dev/sda
-#  exclude:
-#    # - /dev/sdb
+# Notification "urls" look like the following. For more information about service specific configuration see
+# Shoutrrr's documentation: https://containrrr.dev/shoutrrr/services/overview/
 
 #notify:
 #  urls:
 #    - "discord://token@channel"
 #    - "telegram://token@telegram?channels=channel-1[,channel-2,...]"
-#    - "pushover://shoutrrr:apiToken@userKey/?devices=device1[,device2, ...]"
+#    - "pushover://shoutrrr:apiToken@userKey/?priority=1&devices=device1[,device2, ...]"
 #    - "slack://[botname@]token-a/token-b/token-c"
 #    - "smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]"
 #    - "teams://token-a/token-b/token-c"
@@ -58,6 +74,13 @@ log:
 #    - "script:///file/path/on/disk"
 #    - "https://www.example.com/path"
 
+########################################################################################################################
+# FEATURES COMING SOON
+#
+# The following commented out sections are a preview of additional configuration options that will be available soon.
+#
+########################################################################################################################
+
 #limits:
 #  ata:
 #    critical:
@@ -72,14 +95,3 @@ log:
 #    critical: true
 #    standard: true
 
-
-#collect:
-#  metric:
-#    enable: true
-#    command: '-a -o on -S on'
-#  long:
-#    enable: false
-#    command: ''
-#  short:
-#    enable: false
-#    command: ''

go.mod

@@ -1,22 +1,83 @@
 module github.com/analogj/scrutiny
 
-go 1.13
+go 1.17
 
 require (
-	github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae // indirect
 	github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
-	github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a
-	github.com/denisbrodbeck/machineid v1.0.1
-	github.com/fatih/color v1.9.0
+	github.com/containrrr/shoutrrr v0.4.4
+	github.com/fatih/color v1.10.0
 	github.com/gin-gonic/gin v1.6.3
+	github.com/go-gormigrate/gormigrate/v2 v2.0.0
 	github.com/golang/mock v1.4.3
+	github.com/influxdata/influxdb-client-go/v2 v2.9.0
 	github.com/jaypipes/ghw v0.6.1
-	github.com/jinzhu/gorm v1.9.14
-	github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect
-	github.com/sirupsen/logrus v1.2.0
+	github.com/jinzhu/gorm v1.9.16
+	github.com/mitchellh/mapstructure v1.2.2
+	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/viper v1.7.0
 	github.com/stretchr/testify v1.5.1
 	github.com/urfave/cli/v2 v2.2.0
-	golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
-	gopkg.in/yaml.v2 v2.3.0 // indirect
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	gorm.io/driver/sqlite v1.1.3
+	gorm.io/gorm v1.20.2
+)
+
+require (
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/citilinkru/libudev v1.0.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deepmap/oapi-codegen v1.8.2 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/go-playground/locales v0.13.0 // indirect
+	github.com/go-playground/universal-translator v0.17.0 // indirect
+	github.com/go-playground/validator/v10 v10.2.0 // indirect
+	github.com/golang/protobuf v1.4.2 // indirect
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
+	github.com/jaypipes/pcidb v0.5.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.1 // indirect
+	github.com/json-iterator/go v1.1.9 // indirect
+	github.com/klauspost/compress v1.12.1 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect
+	github.com/leodido/go-urn v1.2.0 // indirect
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-sqlite3 v1.14.4 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo v1.16.1 // indirect
+	github.com/pelletier/go-toml v1.7.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/ugorji/go/codec v1.1.7 // indirect
+	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
+	golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
+	golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect
+	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/protobuf v1.23.0 // indirect
+	gopkg.in/ini.v1 v1.55.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gosrc.io/xmpp v0.5.1 // indirect
+	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
+	nhooyr.io/websocket v1.8.7 // indirect
 )

go.sum

@@ -11,14 +11,14 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae h1:iYSadgTTTmFTvZwdDImnytps8Hq9zlpWeNfYpe1RTPs=
-github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae/go.mod h1:0jFBtvNk8rNzZjL8j1b852fcka5/VJfJvRiU2w6OIkI=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/agnivade/wasmbrowsertest v0.3.1/go.mod h1:zQt6ZTdl338xxRaMW395qccVE2eQm0SjC/SDz0mPWQI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14 h1:wsrSjiqQtseStRIoLLxS4C5IEtXkazZVEPDHq8jW7r8=
@@ -33,50 +33,71 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chromedp/cdproto v0.0.0-20190614062957-d6d2f92b486d/go.mod h1:S8mB5wY3vV+vRIzf39xDXsw3XKYewW9X6rW2aEmkrSw=
+github.com/chromedp/cdproto v0.0.0-20190621002710-8cbd498dd7a0/go.mod h1:S8mB5wY3vV+vRIzf39xDXsw3XKYewW9X6rW2aEmkrSw=
+github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0=
+github.com/chromedp/cdproto v0.0.0-20190926234355-1b4886c6fad6/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0=
+github.com/chromedp/chromedp v0.3.1-0.20190619195644-fd957a4d2901/go.mod h1:mJdvfrVn594N9tfiPecUidF6W5jPRKHymqHfzbobPsM=
+github.com/chromedp/chromedp v0.4.0/go.mod h1:DC3QUn4mJ24dwjcaGQLoZrhm4X/uPHZ6spDbS2uFhm4=
+github.com/citilinkru/libudev v1.0.0 h1:upErSdhsJGdiKxwxPmvcz43fwJJD9R+y1j8BqU4wHog=
+github.com/citilinkru/libudev v1.0.0/go.mod h1:yaNdhdtfJMs5flqeXzUOMO0mT9QnyNh/U/jdY4WhA/I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a h1:6ZMiughZYF6fJjFIf2X3D7AfImJeXnTMJ9qC2v75WPw=
-github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a/go.mod h1:z3pUtEhu5zOpu+Q8wZWiEq+ZLL9hM0HiFNhttaI67Ks=
+github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
+github.com/containrrr/shoutrrr v0.4.4 h1:vHZ4E/76pKVY+Jyn/qhBz3X540Bn8NI5ppPHK4PyILY=
+github.com/containrrr/shoutrrr v0.4.4/go.mod h1:zqL2BvfC1W4FujrT4b3/ZCLxvD+uoeEpBL7rg9Dqpbg=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
-github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
+github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU=
+github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
 github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg=
+github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
-github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
-github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
-github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gormigrate/gormigrate/v2 v2.0.0 h1:e2A3Uznk4viUC4UuemuVgsNnvYZyOA8B3awlYk3UioU=
+github.com/go-gormigrate/gormigrate/v2 v2.0.0/go.mod h1:YuVJ+D/dNt4HWrThTBnjgZuRbt7AuwINeg4q52ZE3Jw=
+github.com/go-interpreter/wagon v0.5.1-0.20190713202023-55a163980b6c/go.mod h1:5+b/MBYkclRZngKF5s6qrgWxSLgE9F5dFdO1hAueZLc=
+github.com/go-interpreter/wagon v0.6.0/go.mod h1:5+b/MBYkclRZngKF5s6qrgWxSLgE9F5dFdO1hAueZLc=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -84,10 +105,20 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
+github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
+github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -99,21 +130,42 @@ github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190908185732-236ed259b199/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -139,53 +191,141 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/influxdb-client-go/v2 v2.9.0 h1:1Ejxpt+cpWkadefxd5xvVx7pFgFaafdNp1ItfHzKRW4=
+github.com/influxdata/influxdb-client-go/v2 v2.9.0/go.mod h1:x7Jo5UHHl+w8wu8UnGiNobDDHygojXwJX4mx7rXGKMk=
+github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
+github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
+github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
+github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
+github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
+github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
+github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
+github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
+github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk=
+github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
+github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
+github.com/jackc/pgconn v1.6.4 h1:S7T6cx5o2OqmxdHaXLH1ZeD1SbI8jBznyYE9Ec0RCQ8=
+github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78=
+github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
+github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
+github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
+github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.0.2 h1:q1Hsy66zh4vuNsajBUF2PNqfAMMfxU5mk594lPE9vjY=
+github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
+github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
+github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
+github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0=
+github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
+github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
+github.com/jackc/pgtype v1.4.2 h1:t+6LWm5eWPLX1H5Se702JSBcirq6uWa4jiG4wV1rAWY=
+github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
+github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
+github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
+github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
+github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA=
+github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
+github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
+github.com/jackc/pgx/v4 v4.8.1 h1:SUbCLP2pXvf/Sr/25KsuI4aTxiFYIvpfk4l6aTSdyCw=
+github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0=
+github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
 github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jaypipes/ghw v0.6.1 h1:Ewt3mdpiyhWotGyzg1ursV/6SnToGcG4215X6rR2af8=
 github.com/jaypipes/ghw v0.6.1/go.mod h1:QOXppNRCLGYR1H+hu09FxZPqjNt09bqUZUnOL3Rcero=
 github.com/jaypipes/pcidb v0.5.0 h1:4W5gZ+G7QxydevI8/MmmKdnIPJpURqJ2JNXTzfLxF5c=
 github.com/jaypipes/pcidb v0.5.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jinzhu/gorm v1.9.14 h1:Kg3ShyTPcM6nzVo148fRrcMO6MNKuqtOUwnzqMgVniM=
-github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
+github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o=
+github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
+github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.12.1 h1:/+xsCsk06wE38cyiqOR/o7U2fSftcH72xD+BQXmja/g=
+github.com/klauspost/compress v1.12.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 h1:3tLzEnUizyN9YLWFTT9loC30lSBvh2y70LTDcZOTs1s=
 github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0/go.mod h1:8/LTPeDLaklcUjgSQBHbhBF1ibKAFxzS5o+H7USfMSA=
+github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg=
+github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
-github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
+github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
+github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI=
+github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -196,7 +336,6 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4=
 github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -207,19 +346,29 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.6/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54=
+github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -234,34 +383,42 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
+github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -271,6 +428,7 @@ github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -279,6 +437,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/twitchyliquid64/golang-asm v0.0.0-20190126203739-365674df15fc/go.mod h1:NoCfSFWosfqMqmmD7hApkirIK9ozpHjxRnRxs1l413A=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
@@ -286,23 +445,42 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+go.coder.com/go-tools v0.0.0-20190317003359-0c6a35b74a16/go.mod h1:iKV5yK9t+J5nG9O3uF6KYdPEz3dyfMyB15MN1rbQ8Qw=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180426230345-b49d69b5da94/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -321,11 +499,13 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -338,16 +518,26 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -357,26 +547,51 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190306220234-b354f8bf4d9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190618155005-516e3c20635f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa h1:mQTN3ECqfsViCNBgq+A40vdwhkGykrrQlYe3mPj6BoU=
-golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 h1:iGu644GcxtEcrInvDsQRCwJjtCIOlT2V7IRt6ah2Whw=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -386,18 +601,29 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -420,13 +646,22 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ=
 gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@@ -440,14 +675,35 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gosrc.io/xmpp v0.1.1 h1:iMtE9W3fx254+4E6rI34AOPJDqWvpfQR6EYaVMzhJ4s=
-gosrc.io/xmpp v0.1.1/go.mod h1:4JgaXzw4MnEv2sGltONtK3GMhj+h9gpQ7cO8nwbFJLU=
+gorm.io/driver/mysql v1.0.1 h1:omJoilUzyrAp0xNoio88lGJCroGdIOen9hq2A/+3ifw=
+gorm.io/driver/mysql v1.0.1/go.mod h1:KtqSthtg55lFp3S5kUXqlGaelnWpKitn4k1xZTnoiPw=
+gorm.io/driver/postgres v1.0.0 h1:Yh4jyFQ0a7F+JPU0Gtiam/eKmpT/XFc1FKxotGqc6FM=
+gorm.io/driver/postgres v1.0.0/go.mod h1:wtMFcOzmuA5QigNsgEIb7O5lhvH1tHAF1RbWmLWV4to=
+gorm.io/driver/sqlite v1.1.1/go.mod h1:hm2olEcl8Tmsc6eZyxYSeznnsDaMqamBvEXLNtBg4cI=
+gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc=
+gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c=
+gorm.io/driver/sqlserver v1.0.2 h1:FzxAlw0/7hntMzSiNfotpYCo9Lz8dqWQGdmCGqIiFGo=
+gorm.io/driver/sqlserver v1.0.2/go.mod h1:gb0Y9QePGgqjzrVyTQUZeh9zkd5v0iz71cM1B4ZycEY=
+gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+gorm.io/gorm v1.20.2 h1:bZzSEnq7NDGsrd+n3evOOedDrY5oLM5QPlCjZJUK2ro=
+gorm.io/gorm v1.20.2/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+gosrc.io/xmpp v0.5.1 h1:Rgrm5s2rt+npGggJH3HakQxQXR8ZZz3+QRzakRQqaq4=
+gosrc.io/xmpp v0.5.1/go.mod h1:L3NFMqYOxyLz3JGmgFyWf7r9htE91zVGiK40oW4RwdY=
+gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/gotestsum v0.3.5/go.mod h1:Mnf3e5FUzXbkCfynWBGOwLssY7gTQgCHObK9tMpAriY=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
+mvdan.cc/sh v2.6.4+incompatible/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8=
+nhooyr.io/websocket v1.6.5/go.mod h1:F259lAzPRAH0htX2y3ehpJe09ih1aSHN7udWki1defY=
+nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
+nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
+nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

packagr.yml

@@ -1,18 +1,6 @@
 ---
 engine_enable_code_mutation: true
-engine_cmd_compile:
-  - go build -ldflags '-w -extldflags "-static"' -o scrutiny webapp/backend/cmd/scrutiny/scrutiny.go
-  - 'GOOS=linux GOARCH=amd64 go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go'
-  - 'chmod +x scrutiny-web-linux-amd64'
-  - 'GOOS=linux GOARCH=amd64 go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go'
-  - 'chmod +x scrutiny-collector-metrics-linux-amd64'
+mgr_keep_lock_file: true
 engine_version_metadata_path: 'webapp/backend/pkg/version/version.go'
-engine_cmd_test: 'go test -v -tags "static" $(go list ./... | grep -v /vendor/)'
 engine_golang_package_path: 'github.com/analogj/scrutiny'
-scm_enable_branch_cleanup: true
-engine_disable_lint: true
-scm_release_assets:
-  - local_path: scrutiny-web-linux-amd64
-    artifact_name: scrutiny-web-linux-amd64
-  - local_path: scrutiny-collector-metrics-linux-amd64
-    artifact_name: scrutiny-collector-metrics-linux-amd64
+

rootfs/etc/cont-init.d/01-timezone

@@ -0,0 +1,7 @@
+#!/usr/bin/with-contenv bash
+
+if [ -n "${TZ}" ]
+then
+    ln -snf "/usr/share/zoneinfo/${TZ}" /etc/localtime
+    echo "${TZ}" > /etc/timezone
+fi

rootfs/etc/cont-init.d/50-cron-config

@@ -0,0 +1,15 @@
+#!/usr/bin/with-contenv bash
+
+# Cron runs in its own isolated environment (usually using only /etc/environment )
+# So when the container starts up, we will do a dump of the runtime environment into a .env file that we
+# will then source into the crontab file (/etc/cron.d/scrutiny)
+(set -o posix; export -p) > /env.sh
+
+# adding ability to customize the cron schedule.
+COLLECTOR_CRON_SCHEDULE=${COLLECTOR_CRON_SCHEDULE:-"0 0 * * *"}
+
+# if the cron schedule has been overridden via env variable (eg docker-compose) we should make sure to strip quotes
+[[ "${COLLECTOR_CRON_SCHEDULE}" == \"*\" || "${COLLECTOR_CRON_SCHEDULE}" == \'*\' ]] && COLLECTOR_CRON_SCHEDULE="${COLLECTOR_CRON_SCHEDULE:1:-1}"
+
+# replace placeholder with correct value
+sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron.d/scrutiny

rootfs/etc/cron.d/scrutiny

@@ -0,0 +1,15 @@
+MAILTO=""
+# Example of job definition:
+# .---------------- minute (0 - 59)
+# |  .------------- hour (0 - 23)
+# |  |  .---------- day of month (1 - 31)
+# |  |  |  .------- month (1 - 12) OR jan,feb,mar,apr ...
+# |  |  |  |  .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat
+# |  |  |  |  |
+# *  *  *  *  * user-name command to be executed
+
+# correctly route collector logs (STDOUT & STDERR) to Cron foreground (collectable by Docker STDOUT)
+# cron schedule to run daily at midnight:  '0 0 * * *'
+# System environmental variables are stripped by cron, source our dump of the docker environmental variables before each command (/env.sh)
+{COLLECTOR_CRON_SCHEDULE} root . /env.sh; /opt/scrutiny/bin/scrutiny-collector-metrics run >/proc/1/fd/1 2>/proc/1/fd/2
+# An empty line is required at the end of this file for a valid cron file.

rootfs/etc/services.d/collector-once/run

@@ -0,0 +1,13 @@
+#!/usr/bin/with-contenv bash
+
+echo "waiting for scrutiny service to start"
+s6-svwait -u /var/run/s6/services/scrutiny
+
+#tell s6 to only run this script once
+s6-svc -O /var/run/s6/services/collector-once
+
+# wait until scrutiny is "Ready"
+until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done
+
+echo "starting scrutiny collector (run-once mode. subsequent calls will be triggered via cron service)"
+/opt/scrutiny/bin/scrutiny-collector-metrics run

rootfs/etc/services.d/cron/finish

@@ -1,4 +1,4 @@
 #!/usr/bin/execlineb -S0
 
-echo "jobber/cron exiting"
+echo "cron exiting"
 s6-svscanctl -t /var/run/s6/services

rootfs/etc/services.d/cron/run

@@ -0,0 +1,4 @@
+#!/usr/bin/with-contenv bash
+
+echo "starting cron"
+cron -f -L 15

rootfs/etc/services.d/influxdb/run

@@ -0,0 +1,17 @@
+#!/usr/bin/with-contenv bash
+
+mkdir -p /opt/scrutiny/influxdb/
+
+if [ -f "/opt/scrutiny/influxdb/config.yaml" ]; then
+    echo "influxdb config file already exists. skipping."
+else
+cat << 'EOF' > /opt/scrutiny/influxdb/config.yaml
+bolt-path: /opt/scrutiny/influxdb/influxd.bolt
+engine-path: /opt/scrutiny/influxdb/engine
+http-bind-address: ":8086"
+reporting-disabled: true
+EOF
+fi
+
+echo "starting influxdb"
+influxd run

rootfs/etc/services.d/jobber/run

@@ -1,5 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-echo "starting jobber/cron"
-
-su -c "/usr/lib/x86_64-linux-gnu/jobberrunner /scrutiny/jobber/jobber.yaml" root

rootfs/etc/services.d/scrutiny/run

@@ -1,5 +1,7 @@
 #!/usr/bin/with-contenv bash
 
-echo "starting scrutiny"
+echo "waiting for influxdb"
+until $(curl --output /dev/null --silent --head --fail http://localhost:8086/health); do echo "influxdb not ready" && sleep 5; done
 
+echo "starting scrutiny"
 scrutiny start

rootfs/scrutiny/jobber/jobber.yaml

@@ -1,30 +0,0 @@
-version: 1.4
-
-prefs:
-  logPath: /scrutiny/jobber/log.log
-  runLog:
-    type: file
-    path: /scrutiny/jobber/runlog
-    maxFileLen: 100m
-    maxHistories: 2
-
-resultSinks:
-  - &filesystemSink
-    type: filesystem
-    path: /scrutiny/jobber
-    data:
-      - stdout
-      - stderr
-    maxAgeDays: 10
-
-jobs:
-  MetricsJob:
-    cmd: /scrutiny/bin/scrutiny-collector-metrics run
-    # run daily at midnight.
-    time: '0 0 0 * * *'
-    onError: Backoff
-    notifyOnSuccess:
-      - *filesystemSink
-    notifyOnFailure:
-      - *filesystemSink
-

webapp/backend/cmd/scrutiny/scrutiny.go

@@ -27,8 +27,8 @@ func main() {
 	}
 
 	//we're going to load the config file manually, since we need to validate it.
-	err = config.ReadConfig("/scrutiny/config/scrutiny.yaml") // Find and read the config file
-	if _, ok := err.(errors.ConfigFileMissingError); ok {     // Handle errors reading the config file
+	err = config.ReadConfig("/opt/scrutiny/config/scrutiny.yaml") // Find and read the config file
+	if _, ok := err.(errors.ConfigFileMissingError); ok {         // Handle errors reading the config file
 		//ignore "could not find config file"
 	} else if err != nil {
 		os.Exit(1)
@@ -60,7 +60,7 @@ OPTIONS:
 		},
 		Before: func(c *cli.Context) error {
 
-			drawbridge := "github.com/AnalogJ/scrutiny"
+			scrutiny := "github.com/AnalogJ/scrutiny"
 
 			var versionInfo string
 			if len(goos) > 0 && len(goarch) > 0 {
@@ -69,7 +69,7 @@ OPTIONS:
 				versionInfo = fmt.Sprintf("dev-%s", version.VERSION)
 			}
 
-			subtitle := drawbridge + utils.LeftPad2Len(versionInfo, " ", 65-len(drawbridge))
+			subtitle := scrutiny + utils.LeftPad2Len(versionInfo, " ", 65-len(scrutiny))
 
 			color.New(color.FgGreen).Fprintf(c.App.Writer, fmt.Sprintf(utils.StripIndent(
 				`

webapp/backend/pkg/config/config.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/viper"
 	"log"
 	"os"
+	"strings"
 )
 
 // When initializing this class the following methods must be called:
@@ -29,18 +30,31 @@ func (c *configuration) Init() error {
 	//set defaults
 	c.SetDefault("web.listen.port", "8080")
 	c.SetDefault("web.listen.host", "0.0.0.0")
-	c.SetDefault("web.src.frontend.path", "/scrutiny/web")
-	c.SetDefault("web.database.location", "/scrutiny/config/scrutiny.db")
+	c.SetDefault("web.listen.basepath", "")
+	c.SetDefault("web.src.frontend.path", "/opt/scrutiny/web")
+	c.SetDefault("web.database.location", "/opt/scrutiny/config/scrutiny.db")
 
 	c.SetDefault("log.level", "INFO")
 	c.SetDefault("log.file", "")
 
+	c.SetDefault("notify.urls", []string{})
+
+	c.SetDefault("web.influxdb.scheme", "http")
+	c.SetDefault("web.influxdb.host", "localhost")
+	c.SetDefault("web.influxdb.port", "8086")
+	c.SetDefault("web.influxdb.org", "scrutiny")
+	c.SetDefault("web.influxdb.bucket", "metrics")
+	c.SetDefault("web.influxdb.init_username", "admin")
+	c.SetDefault("web.influxdb.init_password", "password12345")
+	c.SetDefault("web.influxdb.token", "scrutiny-default-admin-token")
+	c.SetDefault("web.influxdb.retention_policy", true)
+
 	//c.SetDefault("disks.include", []string{})
 	//c.SetDefault("disks.exclude", []string{})
 
-	//c.SetDefault("notify.metric.script", "/scrutiny/config/notify-metrics.sh")
-	//c.SetDefault("notify.long.script", "/scrutiny/config/notify-long-test.sh")
-	//c.SetDefault("notify.short.script", "/scrutiny/config/notify-short-test.sh")
+	//c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh")
+	//c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh")
+	//c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh")
 
 	//c.SetDefault("collect.metric.enable", true)
 	//c.SetDefault("collect.metric.command", "-a -o on -S on")
@@ -54,18 +68,26 @@ func (c *configuration) Init() error {
 	//c.SetConfigName("drawbridge")
 	//c.AddConfigPath("$HOME/")
 
+	//configure env variable parsing.
+	c.SetEnvPrefix("SCRUTINY")
+	c.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
+	c.AutomaticEnv()
+
 	//CLI options will be added via the `Set()` function
 	return nil
 }
 
 func (c *configuration) ReadConfig(configFilePath string) error {
+	//make sure that we specify that this is the correct config path (for eventual WriteConfig() calls)
+	c.SetConfigFile(configFilePath)
+
 	configFilePath, err := utils.ExpandPath(configFilePath)
 	if err != nil {
 		return err
 	}
 
 	if !utils.FileExists(configFilePath) {
-		log.Printf("No configuration file found at %v. Skipping", configFilePath)
+		log.Printf("No configuration file found at %v. Using Defaults.", configFilePath)
 		return errors.ConfigFileMissingError("The configuration file could not be found.")
 	}
 

webapp/backend/pkg/config/interface.go

@@ -5,10 +5,11 @@ import (
 )
 
 // Create mock using:
-// mockgen -source=pkg/config/interface.go -destination=pkg/config/mock/mock_config.go
+// mockgen -source=webapp/backend/pkg/config/interface.go -destination=webapp/backend/pkg/config/mock/mock_config.go
 type Interface interface {
 	Init() error
 	ReadConfig(configFilePath string) error
+	WriteConfig() error
 	Set(key string, value interface{})
 	SetDefault(key string, value interface{})
 

webapp/backend/pkg/config/mock/mock_config.go

@@ -5,87 +5,36 @@
 package mock_config
 
 import (
+	reflect "reflect"
+
 	gomock "github.com/golang/mock/gomock"
 	viper "github.com/spf13/viper"
-	reflect "reflect"
 )
 
-// MockInterface is a mock of Interface interface
+// MockInterface is a mock of Interface interface.
 type MockInterface struct {
 	ctrl     *gomock.Controller
 	recorder *MockInterfaceMockRecorder
 }
 
-// MockInterfaceMockRecorder is the mock recorder for MockInterface
+// MockInterfaceMockRecorder is the mock recorder for MockInterface.
 type MockInterfaceMockRecorder struct {
 	mock *MockInterface
 }
 
-// NewMockInterface creates a new mock instance
+// NewMockInterface creates a new mock instance.
 func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
 	mock := &MockInterface{ctrl: ctrl}
 	mock.recorder = &MockInterfaceMockRecorder{mock}
 	return mock
 }
 
-// EXPECT returns an object that allows the caller to indicate expected use
+// EXPECT returns an object that allows the caller to indicate expected use.
 func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
-// Init mocks base method
-func (m *MockInterface) Init() error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Init")
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Init indicates an expected call of Init
-func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
-}
-
-// ReadConfig mocks base method
-func (m *MockInterface) ReadConfig(configFilePath string) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// ReadConfig indicates an expected call of ReadConfig
-func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
-}
-
-// Set mocks base method
-func (m *MockInterface) Set(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Set", key, value)
-}
-
-// Set indicates an expected call of Set
-func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
-}
-
-// SetDefault mocks base method
-func (m *MockInterface) SetDefault(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "SetDefault", key, value)
-}
-
-// SetDefault indicates an expected call of SetDefault
-func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
-}
-
-// AllSettings mocks base method
+// AllSettings mocks base method.
 func (m *MockInterface) AllSettings() map[string]interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "AllSettings")
@@ -93,27 +42,13 @@ func (m *MockInterface) AllSettings() map[string]interface{} {
 	return ret0
 }
 
-// AllSettings indicates an expected call of AllSettings
+// AllSettings indicates an expected call of AllSettings.
 func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings))
 }
 
-// IsSet mocks base method
-func (m *MockInterface) IsSet(key string) bool {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "IsSet", key)
-	ret0, _ := ret[0].(bool)
-	return ret0
-}
-
-// IsSet indicates an expected call of IsSet
-func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
-}
-
-// Get mocks base method
+// Get mocks base method.
 func (m *MockInterface) Get(key string) interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Get", key)
@@ -121,13 +56,13 @@ func (m *MockInterface) Get(key string) interface{} {
 	return ret0
 }
 
-// Get indicates an expected call of Get
+// Get indicates an expected call of Get.
 func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key)
 }
 
-// GetBool mocks base method
+// GetBool mocks base method.
 func (m *MockInterface) GetBool(key string) bool {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetBool", key)
@@ -135,13 +70,13 @@ func (m *MockInterface) GetBool(key string) bool {
 	return ret0
 }
 
-// GetBool indicates an expected call of GetBool
+// GetBool indicates an expected call of GetBool.
 func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key)
 }
 
-// GetInt mocks base method
+// GetInt mocks base method.
 func (m *MockInterface) GetInt(key string) int {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetInt", key)
@@ -149,13 +84,13 @@ func (m *MockInterface) GetInt(key string) int {
 	return ret0
 }
 
-// GetInt indicates an expected call of GetInt
+// GetInt indicates an expected call of GetInt.
 func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
 }
 
-// GetString mocks base method
+// GetString mocks base method.
 func (m *MockInterface) GetString(key string) string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetString", key)
@@ -163,13 +98,13 @@ func (m *MockInterface) GetString(key string) string {
 	return ret0
 }
 
-// GetString indicates an expected call of GetString
+// GetString indicates an expected call of GetString.
 func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key)
 }
 
-// GetStringSlice mocks base method
+// GetStringSlice mocks base method.
 func (m *MockInterface) GetStringSlice(key string) []string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetStringSlice", key)
@@ -177,13 +112,79 @@ func (m *MockInterface) GetStringSlice(key string) []string {
 	return ret0
 }
 
-// GetStringSlice indicates an expected call of GetStringSlice
+// GetStringSlice indicates an expected call of GetStringSlice.
 func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key)
 }
 
-// UnmarshalKey mocks base method
+// Init mocks base method.
+func (m *MockInterface) Init() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Init")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Init indicates an expected call of Init.
+func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
+}
+
+// IsSet mocks base method.
+func (m *MockInterface) IsSet(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsSet", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsSet indicates an expected call of IsSet.
+func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
+}
+
+// ReadConfig mocks base method.
+func (m *MockInterface) ReadConfig(configFilePath string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReadConfig indicates an expected call of ReadConfig.
+func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
+}
+
+// Set mocks base method.
+func (m *MockInterface) Set(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Set", key, value)
+}
+
+// Set indicates an expected call of Set.
+func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
+}
+
+// SetDefault mocks base method.
+func (m *MockInterface) SetDefault(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetDefault", key, value)
+}
+
+// SetDefault indicates an expected call of SetDefault.
+func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
+}
+
+// UnmarshalKey mocks base method.
 func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{key, rawVal}
@@ -195,9 +196,23 @@ func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts
 	return ret0
 }
 
-// UnmarshalKey indicates an expected call of UnmarshalKey
+// UnmarshalKey indicates an expected call of UnmarshalKey.
 func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{key, rawVal}, decoderOpts...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...)
 }
+
+// WriteConfig mocks base method.
+func (m *MockInterface) WriteConfig() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "WriteConfig")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// WriteConfig indicates an expected call of WriteConfig.
+func (mr *MockInterfaceMockRecorder) WriteConfig() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteConfig", reflect.TypeOf((*MockInterface)(nil).WriteConfig))
+}

webapp/backend/pkg/constants.go

@@ -0,0 +1,37 @@
+package pkg
+
+const DeviceProtocolAta = "ATA"
+const DeviceProtocolScsi = "SCSI"
+const DeviceProtocolNvme = "NVMe"
+
+type AttributeStatus uint8
+
+const (
+	// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
+	AttributeStatusPassed          AttributeStatus = 0
+	AttributeStatusFailedSmart     AttributeStatus = 1
+	AttributeStatusWarningScrutiny AttributeStatus = 2
+	AttributeStatusFailedScrutiny  AttributeStatus = 4
+)
+
+const AttributeWhenFailedFailingNow = "FAILING_NOW"
+const AttributeWhenFailedInThePast = "IN_THE_PAST"
+
+func AttributeStatusSet(b, flag AttributeStatus) AttributeStatus    { return b | flag }
+func AttributeStatusClear(b, flag AttributeStatus) AttributeStatus  { return b &^ flag }
+func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ flag }
+func AttributeStatusHas(b, flag AttributeStatus) bool               { return b&flag != 0 }
+
+type DeviceStatus uint8
+
+const (
+	// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
+	DeviceStatusPassed         DeviceStatus = 0
+	DeviceStatusFailedSmart    DeviceStatus = 1
+	DeviceStatusFailedScrutiny DeviceStatus = 2
+)
+
+func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus    { return b | flag }
+func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
+func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
+func DeviceStatusHas(b, flag DeviceStatus) bool            { return b&flag != 0 }

webapp/backend/pkg/database/interface.go

@@ -0,0 +1,31 @@
+package database
+
+import (
+	"context"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+)
+
+type DeviceRepo interface {
+	Close() error
+
+	//GetSettings()
+	//SaveSetting()
+
+	RegisterDevice(ctx context.Context, dev models.Device) error
+	GetDevices(ctx context.Context) ([]models.Device, error)
+	UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error)
+	UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error)
+	GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error)
+	DeleteDevice(ctx context.Context, wwn string) error
+
+	SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error)
+	GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error)
+
+	SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error
+
+	GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
+	GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)
+}

webapp/backend/pkg/database/migrations/m20201107210306/device.go

@@ -0,0 +1,47 @@
+package m20201107210306
+
+import (
+	"time"
+)
+
+// Deprecated: m20201107210306.Device is deprecated, only used by db migrations
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN    string `json:"wwn" gorm:"primary_key"`
+	HostId string `json:"host_id"`
+
+	DeviceName     string  `json:"device_name"`
+	Manufacturer   string  `json:"manufacturer"`
+	ModelName      string  `json:"model_name"`
+	InterfaceType  string  `json:"interface_type"`
+	InterfaceSpeed string  `json:"interface_speed"`
+	SerialNumber   string  `json:"serial_number"`
+	Firmware       string  `json:"firmware"`
+	RotationSpeed  int     `json:"rotational_speed"`
+	Capacity       int64   `json:"capacity"`
+	FormFactor     string  `json:"form_factor"`
+	SmartSupport   bool    `json:"smart_support"`
+	DeviceProtocol string  `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string  `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+	SmartResults   []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
+}
+
+const DeviceProtocolAta = "ATA"
+const DeviceProtocolScsi = "SCSI"
+const DeviceProtocolNvme = "NVMe"
+
+func (dv *Device) IsAta() bool {
+	return dv.DeviceProtocol == DeviceProtocolAta
+}
+
+func (dv *Device) IsScsi() bool {
+	return dv.DeviceProtocol == DeviceProtocolScsi
+}
+
+func (dv *Device) IsNvme() bool {
+	return dv.DeviceProtocol == DeviceProtocolNvme
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart.go

@@ -0,0 +1,26 @@
+package m20201107210306
+
+import (
+	"gorm.io/gorm"
+	"time"
+)
+
+// Deprecated: m20201107210306.Smart is deprecated, only used by db migrations
+type Smart struct {
+	gorm.Model
+
+	DeviceWWN string `json:"device_wwn"`
+	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
+
+	TestDate    time.Time `json:"date"`
+	SmartStatus string    `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed
+
+	//Metrics
+	Temp            int64 `json:"temp"`
+	PowerOnHours    int64 `json:"power_on_hours"`
+	PowerCycleCount int64 `json:"power_cycle_count"`
+
+	AtaAttributes  []SmartAtaAttribute  `json:"ata_attributes" gorm:"foreignkey:SmartId"`
+	NvmeAttributes []SmartNvmeAttribute `json:"nvme_attributes" gorm:"foreignkey:SmartId"`
+	ScsiAttributes []SmartScsiAttribute `json:"scsi_attributes" gorm:"foreignkey:SmartId"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go

@@ -0,0 +1,26 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartAtaAttribute is deprecated, only used by db migrations
+type SmartAtaAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId int    `json:"attribute_id"`
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Worst       int    `json:"worst"`
+	Threshold   int    `json:"thresh"`
+	RawValue    int64  `json:"raw_value"`
+	RawString   string `json:"raw_string"`
+	WhenFailed  string `json:"when_failed"`
+
+	TransformedValue int64               `json:"transformed_value"`
+	Status           string              `gorm:"-" json:"status,omitempty"`
+	StatusReason     string              `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64             `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartAtaAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go

@@ -0,0 +1,22 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartNvmeAttribute is deprecated, only used by db migrations
+type SmartNvmeAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Threshold   int    `json:"thresh"`
+
+	TransformedValue int64                `json:"transformed_value"`
+	Status           string               `gorm:"-" json:"status,omitempty"`
+	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartNvmeAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go

@@ -0,0 +1,22 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartScsiAttribute is deprecated, only used by db migrations
+type SmartScsiAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Threshold   int    `json:"thresh"`
+
+	TransformedValue int64                `json:"transformed_value"`
+	Status           string               `gorm:"-" json:"status,omitempty"`
+	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartScsiAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/migrations/m20220503120000/device.go

@@ -0,0 +1,37 @@
+package m20220503120000
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"time"
+)
+
+// Deprecated: m20220503120000.Device is deprecated, only used by db migrations
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN string `json:"wwn" gorm:"primary_key"`
+
+	DeviceName     string `json:"device_name"`
+	Manufacturer   string `json:"manufacturer"`
+	ModelName      string `json:"model_name"`
+	InterfaceType  string `json:"interface_type"`
+	InterfaceSpeed string `json:"interface_speed"`
+	SerialNumber   string `json:"serial_number"`
+	Firmware       string `json:"firmware"`
+	RotationSpeed  int    `json:"rotational_speed"`
+	Capacity       int64  `json:"capacity"`
+	FormFactor     string `json:"form_factor"`
+	SmartSupport   bool   `json:"smart_support"`
+	DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string `json:"label"`
+	HostId string `json:"host_id"`
+
+	// Data set by Scrutiny
+	DeviceStatus pkg.DeviceStatus `json:"device_status"`
+}

webapp/backend/pkg/database/migrations/m20220509170100/device.go

@@ -0,0 +1,41 @@
+package m20220509170100
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"time"
+)
+
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN string `json:"wwn" gorm:"primary_key"`
+
+	DeviceName     string `json:"device_name"`
+	DeviceUUID	   string `json:"device_uuid"`
+	DeviceSerialID	   string `json:"device_serial_id"`
+	DeviceLabel	   string `json:"device_label"`
+
+	Manufacturer   string `json:"manufacturer"`
+	ModelName      string `json:"model_name"`
+	InterfaceType  string `json:"interface_type"`
+	InterfaceSpeed string `json:"interface_speed"`
+	SerialNumber   string `json:"serial_number"`
+	Firmware       string `json:"firmware"`
+	RotationSpeed  int    `json:"rotational_speed"`
+	Capacity       int64  `json:"capacity"`
+	FormFactor     string `json:"form_factor"`
+	SmartSupport   bool   `json:"smart_support"`
+	DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string `json:"label"`
+	HostId string `json:"host_id"`
+
+	// Data set by Scrutiny
+	DeviceStatus pkg.DeviceStatus `json:"device_status"`
+}
+

webapp/backend/pkg/database/scrutiny_repository.go

@@ -0,0 +1,444 @@
+package database
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api"
+	"github.com/influxdata/influxdb-client-go/v2/domain"
+	"github.com/sirupsen/logrus"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+const (
+	// 60seconds * 60minutes * 24hours * 15 days
+	RETENTION_PERIOD_15_DAYS_IN_SECONDS = 1_296_000
+
+	// 60seconds * 60minutes * 24hours * 7 days * 9 weeks
+	RETENTION_PERIOD_9_WEEKS_IN_SECONDS = 5_443_200
+
+	// 60seconds * 60minutes * 24hours * 7 days * (52 + 52 + 4)weeks
+	RETENTION_PERIOD_25_MONTHS_IN_SECONDS = 65_318_400
+
+	DURATION_KEY_WEEK    = "week"
+	DURATION_KEY_MONTH   = "month"
+	DURATION_KEY_YEAR    = "year"
+	DURATION_KEY_FOREVER = "forever"
+)
+
+//// GormLogger is a custom logger for Gorm, making it use logrus.
+//type GormLogger struct{ Logger logrus.FieldLogger }
+//
+//// Print handles log events from Gorm for the custom logger.
+//func (gl *GormLogger) Print(v ...interface{}) {
+//	switch v[0] {
+//	case "sql":
+//		gl.Logger.WithFields(
+//			logrus.Fields{
+//				"module":  "gorm",
+//				"type":    "sql",
+//				"rows":    v[5],
+//				"src_ref": v[1],
+//				"values":  v[4],
+//			},
+//		).Debug(v[3])
+//	case "log":
+//		gl.Logger.WithFields(logrus.Fields{"module": "gorm", "type": "log"}).Print(v[2])
+//	}
+//}
+
+func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.FieldLogger) (DeviceRepo, error) {
+	backgroundContext := context.Background()
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// Gorm/SQLite setup
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
+	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
+		//TODO: figure out how to log database queries again.
+		//Logger: logger
+		DisableForeignKeyConstraintWhenMigrating: true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Failed to connect to database! - %v", err)
+	}
+	globalLogger.Infof("Successfully connected to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
+
+	//database.SetLogger()
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// InfluxDB setup
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+	// Create a new client using an InfluxDB server base URL and an authentication token
+	influxdbUrl := fmt.Sprintf("%s://%s:%s", appConfig.GetString("web.influxdb.scheme"), appConfig.GetString("web.influxdb.host"), appConfig.GetString("web.influxdb.port"))
+	globalLogger.Debugf("InfluxDB url: %s", influxdbUrl)
+
+	client := influxdb2.NewClient(influxdbUrl, appConfig.GetString("web.influxdb.token"))
+
+	//if !appConfig.IsSet("web.influxdb.token") {
+	globalLogger.Debugf("Determine Influxdb setup status...")
+	influxSetupComplete, err := InfluxSetupComplete(influxdbUrl)
+	if err != nil {
+		return nil, fmt.Errorf("failed to check influxdb setup status - %w", err)
+	}
+
+	if !influxSetupComplete {
+		globalLogger.Debugf("Influxdb un-initialized, running first-time setup...")
+
+		// if no token is provided, but we have a valid server, we're going to assume this is the first setup of our server.
+		// we will initialize with a predetermined username & password, that you should change.
+
+		// metrics bucket will have a retention period of 8 days (since it will be down-sampled once a week)
+		// in seconds (60seconds * 60minutes * 24hours * 15 days) = 1_296_000 (see EnsureBucket() function)
+		_, err := client.SetupWithToken(
+			backgroundContext,
+			appConfig.GetString("web.influxdb.init_username"),
+			appConfig.GetString("web.influxdb.init_password"),
+			appConfig.GetString("web.influxdb.org"),
+			appConfig.GetString("web.influxdb.bucket"),
+			0,
+			appConfig.GetString("web.influxdb.token"),
+		)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Use blocking write client for writes to desired bucket
+	writeAPI := client.WriteAPIBlocking(appConfig.GetString("web.influxdb.org"), appConfig.GetString("web.influxdb.bucket"))
+
+	// Get query client
+	queryAPI := client.QueryAPI(appConfig.GetString("web.influxdb.org"))
+
+	// Get task client
+	taskAPI := client.TasksAPI()
+
+	if writeAPI == nil || queryAPI == nil || taskAPI == nil {
+		return nil, fmt.Errorf("Failed to connect to influxdb!")
+	}
+
+	deviceRepo := scrutinyRepository{
+		appConfig:      appConfig,
+		logger:         globalLogger,
+		influxClient:   client,
+		influxWriteApi: writeAPI,
+		influxQueryApi: queryAPI,
+		influxTaskApi:  taskAPI,
+		gormClient:     database,
+	}
+
+	orgInfo, err := client.OrganizationsAPI().FindOrganizationByName(backgroundContext, appConfig.GetString("web.influxdb.org"))
+	if err != nil {
+		return nil, err
+	}
+
+	// Initialize Buckets (if necessary)
+	err = deviceRepo.EnsureBuckets(backgroundContext, orgInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	// Initialize Background Tasks
+	err = deviceRepo.EnsureTasks(backgroundContext, *orgInfo.Id)
+	if err != nil {
+		return nil, err
+	}
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// InfluxDB & SQLite migrations
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	//database.AutoMigrate(&models.Device{})
+	err = deviceRepo.Migrate(backgroundContext)
+	if err != nil {
+		return nil, err
+	}
+
+	return &deviceRepo, nil
+}
+
+type scrutinyRepository struct {
+	appConfig config.Interface
+	logger    logrus.FieldLogger
+
+	influxWriteApi api.WriteAPIBlocking
+	influxQueryApi api.QueryAPI
+	influxTaskApi  api.TasksAPI
+	influxClient   influxdb2.Client
+
+	gormClient *gorm.DB
+}
+
+func (sr *scrutinyRepository) Close() error {
+	sr.influxClient.Close()
+	return nil
+}
+
+func InfluxSetupComplete(influxEndpoint string) (bool, error) {
+	influxUri, err := url.Parse(influxEndpoint)
+	if err != nil {
+		return false, err
+	}
+	influxUri, err = influxUri.Parse("/api/v2/setup")
+	if err != nil {
+		return false, err
+	}
+
+	res, err := http.Get(influxUri.String())
+	if err != nil {
+		return false, err
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return false, err
+	}
+
+	type SetupStatus struct {
+		Allowed bool `json:"allowed"`
+	}
+	var data SetupStatus
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return false, err
+	}
+	return !data.Allowed, nil
+}
+
+func (sr *scrutinyRepository) EnsureBuckets(ctx context.Context, org *domain.Organization) error {
+
+	var mainBucketRetentionRule domain.RetentionRule
+	var weeklyBucketRetentionRule domain.RetentionRule
+	var monthlyBucketRetentionRule domain.RetentionRule
+	if sr.appConfig.GetBool("web.influxdb.retention_policy") {
+
+		// in tests, we may not want to set a retention policy. If "false", we can set data with old timestamps,
+		// then manually run the down sampling scripts. This should be true for production environments.
+		mainBucketRetentionRule = domain.RetentionRule{EverySeconds: RETENTION_PERIOD_15_DAYS_IN_SECONDS}
+		weeklyBucketRetentionRule = domain.RetentionRule{EverySeconds: RETENTION_PERIOD_9_WEEKS_IN_SECONDS}
+		monthlyBucketRetentionRule = domain.RetentionRule{EverySeconds: RETENTION_PERIOD_25_MONTHS_IN_SECONDS}
+	}
+
+	mainBucket := sr.appConfig.GetString("web.influxdb.bucket")
+	if foundMainBucket, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, mainBucket); foundErr != nil {
+		// metrics bucket will have a retention period of 15 days (since it will be down-sampled once a week)
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, mainBucket, mainBucketRetentionRule)
+		if err != nil {
+			return err
+		}
+	} else if sr.appConfig.GetBool("web.influxdb.retention_policy") {
+		//correctly set the retention period for the main bucket (cant do it during setup/creation)
+		foundMainBucket.RetentionRules = domain.RetentionRules{mainBucketRetentionRule}
+		sr.influxClient.BucketsAPI().UpdateBucket(ctx, foundMainBucket)
+	}
+
+	//create buckets (used for downsampling)
+	weeklyBucket := fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, weeklyBucket); foundErr != nil {
+		// metrics_weekly bucket will have a retention period of 8+1 weeks (since it will be down-sampled once a month)
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, weeklyBucket, weeklyBucketRetentionRule)
+		if err != nil {
+			return err
+		}
+	}
+
+	monthlyBucket := fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, monthlyBucket); foundErr != nil {
+		// metrics_monthly bucket will have a retention period of 24+1 months (since it will be down-sampled once a year)
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, monthlyBucket, monthlyBucketRetentionRule)
+		if err != nil {
+			return err
+		}
+	}
+
+	yearlyBucket := fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
+	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, yearlyBucket); foundErr != nil {
+		// metrics_yearly bucket will have an infinite retention period
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, yearlyBucket)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// DeviceSummary
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// get a map of all devices and associated SMART data
+func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) {
+	devices, err := sr.GetDevices(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	summaries := map[string]*models.DeviceSummary{}
+
+	for _, device := range devices {
+		summaries[device.WWN] = &models.DeviceSummary{Device: device}
+	}
+
+	// Get parser flux query result
+	//appConfig.GetString("web.influxdb.bucket")
+	queryStr := fmt.Sprintf(`
+  	import "influxdata/influxdb/schema"
+  	bucketBaseName = "%s"
+
+	dailyData = from(bucket: bucketBaseName)
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	weeklyData = from(bucket: bucketBaseName + "_weekly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	monthlyData = from(bucket: bucketBaseName + "_monthly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	yearlyData = from(bucket: bucketBaseName + "_yearly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	union(tables: [dailyData, weeklyData, monthlyData, yearlyData])
+	|> sort(columns: ["_time"], desc: false)
+	|> group(columns: ["device_wwn"])
+	|> last(column: "device_wwn")
+	|> yield(name: "last")
+		`,
+		sr.appConfig.GetString("web.influxdb.bucket"),
+	)
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+			// Observe when there is new grouping key producing new table
+			if result.TableChanged() {
+				//fmt.Printf("table: %s\n", result.TableMetadata().String())
+			}
+			// read result
+
+			//get summary data from Influxdb.
+			//result.Record().Values()
+			if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
+
+				//ensure summaries is intialized for this wwn
+				if _, exists := summaries[deviceWWN.(string)]; !exists {
+					summaries[deviceWWN.(string)] = &models.DeviceSummary{}
+				}
+
+				summaries[deviceWWN.(string)].SmartResults = &models.SmartSummary{
+					Temp:          result.Record().Values()["temp"].(int64),
+					PowerOnHours:  result.Record().Values()["power_on_hours"].(int64),
+					CollectorDate: result.Record().Values()["_time"].(time.Time),
+				}
+			}
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+
+	deviceTempHistory, err := sr.GetSmartTemperatureHistory(ctx, DURATION_KEY_FOREVER)
+	if err != nil {
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("Error: %v", err)
+	}
+	for wwn, tempHistory := range deviceTempHistory {
+		summaries[wwn].TempHistory = tempHistory
+	}
+
+	return summaries, nil
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Helper Methods
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+func (sr *scrutinyRepository) lookupBucketName(durationKey string) string {
+	switch durationKey {
+	case DURATION_KEY_WEEK:
+		//data stored in the last week
+		return sr.appConfig.GetString("web.influxdb.bucket")
+	case DURATION_KEY_MONTH:
+		// data stored in the last month (after the first week)
+		return fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+	case DURATION_KEY_YEAR:
+		// data stored in the last year (after the first month)
+		return fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+	case DURATION_KEY_FOREVER:
+		//data stored before the last year
+		return fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
+	}
+	return sr.appConfig.GetString("web.influxdb.bucket")
+}
+
+func (sr *scrutinyRepository) lookupDuration(durationKey string) []string {
+
+	switch durationKey {
+	case DURATION_KEY_WEEK:
+		//data stored in the last week
+		return []string{"-1w", "now()"}
+	case DURATION_KEY_MONTH:
+		// data stored in the last month (after the first week)
+		return []string{"-1mo", "-1w"}
+	case DURATION_KEY_YEAR:
+		// data stored in the last year (after the first month)
+		return []string{"-1y", "-1mo"}
+	case DURATION_KEY_FOREVER:
+		//data stored before the last year
+		return []string{"-10y", "-1y"}
+	}
+	return []string{"-1w", "now()"}
+}
+
+func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []string {
+	switch durationKey {
+	case DURATION_KEY_WEEK:
+		//all data is stored in a single bucket
+		return []string{DURATION_KEY_WEEK}
+	case DURATION_KEY_MONTH:
+		//data is stored in the week bucket and the month bucket
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH}
+	case DURATION_KEY_YEAR:
+		// data stored in the last year (after the first month)
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR}
+	case DURATION_KEY_FOREVER:
+		//data stored before the last year
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR, DURATION_KEY_FOREVER}
+	}
+	return []string{DURATION_KEY_WEEK}
+}

webapp/backend/pkg/database/scrutiny_repository_device.go

@@ -0,0 +1,105 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"gorm.io/gorm/clause"
+	"time"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Device
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+//insert device into DB (and update specified columns if device is already registered)
+// update device fields that may change: (DeviceType, HostID)
+func (sr *scrutinyRepository) RegisterDevice(ctx context.Context, dev models.Device) error {
+	if err := sr.gormClient.WithContext(ctx).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "wwn"}},
+		DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type", "device_uuid", "device_serial_id", "device_label"}),
+	}).Create(&dev).Error; err != nil {
+		return err
+	}
+	return nil
+}
+
+// get a list of all devices (only device metadata, no SMART data)
+func (sr *scrutinyRepository) GetDevices(ctx context.Context) ([]models.Device, error) {
+	//Get a list of all the active devices.
+	devices := []models.Device{}
+	if err := sr.gormClient.WithContext(ctx).Find(&devices).Error; err != nil {
+		return nil, fmt.Errorf("Could not get device summary from DB: %v", err)
+	}
+	return devices, nil
+}
+
+// update device (only metadata) from collector
+func (sr *scrutinyRepository) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) {
+	var device models.Device
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return device, fmt.Errorf("Could not get device from DB: %v", err)
+	}
+
+	//TODO catch GormClient err
+	err := device.UpdateFromCollectorSmartInfo(collectorSmartData)
+	if err != nil {
+		return device, err
+	}
+	return device, sr.gormClient.Model(&device).Updates(device).Error
+}
+
+//Update Device Status
+func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) {
+	var device models.Device
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return device, fmt.Errorf("Could not get device from DB: %v", err)
+	}
+
+	device.DeviceStatus = pkg.DeviceStatusSet(device.DeviceStatus, status)
+	return device, sr.gormClient.Model(&device).Updates(device).Error
+}
+
+func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) {
+	var device models.Device
+
+	fmt.Println("GetDeviceDetails from GORM")
+
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return models.Device{}, err
+	}
+
+	return device, nil
+}
+
+func (sr *scrutinyRepository) DeleteDevice(ctx context.Context, wwn string) error {
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).Delete(&models.Device{}).Error; err != nil {
+		return err
+	}
+
+	//delete data from influxdb.
+	buckets := []string{
+		sr.appConfig.GetString("web.influxdb.bucket"),
+		fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket")),
+		fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket")),
+		fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket")),
+	}
+
+	for _, bucket := range buckets {
+		sr.logger.Infof("Deleting data for %s in bucket: %s", wwn, bucket)
+		if err := sr.influxClient.DeleteAPI().DeleteWithName(
+			ctx,
+			sr.appConfig.GetString("web.influxdb.org"),
+			bucket,
+			time.Now().AddDate(-10, 0, 0),
+			time.Now(),
+			fmt.Sprintf(`device_wwn="%s"`, wwn),
+		); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}