(v0.4.1) Automated packaging of release by Packagr

fixes #230

.dockerignore

@@ -0,0 +1,6 @@
+/dist
+/vendor
+/.idea
+/.github
+/.git
+/webapp/frontend/node_modules

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,6 +18,7 @@ If applicable, add screenshots to help explain your problem.
 
 **Log Files**
 If related to missing devices or SMART data, please run the `collector` in DEBUG mode, and attach the log file.
+See [/docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md) for other troubleshooting tips. 
 
 ```
 docker run -it --rm -p 8080:8080 \
@@ -29,7 +30,7 @@ docker run -it --rm -p 8080:8080 \
 -e COLLECTOR_LOG_FILE=/tmp/collector.log \
 -e SCRUTINY_LOG_FILE=/tmp/web.log \
 --name scrutiny \
-analogj/scrutiny
+ghcr.io/analogj/scrutiny:master-omnibus
 
 # in another terminal trigger the collector
 docker exec scrutiny scrutiny-collector-metrics run
@@ -37,5 +38,4 @@ docker exec scrutiny scrutiny-collector-metrics run
 # then use docker cp to copy the log files out of the container.
 docker cp scrutiny:/tmp/collector.log collector.log
 docker cp scrutiny:/tmp/web.log web.log
-
 ```

.github/workflows/build.yaml

@@ -6,7 +6,21 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container: karalabe/xgo-1.13.x
+    container: techknowlogick/xgo:go-1.13.x
+
+    # Service containers to run with `build` (Required for end-to-end testing)
+    services:
+      influxdb:
+        image: influxdb:2.2
+        env:
+          DOCKER_INFLUXDB_INIT_MODE: setup
+          DOCKER_INFLUXDB_INIT_USERNAME: admin
+          DOCKER_INFLUXDB_INIT_PASSWORD: password12345
+          DOCKER_INFLUXDB_INIT_ORG: scrutiny
+          DOCKER_INFLUXDB_INIT_BUCKET: metrics
+          DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token
+        ports:
+          - 8086:8086
     env:
       PROJECT_PATH: /go/src/github.com/analogj/scrutiny
       CGO_ENABLED: 1
@@ -26,6 +40,13 @@ jobs:
 
           go mod vendor
           go test -race -coverprofile=coverage.txt -covermode=atomic -v -tags "static" $(go list ./... | grep -v /vendor/)
+      - name: Generate coverage report
+        uses: codecov/codecov-action@v2
+        with:
+          files: ${{ env.PROJECT_PATH }}/coverage.txt
+          flags: unittests
+          fail_ci_if_error: true
+          verbose: true
       - name: Build Binaries
         run: |
 
@@ -49,10 +70,16 @@ jobs:
             /build/scrutiny-collector-metrics-linux-arm-7
             /build/scrutiny-web-windows-4.0-amd64.exe
             /build/scrutiny-collector-metrics-windows-4.0-amd64.exe
+#            /build/scrutiny-web-darwin-arm64
+#            /build/scrutiny-collector-metrics-darwin-arm64
+#            /build/scrutiny-web-darwin-amd64
+#            /build/scrutiny-collector-metrics-darwin-amd64
 #            /build/scrutiny-web-freebsd-amd64
 #            /build/scrutiny-collector-metrics-freebsd-amd64
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@v2
         with:
           file: ${{ env.PROJECT_PATH }}/coverage.txt
           flags: unittests
           fail_ci_if_error: false
+
+

.github/workflows/docker-build.yaml

@@ -0,0 +1,148 @@
+name: Docker
+on:
+  schedule:
+    - cron: '36 12 * * *'
+  push:
+    branches: [ master, influxdb ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  collector:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          tags: |
+            type=ref,enable=true,event=branch,suffix=-collector
+            type=ref,enable=true,event=tag,suffix=-collector
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          platforms: linux/amd64,linux/arm64
+          context: .
+          file: docker/Dockerfile.collector
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  web:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          tags: |
+            type=ref,enable=true,event=branch,suffix=-web
+            type=ref,enable=true,event=tag,suffix=-web
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          platforms: linux/amd64,linux/arm64
+          context: .
+          file: docker/Dockerfile.web
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  omnibus:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          tags: |
+            type=ref,enable=true,event=branch,suffix=-omnibus
+            type=ref,enable=true,event=tag,suffix=-omnibus
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          platforms: linux/amd64,linux/arm64
+          context: .
+          file: docker/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/release-freebsd.yaml

@@ -0,0 +1,77 @@
+# compiles FreeBSD artifacts and attaches them to build
+name: Release FreeBSD
+
+on:
+  release:
+    # Only use the types keyword to narrow down the activity types that will trigger your workflow.
+    types: [published]
+jobs:
+
+  release-freebsd:
+    name: Release FreeBSD
+    runs-on: macos-latest
+    env:
+      PROJECT_PATH: /go/src/github.com/analogj/scrutiny
+      GOPATH: /go
+      GOOS: freebsd
+      GOARCH: amd64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{github.event.release.tag_name}}
+      - name: Build Binaries
+        uses: vmactions/freebsd-vm@v0.1.3
+        with:
+          envs: 'PROJECT_PATH GOPATH GOOS GOARCH'
+          usesh: true
+          #TODO: lock go version using https://www.jeremymorgan.com/tutorials/golang/how-to-install-go-freebsd/
+          prepare: pkg install -y curl go gmake
+          run: |
+            pwd
+            ls -lah
+            whoami
+            freebsd-version
+
+            mkdir -p $(dirname "$PROJECT_PATH")
+            cp -R $GITHUB_WORKSPACE $PROJECT_PATH
+            cd $PROJECT_PATH
+
+            mkdir -p $GITHUB_WORKSPACE/dist
+
+            echo "building web binary (OS = ${GOOS}, ARCH = ${GOARCH})"
+            go build -ldflags "-extldflags=-static -X main.goos=${GOOS} -X main.goarch=${GOARCH}" -o $GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH} -tags "static netgo sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go
+
+            chmod +x "$GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH}"
+            file "$GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH}" || true
+            ldd "$GITHUB_WORKSPACE/dist/scrutiny-web-${GOOS}-${GOARCH}" || true
+
+            echo "building collector binary (OS = ${GOOS}, ARCH = ${GOARCH})"
+            go build -ldflags "-extldflags=-static -X main.goos=${GOOS} -X main.goarch=${GOARCH}" -o $GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH} -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go
+
+            chmod +x "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}"
+            file "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}" || true
+            ldd "$GITHUB_WORKSPACE/dist/scrutiny-collector-metrics-${GOOS}-${GOARCH}" || true
+
+
+      - name: Release Asset - Web - freebsd-amd64
+        id: upload-release-asset1
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: './dist/scrutiny-web-freebsd-amd64'
+          asset_name: scrutiny-web-freebsd-amd64
+          asset_content_type: application/octet-stream
+
+      - name: Release Asset - Collector - freebsd-amd64
+        id: upload-release-asset2
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: './dist/scrutiny-collector-metrics-freebsd-amd64'
+          asset_name: scrutiny-collector-metrics-freebsd-amd64
+          asset_content_type: application/octet-stream

.github/workflows/release.yaml

@@ -16,7 +16,20 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container: karalabe/xgo-1.13.x
+    container: techknowlogick/xgo:go-1.13.x
+    # Service containers to run with `build` (Required for end-to-end testing)
+    services:
+      influxdb:
+        image: influxdb:2.2
+        env:
+          DOCKER_INFLUXDB_INIT_MODE: setup
+          DOCKER_INFLUXDB_INIT_USERNAME: admin
+          DOCKER_INFLUXDB_INIT_PASSWORD: password12345
+          DOCKER_INFLUXDB_INIT_ORG: scrutiny
+          DOCKER_INFLUXDB_INIT_BUCKET: metrics
+          DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token
+        ports:
+          - 8086:8086
     env:
       PROJECT_PATH: /go/src/github.com/analogj/scrutiny
       CGO_ENABLED: 1
@@ -28,13 +41,16 @@ jobs:
           git --version
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Bump version
         id: bump_version
         uses: packagrio/action-bumpr-go@master
         with:
           version_bump_type: ${{ github.event.inputs.version_bump_type }}
           version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
-          github_token: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }} # Leave this line unchanged
       - name: Test
         run: |
           mkdir -p $(dirname "$PROJECT_PATH")
@@ -50,154 +66,38 @@ jobs:
           cd $PROJECT_PATH
           make all
 
-      - name: Commit
-        uses: EndBug/add-and-commit@v4 # You can change this to use a specific version
-        with:
+          # restore modified dir to GH workspace.
+          cp -arf $PROJECT_PATH/. $GITHUB_WORKSPACE/
 
-          author_name: Jason Kulatunga
-          author_email: jason@thesparktree.com
-          cwd: ${{ env.PROJECT_PATH }}
-          force: false
-          signoff: true
-          message: '(${{steps.bump_version.outputs.release_version}}) Automated packaging of release by Packagr'
-          tag: ${{steps.bump_version.outputs.release_version}}
+          # copy all the build artifacts to the GH workspace
+          cp -arf /build/. $GITHUB_WORKSPACE/
+
+      - name: Commit Changes
+        id: commit
+        uses: packagrio/action-releasr-go@master
         env:
+          # This is necessary in order to push a commit to the repo
           GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }} # Leave this line unchanged
-
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
         with:
-          tag_name: ${{ steps.bump_version.outputs.release_version }}
-          release_name: Release ${{ steps.bump_version.outputs.release_version }}
-          draft: false
-          prerelease: false
-
-      - name: Release Asset - Web - linux-amd64
-        id: upload-release-asset1
-        uses: actions/upload-release-asset@v1
+          version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
+      - name: Publish Release
+        id: publish
+        uses: packagrio/action-publishr-go@master
         env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+          # This is necessary in order to push a commit to the repo
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }} # Leave this line unchanged
         with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /build/scrutiny-web-linux-amd64
-          asset_name: scrutiny-web-linux-amd64
-          asset_content_type: application/octet-stream
-      - name: Release Asset - Collector - linux-amd64
-        id: upload-release-asset2
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /build/scrutiny-collector-metrics-linux-amd64
-          asset_name: scrutiny-collector-metrics-linux-amd64
-          asset_content_type: application/octet-stream
-
-
-      - name: Release Asset - Web - linux-arm64
-        id: upload-release-asset3
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /build/scrutiny-web-linux-arm64
-          asset_name: scrutiny-web-linux-arm64
-          asset_content_type: application/octet-stream
-      - name: Release Asset - Collector - linux-arm64
-        id: upload-release-asset4
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /build/scrutiny-collector-metrics-linux-arm64
-          asset_name: scrutiny-collector-metrics-linux-arm64
-          asset_content_type: application/octet-stream
-
-      - name: Release Asset - Web - linux-arm-5
-        id: upload-release-asset5
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /build/scrutiny-web-linux-arm-5
-          asset_name: scrutiny-web-linux-arm-5
-          asset_content_type: application/octet-stream
-      - name: Release Asset - Collector - linux-arm-5
-        id: upload-release-asset6
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /build/scrutiny-collector-metrics-linux-arm-5
-          asset_name: scrutiny-collector-metrics-linux-arm-5
-          asset_content_type: application/octet-stream
-
-      - name: Release Asset - Web - linux-arm-6
-        id: upload-release-asset7
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /build/scrutiny-web-linux-arm-6
-          asset_name: scrutiny-web-linux-arm-6
-          asset_content_type: application/octet-stream
-      - name: Release Asset - Collector - linux-arm-6
-        id: upload-release-asset8
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /buildd/scrutiny-collector-metrics-linux-arm-6
-          asset_name: scrutiny-collector-metrics-linux-arm-6
-          asset_content_type: application/octet-stream
-
-      - name: Release Asset - Web - linux-arm-7
-        id: upload-release-asset9
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /build/scrutiny-web-linux-arm-7
-          asset_name: scrutiny-web-linux-arm-7
-          asset_content_type: application/octet-stream
-      - name: Release Asset - Collector - linux-arm-7
-        id: upload-release-asset10
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /buildd/scrutiny-collector-metrics-linux-arm-7
-          asset_name: scrutiny-collector-metrics-linux-arm-7
-          asset_content_type: application/octet-stream
-
-#      - name: Release Asset - Web - freebsd-amd64
-#        id: upload-release-asset7
-#        uses: actions/upload-release-asset@v1
-#        env:
-#          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-#        with:
-#          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-#          asset_path: ${{ env.PROJECT_PATH }}/scrutiny-web-freebsd-amd64
-#          asset_name: scrutiny-web-freebsd-amd64
-#          asset_content_type: application/octet-stream
-#      - name: Release Asset - Collector - freebsd-amd64
-#        id: upload-release-asset8
-#        uses: actions/upload-release-asset@v1
-#        env:
-#          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
-#        with:
-#          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-#          asset_path: ${{ env.PROJECT_PATH }}/scrutiny-collector-metrics-freebsd-amd64
-#          asset_name: scrutiny-collector-metrics-freebsd-amd64
-#          asset_content_type: application/octet-stream
+          version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
+          upload_assets:
+            scrutiny-web-linux-amd64
+            scrutiny-collector-metrics-linux-amd64
+            scrutiny-web-linux-arm64
+            scrutiny-collector-metrics-linux-arm64
+            scrutiny-web-linux-arm-5
+            scrutiny-collector-metrics-linux-arm-5
+            scrutiny-web-linux-arm-6
+            scrutiny-collector-metrics-linux-arm-6
+            scrutiny-web-linux-arm-7
+            scrutiny-collector-metrics-linux-arm-7
+            scrutiny-web-windows-4.0-amd64.exe
+            scrutiny-collector-metrics-windows-4.0-amd64.exe

.github/workflows/sponsors.yaml

@@ -9,6 +9,6 @@ jobs:
     name: is-sponsor-label
     runs-on: ubuntu-latest
     steps:
-      - uses: JasonEtco/is-sponsor-label-action@v1
+      - uses: JasonEtco/is-sponsor-label-action@v1.2.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CONTRIBUTING.md

@@ -6,14 +6,14 @@ There are multiple ways to develop on the scrutiny codebase locally. The two mos
 
 ## Docker Development
 ```
-docker build -f docker/Dockerfile . -t analogj/scrutiny
+docker build -f docker/Dockerfile . -t chcr.io/analogj/scrutiny:master-omnibus
 docker run -it --rm -p 8080:8080 \
 -v /run/udev:/run/udev:ro \
 --cap-add SYS_RAWIO \
 --device=/dev/sda \
 --device=/dev/sdb \
-analogj/scrutiny
-/scrutiny/bin/scrutiny-collector-metrics run
+ghcr.io/analogj/scrutiny:master-omnibus
+/opt/scrutiny/bin/scrutiny-collector-metrics run
 ```
 
 
@@ -23,12 +23,14 @@ analogj/scrutiny
 The frontend is written in Angular.
 If you're working on the frontend and can use mocked data rather than a real backend, you can use
 ```
-cd webapp/frontend && ng serve
+cd webapp/frontend
+npm install
+ng serve --deploy-url="/web/" --base-href="/web/"
 ```
 
 However, if you need to also run the backend, and use real data, you'll need to run the following command:
 ```
-cd webapp/frontend && ng build --watch --output-path=../../dist --deploy-url="/web/" --base-href="/web/" --prod
+cd webapp/frontend && ng build --watch --output-path=../../dist --prod
 ```
 
 > Note: if you do not add `--prod` flag, app will display mocked data for api calls.
@@ -52,7 +54,8 @@ web:
   src:
     frontend:
       path: ./dist
-
+  influxdb:
+    retention_policy: false
 
 log:
   file: 'web.log' #absolute or relative paths allowed, eg. web.log
@@ -69,6 +72,39 @@ go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
 Now visit http://localhost:8080
 
 
+If you'd like to populate the database with some test data,  you can run the following commands:
+
+> NOTE: you may need to update the `local_time` key within the JSON file, any timestamps older than ~3 weeks will be automatically ignored
+> (since the downsampling & retention policy takes effect at 2 weeks)
+> This is done automatically by the `webapp/backend/pkg/models/testdata/helper.go` script
+
+```
+docker run -p 8086:8086 --rm influxdb:2.2
+
+
+docker run --rm -p 8086:8086 \
+      -e DOCKER_INFLUXDB_INIT_MODE=setup \
+      -e DOCKER_INFLUXDB_INIT_USERNAME=admin \
+      -e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
+      -e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
+      -e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
+      influxdb:2.2
+
+
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/web/testdata/register-devices-req.json localhost:8080/api/devices/register
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata-date.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata-date2.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-fail2.json localhost:8080/api/device/0x5000cca264ec3183/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-nvme.json localhost:8080/api/device/0x5002538e40a22954/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-scsi.json localhost:8080/api/device/0x5000cca252c859cc/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-scsi2.json localhost:8080/api/device/0x5000cca264ebc248/smart
+go run webapp/backend/pkg/models/testdata/helper.go
+
+curl localhost:8080/api/summary
+
+```
+
 ### Collector
 ```
 brew install smartmontools

Makefile

@@ -9,7 +9,6 @@ BINARY=\
   linux/arm-7   \
   linux/arm64   \
 
-
 .PHONY: all $(BINARY)
 all: $(BINARY) windows/amd64
 
@@ -39,46 +38,5 @@ windows/amd64:
 	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
 	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
 
-freebsd/amd64: export GOOS = freebsd
-freebsd/amd64: export GOARCH = amd64
-freebsd/amd64:
-	mkdir -p /build
-
-	@echo "building web binary (OS = $(GOOS), ARCH = $(GOARCH))"
-	go build -ldflags "-extldflags=-static -X main.goos=$(GOOS) -X main.goarch=$(GOARCH)" -o /build/scrutiny-web-$(GOOS)-$(GOARCH) -tags "static netgo sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go
-
-	chmod +x "/build/scrutiny-web-$(GOOS)-$(GOARCH)"
-	file "/build/scrutiny-web-$(GOOS)-$(GOARCH)" || true
-	ldd "/build/scrutiny-web-$(GOOS)-$(GOARCH)" || true
-
-	@echo "building collector binary (OS = $(GOOS), ARCH = $(GOARCH))"
-	go build -ldflags "-extldflags=-static -X main.goos=$(GOOS) -X main.goarch=$(GOARCH)" -o /build/scrutiny-collector-metrics-$(GOOS)-$(GOARCH) -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go
-
-	chmod +x "/build/scrutiny-collector-metrics-$(GOOS)-$(GOARCH)"
-	file "/build/scrutiny-collector-metrics-$(GOOS)-$(GOARCH)" || true
-	ldd "/build/scrutiny-collector-metrics-$(GOOS)-$(GOARCH)" || true
-
-freebsd/386: export GOOS = freebsd
-freebsd/386: export GOARCH = 386
-freebsd/386:
-	mkdir -p /build
-
-	@echo "building web binary (OS = $(GOOS), ARCH = $(GOARCH))"
-	go build -ldflags "-extldflags=-static -X main.goos=$(GOOS) -X main.goarch=$(GOARCH)" -o /build/scrutiny-web-$(GOOS)-$(GOARCH) -tags "static netgo sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go
-
-	chmod +x "/build/scrutiny-web-$(GOOS)-$(GOARCH)"
-	file "/build/scrutiny-web-$(GOOS)-$(GOARCH)" || true
-	ldd "/build/scrutiny-web-$(GOOS)-$(GOARCH)" || true
-
-	@echo "building collector binary (OS = $(GOOS), ARCH = $(GOARCH))"
-	go build -ldflags "-extldflags=-static -X main.goos=$(GOOS) -X main.goarch=$(GOARCH)" -o /build/scrutiny-collector-metrics-$(GOOS)-$(GOARCH) -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go
-
-	chmod +x "/build/scrutiny-collector-metrics-$(GOOS)-$(GOARCH)"
-	file "/build/scrutiny-collector-metrics-$(GOOS)-$(GOARCH)" || true
-	ldd "/build/scrutiny-collector-metrics-$(GOOS)-$(GOARCH)" || true
-
-
-
-
 # clean:
 # 	rm scrutiny-collector-metrics-* scrutiny-web-*

README.md

@@ -14,10 +14,11 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/AnalogJ/scrutiny?style=flat-square)](https://goreportcard.com/report/github.com/AnalogJ/scrutiny)
 [![GitHub release](http://img.shields.io/github/release/AnalogJ/scrutiny.svg?style=flat-square)](https://github.com/AnalogJ/scrutiny/releases)
 
-
 WebUI for smartd S.M.A.R.T monitoring
 
 > NOTE: Scrutiny is a Work-in-Progress and still has some rough edges.
+>
+> WARNING: Once the [InfluxDB](https://github.com/AnalogJ/scrutiny/tree/influxdb) branch is merged, Scrutiny will use both sqlite and InfluxDB for data storage. Unfortunately, this may not be backwards compatible with the database structures in the master (sqlite only) branch. 
 
 [![](docs/dashboard.png)](https://imgur.com/a/5k8qMzS)
 
@@ -71,40 +72,48 @@ If you're using Docker, getting started is as simple as running the following co
 
 ```bash
 docker run -it --rm -p 8080:8080 \
--v /run/udev:/run/udev:ro \
---cap-add SYS_RAWIO \
---device=/dev/sda \
---device=/dev/sdb \
---name scrutiny \
-analogj/scrutiny
+  -v `pwd`/scrutiny:/opt/scrutiny/config \
+  -v `pwd`/influxdb2:/opt/scrutiny/influxdb \
+  -v /run/udev:/run/udev:ro \
+  --cap-add SYS_RAWIO \
+  --device=/dev/sda \
+  --device=/dev/sdb \
+  --name scrutiny \
+  ghcr.io/analogj/scrutiny:master-omnibus
 ```
 
 - `/run/udev` is necessary to provide the Scrutiny collector with access to your device metadata
 - `--cap-add SYS_RAWIO` is necessary to allow `smartctl` permission to query your device SMART data
     - NOTE: If you have **NVMe** drives, you must add `--cap-add SYS_ADMIN` as well. See issue [#26](https://github.com/AnalogJ/scrutiny/issues/26#issuecomment-696817130)
 - `--device` entries are required to ensure that your hard disk devices are accessible within the container.
-- `analogj/scrutiny` is a omnibus image, containing both the webapp server (frontend & api) as well as the S.M.A.R.T metric collector. (see below)
+- `ghcr.io/analogj/scrutiny:master-omnibus` is a omnibus image, containing both the webapp server (frontend & api) as well as the S.M.A.R.T metric collector. (see below)
 
 ### Hub/Spoke Deployment
 
 In addition to the Omnibus image (available under the `latest` tag) there are 2 other Docker images available:
 
-- `analogj/scrutiny:collector` - Contains the Scrutiny data collector, `smartctl` binary and cron-like scheduler. You can run one collector on each server.
-- `analogj/scrutiny:web` - Contains the Web UI, API and Database. Only one container necessary
+- `ghcr.io/analogj/scrutiny:master-collector` - Contains the Scrutiny data collector, `smartctl` binary and cron-like scheduler. You can run one collector on each server.
+- `ghcr.io/analogj/scrutiny:master-web` - Contains the Web UI, API and Database. Only one container necessary
 
 ```bash
-docker run -it --rm -p 8080:8080 \
---name scrutiny-web \
-analogj/scrutiny:web
+docker run --rm -p 8086:8086 \
+  -v `pwd`/influxdb2:/var/lib/influxdb2 \
+  --name scrutiny-influxdb \
+  influxdb:2.2
 
-docker run -it --rm \
--v /run/udev:/run/udev:ro \
---cap-add SYS_RAWIO \
---device=/dev/sda \
---device=/dev/sdb \
--e SCRUTINY_API_ENDPOINT=http://SCRUTINY_WEB_IPADDRESS:8080 \
---name scrutiny-collector \
-analogj/scrutiny:collector
+docker run --rm -p 8080:8080 \
+  -v `pwd`/scrutiny:/opt/scrutiny/config \
+  --name scrutiny-web \
+  ghcr.io/analogj/scrutiny:master-web
+
+docker run --rm \
+  -v /run/udev:/run/udev:ro \
+  --cap-add SYS_RAWIO \
+  --device=/dev/sda \
+  --device=/dev/sdb \
+  -e SCRUTINY_API_ENDPOINT=http://SCRUTINY_WEB_IPADDRESS:8080 \
+  --name scrutiny-collector \
+  ghcr.io/analogj/scrutiny:master-collector
 ```
 
 ## Manual Installation (without-Docker)
@@ -125,12 +134,12 @@ drive that Scrutiny detected. The collector is configured to run once a day, but
 For users of the docker Hub/Spoke deployment or manual install: initially the dashboard will be empty.
 After the first collector run, you'll be greeted with a list of all your hard drives and their current smart status.
 
-```
-docker exec scrutiny /scrutiny/bin/scrutiny-collector-metrics run
+```bash
+docker exec scrutiny /opt/scrutiny/bin/scrutiny-collector-metrics run
 ```
 
 # Configuration
-By default Scrutiny looks for its YAML configuration files in `/scrutiny/config`
+By default Scrutiny looks for its YAML configuration files in `/opt/scrutiny/config`
 
 There are two configuration files available:
 
@@ -139,6 +148,13 @@ There are two configuration files available:
 
 Neither file is required, however if provided, it allows you to configure how Scrutiny functions.
 
+## Cron Schedule
+Unfortunately the Cron schedule cannot be configured via the `collector.yaml` (as the collector binary needs to be trigged by a scheduler/cron).
+However, if you are using the official `ghcr.io/analogj/scrutiny:master-collector` or `ghcr.io/analogj/scrutiny:master-omnibus` docker images, 
+you can use the `COLLECTOR_CRON_SCHEDULE` environmental variable to override the default cron schedule (daily @ midnight - `0 0 * * *`).
+
+`docker run -e COLLECTOR_CRON_SCHEDULE="0 0 * * *" ...`
+
 ## Notifications
 
 Scrutiny supports sending SMART device failure notifications via the following services:
@@ -164,7 +180,7 @@ Check the `notify.urls` section of [example.scrutiny.yml](example.scrutiny.yaml)
 
 You can test that your notifications are configured correctly by posting an empty payload to the notifications health check API.
 
-```
+```bash
 curl -X POST http://localhost:8080/api/health/notify
 ```
 
@@ -175,14 +191,14 @@ Scrutiny provides various methods to change the log level to debug and generate
 
 You can use environmental variables to enable debug logging and/or log files for the web server:
 
-```
+```bash
 DEBUG=true
 SCRUTINY_LOG_FILE=/tmp/web.log
 ```
 
 You can configure the log level and log file in the config file:
 
-```
+```yml
 log:
   file: '/tmp/web.log'
   level: DEBUG
@@ -190,7 +206,7 @@ log:
 
 Or if you're not using docker, you can pass CLI arguments to the web server during startup:
 
-```
+```bash
 scrutiny start --debug --log-file /tmp/web.log
 ```
 
@@ -198,14 +214,14 @@ scrutiny start --debug --log-file /tmp/web.log
 
 You can use environmental variables to enable debug logging and/or log files for the collector:
 
-```
+```bash
 DEBUG=true
 COLLECTOR_LOG_FILE=/tmp/collector.log
 ```
 
 Or if you're not using docker, you can pass CLI arguments to the collector during startup:
 
-```
+```bash
 scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
 ```
 

collector/cmd/collector-metrics/collector-metrics.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"log"
 	"os"
+	"strings"
 	"time"
 
 	utils "github.com/analogj/go-util/utils"
@@ -29,8 +30,8 @@ func main() {
 	}
 
 	//we're going to load the config file manually, since we need to validate it.
-	err = config.ReadConfig("/scrutiny/config/collector.yaml") // Find and read the config file
-	if _, ok := err.(errors.ConfigFileMissingError); ok {      // Handle errors reading the config file
+	err = config.ReadConfig("/opt/scrutiny/config/collector.yaml") // Find and read the config file
+	if _, ok := err.(errors.ConfigFileMissingError); ok {          // Handle errors reading the config file
 		//ignore "could not find config file"
 	} else if err != nil {
 		os.Exit(1)
@@ -99,24 +100,40 @@ OPTIONS:
 							return err
 						}
 					}
+					//override config with flags if set
 					if c.IsSet("host-id") {
 						config.Set("host.id", c.String("host-id")) // set/override the host-id using CLI.
 					}
 
+					if c.Bool("debug") {
+						config.Set("log.level", "DEBUG")
+					}
+
+					if c.IsSet("log-file") {
+						config.Set("log.file", c.String("log-file"))
+					}
+
+					if c.IsSet("api-endpoint") {
+						//if the user is providing an api-endpoint with a basepath (eg. http://localhost:8080/scrutiny),
+						//we need to ensure the basepath has a trailing slash, otherwise the url.Parse() path concatenation doesnt work.
+						apiEndpoint := strings.TrimSuffix(c.String("api-endpoint"), "/") + "/"
+						config.Set("api.endpoint", apiEndpoint)
+					}
+
 					collectorLogger := logrus.WithFields(logrus.Fields{
 						"type": "metrics",
 					})
 
-					if c.Bool("debug") {
-						logrus.SetLevel(logrus.DebugLevel)
+					if level, err := logrus.ParseLevel(config.GetString("log.level")); err == nil {
+						logrus.SetLevel(level)
 					} else {
 						logrus.SetLevel(logrus.InfoLevel)
 					}
 
-					if c.IsSet("log-file") {
-						logFile, err := os.OpenFile(c.String("log-file"), os.O_CREATE|os.O_WRONLY, 0644)
+					if config.IsSet("log.file") && len(config.GetString("log.file")) > 0 {
+						logFile, err := os.OpenFile(config.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
 						if err != nil {
-							logrus.Errorf("Failed to open log file %s for output: %s", c.String("log-file"), err)
+							logrus.Errorf("Failed to open log file %s for output: %s", config.GetString("log.file"), err)
 							return err
 						}
 						defer logFile.Close()
@@ -126,7 +143,7 @@ OPTIONS:
 					metricCollector, err := collector.CreateMetricsCollector(
 						config,
 						collectorLogger,
-						c.String("api-endpoint"),
+						config.GetString("api.endpoint"),
 					)
 
 					if err != nil {
@@ -144,14 +161,12 @@ OPTIONS:
 					&cli.StringFlag{
 						Name:    "api-endpoint",
 						Usage:   "The api server endpoint",
-						Value:   "http://localhost:8080",
 						EnvVars: []string{"SCRUTINY_API_ENDPOINT"},
 					},
 
 					&cli.StringFlag{
 						Name:    "log-file",
 						Usage:   "Path to file for logging. Leave empty to use STDOUT",
-						Value:   "",
 						EnvVars: []string{"COLLECTOR_LOG_FILE"},
 					},
 

collector/cmd/collector-selftest/collector-selftest.go

@@ -96,6 +96,7 @@ OPTIONS:
 						logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
 					}
 
+					//TODO: pass in the collector, use configuration from collector-metrics
 					stCollector, err := collector.CreateSelfTestCollector(
 						collectorLogger,
 						c.String("api-endpoint"),

collector/pkg/collector/base.go

@@ -40,6 +40,7 @@ func (c *BaseCollector) postJson(url string, body interface{}, target interface{
 	return json.NewDecoder(r.Body).Decode(target)
 }
 
+// http://www.linuxguide.it/command_line/linux-manpage/do.php?file=smartctl#sect7
 func (c *BaseCollector) LogSmartctlExitCode(exitCode int) {
 	if exitCode&0x01 != 0 {
 		c.logger.Errorln("smartctl could not parse commandline")

collector/pkg/collector/metrics.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
@@ -20,6 +20,7 @@ type MetricsCollector struct {
 	config config.Interface
 	BaseCollector
 	apiEndpoint *url.URL
+	shell       shell.Interface
 }
 
 func CreateMetricsCollector(appConfig config.Interface, logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
@@ -34,6 +35,7 @@ func CreateMetricsCollector(appConfig config.Interface, logger *logrus.Entry, ap
 		BaseCollector: BaseCollector{
 			logger: logger,
 		},
+		shell: shell.Create(),
 	}
 
 	return sc, nil
@@ -46,7 +48,7 @@ func (mc *MetricsCollector) Run() error {
 	}
 
 	apiEndpoint, _ := url.Parse(mc.apiEndpoint.String())
-	apiEndpoint.Path = "/api/devices/register"
+	apiEndpoint, _ = apiEndpoint.Parse("api/devices/register") //this acts like filepath.Join()
 
 	deviceRespWrapper := new(models.DeviceWrapper)
 
@@ -71,6 +73,7 @@ func (mc *MetricsCollector) Run() error {
 
 	if !deviceRespWrapper.Success {
 		mc.logger.Errorln("An error occurred while retrieving filtered devices")
+		mc.logger.Debugln(deviceRespWrapper)
 		return errors.ApiServerCommunicationError("An error occurred while retrieving filtered devices")
 	} else {
 		mc.logger.Debugln(deviceRespWrapper)
@@ -107,6 +110,10 @@ func (mc *MetricsCollector) Validate() error {
 //func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
 func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceType string) {
 	//defer wg.Done()
+	if len(deviceWWN) == 0 {
+		mc.logger.Errorf("no device WWN detected for %s. Skipping collection for this device (no data association possible).\n", deviceName)
+		return
+	}
 	mc.logger.Infof("Collecting smartctl results for %s\n", deviceName)
 
 	args := []string{"-x", "-j"}
@@ -116,7 +123,7 @@ func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceT
 	}
 	args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName))
 
-	result, err := common.ExecCmd(mc.logger, "smartctl", args, "", os.Environ())
+	result, err := mc.shell.Command(mc.logger, "smartctl", args, "", os.Environ())
 	resultBytes := []byte(result)
 	if err != nil {
 		if exitError, ok := err.(*exec.ExitError); ok {
@@ -140,7 +147,7 @@ func (mc *MetricsCollector) Publish(deviceWWN string, payload []byte) error {
 	mc.logger.Infof("Publishing smartctl results for %s\n", deviceWWN)
 
 	apiEndpoint, _ := url.Parse(mc.apiEndpoint.String())
-	apiEndpoint.Path = fmt.Sprintf("/api/device/%s/smart", strings.ToLower(deviceWWN))
+	apiEndpoint, _ = apiEndpoint.Parse(fmt.Sprintf("api/device/%s/smart", strings.ToLower(deviceWWN)))
 
 	resp, err := httpClient.Post(apiEndpoint.String(), "application/json", bytes.NewBuffer(payload))
 	if err != nil {

collector/pkg/collector/metrics_test.go

@@ -0,0 +1,38 @@
+package collector
+
+import (
+	"github.com/stretchr/testify/require"
+	"net/url"
+	"testing"
+)
+
+func TestApiEndpointParse(t *testing.T) {
+	baseURL, _ := url.Parse("http://localhost:8080/")
+
+	url1, _ := baseURL.Parse("d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url1.String())
+
+	url2, _ := baseURL.Parse("/d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url2.String())
+}
+
+func TestApiEndpointParse_WithBasepathWithoutTrailingSlash(t *testing.T) {
+	baseURL, _ := url.Parse("http://localhost:8080/scrutiny")
+
+	//This testcase is unexpected and can cause issues. We need to ensure the apiEndpoint always has a trailing slash.
+	url1, _ := baseURL.Parse("d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url1.String())
+
+	url2, _ := baseURL.Parse("/d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url2.String())
+}
+
+func TestApiEndpointParse_WithBasepathWithTrailingSlash(t *testing.T) {
+	baseURL, _ := url.Parse("http://localhost:8080/scrutiny/")
+
+	url1, _ := baseURL.Parse("d/e")
+	require.Equal(t, "http://localhost:8080/scrutiny/d/e", url1.String())
+
+	url2, _ := baseURL.Parse("/d/e")
+	require.Equal(t, "http://localhost:8080/d/e", url2.String())
+}

collector/pkg/common/shell/factory.go

@@ -0,0 +1,5 @@
+package shell
+
+func Create() Interface {
+	return new(localShell)
+}

collector/pkg/common/shell/interface.go

@@ -0,0 +1,11 @@
+package shell
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+// Create mock using:
+// mockgen -source=collector/pkg/common/shell/interface.go -destination=collector/pkg/common/shell/mock/mock_shell.go
+type Interface interface {
+	Command(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error)
+}

collector/pkg/common/shell/local_shell.go

@@ -1,4 +1,4 @@
-package common
+package shell
 
 import (
 	"bytes"
@@ -10,12 +10,22 @@ import (
 	"strings"
 )
 
-func ExecCmd(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
+type localShell struct{}
+
+func (s *localShell) Command(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
 	logger.Infof("Executing command: %s %s", cmdName, strings.Join(cmdArgs, " "))
 
 	cmd := exec.Command(cmdName, cmdArgs...)
 	var stdBuffer bytes.Buffer
-	mw := io.MultiWriter(logger.Logger.Out, &stdBuffer)
+
+	logWriters := []io.Writer{
+		&stdBuffer,
+	}
+	if logger.Logger.Level == logrus.DebugLevel {
+		logWriters = append(logWriters, logger.Logger.Out)
+	}
+
+	mw := io.MultiWriter(logWriters...)
 
 	cmd.Stdout = mw
 	cmd.Stderr = mw

collector/pkg/common/shell/local_shell_test.go

@@ -1,33 +1,33 @@
-package common_test
+package shell
 
 import (
-	"github.com/analogj/scrutiny/collector/pkg/common"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"os/exec"
 	"testing"
 )
 
-func TestExecCmd(t *testing.T) {
+func TestLocalShellCommand(t *testing.T) {
 	t.Parallel()
 
 	//setup
-
+	testShell := localShell{}
 	//test
-	result, err := common.ExecCmd(logrus.WithField("exec", "test"), "echo", []string{"hello world"}, "", nil)
+	result, err := testShell.Command(logrus.WithField("exec", "test"), "echo", []string{"hello world"}, "", nil)
 
 	//assert
 	require.NoError(t, err)
 	require.Equal(t, "hello world\n", result)
 }
 
-func TestExecCmd_Date(t *testing.T) {
+func TestLocalShellCommand_Date(t *testing.T) {
 	t.Parallel()
 
 	//setup
+	testShell := localShell{}
 
 	//test
-	_, err := common.ExecCmd(logrus.WithField("exec", "test"), "date", []string{}, "", nil)
+	_, err := testShell.Command(logrus.WithField("exec", "test"), "date", []string{}, "", nil)
 
 	//assert
 	require.NoError(t, err)
@@ -51,13 +51,14 @@ func TestExecCmd_Date(t *testing.T) {
 //}
 //
 
-func TestExecCmd_InvalidCommand(t *testing.T) {
+func TestLocalShellCommand_InvalidCommand(t *testing.T) {
 	t.Parallel()
 
 	//setup
+	testShell := localShell{}
 
 	//test
-	_, err := common.ExecCmd(logrus.WithField("exec", "test"), "invalid_binary", []string{}, "", nil)
+	_, err := testShell.Command(logrus.WithField("exec", "test"), "invalid_binary", []string{}, "", nil)
 
 	//assert
 	_, castOk := err.(*exec.ExitError)

collector/pkg/common/shell/mock/mock_shell.go

@@ -0,0 +1,50 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: collector/pkg/common/shell/interface.go
+
+// Package mock_shell is a generated GoMock package.
+package mock_shell
+
+import (
+	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
+	logrus "github.com/sirupsen/logrus"
+)
+
+// MockInterface is a mock of Interface interface.
+type MockInterface struct {
+	ctrl     *gomock.Controller
+	recorder *MockInterfaceMockRecorder
+}
+
+// MockInterfaceMockRecorder is the mock recorder for MockInterface.
+type MockInterfaceMockRecorder struct {
+	mock *MockInterface
+}
+
+// NewMockInterface creates a new mock instance.
+func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
+	mock := &MockInterface{ctrl: ctrl}
+	mock.recorder = &MockInterfaceMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
+	return m.recorder
+}
+
+// Command mocks base method.
+func (m *MockInterface) Command(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Command", logger, cmdName, cmdArgs, workingDir, environ)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Command indicates an expected call of Command.
+func (mr *MockInterfaceMockRecorder) Command(logger, cmdName, cmdArgs, workingDir, environ interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Command", reflect.TypeOf((*MockInterface)(nil).Command), logger, cmdName, cmdArgs, workingDir, environ)
+}

collector/pkg/config/config.go

@@ -33,6 +33,11 @@ func (c *configuration) Init() error {
 
 	c.SetDefault("devices", []string{})
 
+	c.SetDefault("log.level", "INFO")
+	c.SetDefault("log.file", "")
+
+	c.SetDefault("api.endpoint", "http://localhost:8080")
+
 	//c.SetDefault("collect.short.command", "-a -o on -S on")
 
 	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig

collector/pkg/config/config_test.go

@@ -8,6 +8,19 @@ import (
 	"testing"
 )
 
+func TestConfiguration_InvalidConfigPath(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig("does_not_exist.yaml")
+
+	//assert
+	require.Error(t, err, "should return an error")
+}
+
 func TestConfiguration_GetScanOverrides_Simple(t *testing.T) {
 	t.Parallel()
 

collector/pkg/detect/detect.go

@@ -3,7 +3,7 @@ package detect
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
@@ -15,6 +15,7 @@ import (
 type Detect struct {
 	Logger *logrus.Entry
 	Config config.Interface
+	Shell  shell.Interface
 }
 
 //private/common functions
@@ -27,7 +28,7 @@ type Detect struct {
 // models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
 func (d *Detect) SmartctlScan() ([]models.Device, error) {
 	//we use smartctl to detect all the drives available.
-	detectedDeviceConnJson, err := common.ExecCmd(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
+	detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
 	if err != nil {
 		d.Logger.Errorf("Error scanning for devices: %v", err)
 		return nil, err
@@ -58,7 +59,7 @@ func (d *Detect) SmartCtlInfo(device *models.Device) error {
 	}
 	args = append(args, fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName))
 
-	availableDeviceInfoJson, err := common.ExecCmd(d.Logger, "smartctl", args, "", os.Environ())
+	availableDeviceInfoJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ())
 	if err != nil {
 		d.Logger.Errorf("Could not retrieve device information for %s: %v", device.DeviceName, err)
 		return err
@@ -100,6 +101,12 @@ func (d *Detect) SmartCtlInfo(device *models.Device) error {
 		d.Logger.Info("Using WWN Fallback")
 		d.wwnFallback(device)
 	}
+	if len(device.WWN) == 0 {
+		// no WWN populated after WWN lookup and fallback. we need to throw an error
+		errMsg := fmt.Sprintf("no WWN (or fallback) populated for device: %s. Device will be registered, but no data will be published for this device. ", device.DeviceName)
+		d.Logger.Errorf(errMsg)
+		return fmt.Errorf(errMsg)
+	}
 
 	return nil
 }

collector/pkg/detect/detect_test.go

@@ -1,14 +1,103 @@
 package detect_test
 
 import (
+	mock_shell "github.com/analogj/scrutiny/collector/pkg/common/shell/mock"
 	mock_config "github.com/analogj/scrutiny/collector/pkg/config/mock"
 	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/golang/mock/gomock"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
+	"io/ioutil"
 	"testing"
 )
 
+func TestDetect_SmartctlScan(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+
+	fakeShell := mock_shell.NewMockInterface(mockCtrl)
+	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_simple.json")
+	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
+
+	d := detect.Detect{
+		Logger: logrus.WithFields(logrus.Fields{}),
+		Shell:  fakeShell,
+		Config: fakeConfig,
+	}
+
+	//test
+	scannedDevices, err := d.SmartctlScan()
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, 7, len(scannedDevices))
+	require.Equal(t, "scsi", scannedDevices[0].DeviceType)
+}
+
+func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+
+	fakeShell := mock_shell.NewMockInterface(mockCtrl)
+	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_megaraid.json")
+	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
+
+	d := detect.Detect{
+		Logger: logrus.WithFields(logrus.Fields{}),
+		Shell:  fakeShell,
+		Config: fakeConfig,
+	}
+
+	//test
+	scannedDevices, err := d.SmartctlScan()
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, 2, len(scannedDevices))
+	require.Equal(t, []models.Device{
+		models.Device{DeviceName: "bus/0", DeviceType: "megaraid,0"},
+		models.Device{DeviceName: "bus/0", DeviceType: "megaraid,1"},
+	}, scannedDevices)
+}
+
+func TestDetect_SmartctlScan_Nvme(t *testing.T) {
+	//setup
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
+	fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
+
+	fakeShell := mock_shell.NewMockInterface(mockCtrl)
+	testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_nvme.json")
+	fakeShell.EXPECT().Command(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(string(testScanResults), err)
+
+	d := detect.Detect{
+		Logger: logrus.WithFields(logrus.Fields{}),
+		Shell:  fakeShell,
+		Config: fakeConfig,
+	}
+
+	//test
+	scannedDevices, err := d.SmartctlScan()
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, 1, len(scannedDevices))
+	require.Equal(t, []models.Device{
+		models.Device{DeviceName: "nvme0", DeviceType: "nvme"},
+	}, scannedDevices)
+}
+
 func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
 	//setup
 	mockCtrl := gomock.NewController(t)

collector/pkg/detect/devices_darwin.go

@@ -1,6 +1,7 @@
 package detect
 
 import (
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/jaypipes/ghw"
 	"strings"
@@ -11,6 +12,7 @@ func DevicePrefix() string {
 }
 
 func (d *Detect) Start() ([]models.Device, error) {
+	d.Shell = shell.Create()
 	// call the base/common functionality to get a list of devicess
 	detectedDevices, err := d.SmartctlScan()
 	if err != nil {

collector/pkg/detect/devices_freebsd.go

@@ -1,6 +1,7 @@
 package detect
 
 import (
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/jaypipes/ghw"
 	"strings"
@@ -11,6 +12,7 @@ func DevicePrefix() string {
 }
 
 func (d *Detect) Start() ([]models.Device, error) {
+	d.Shell = shell.Create()
 	// call the base/common functionality to get a list of devices
 	detectedDevices, err := d.SmartctlScan()
 	if err != nil {

collector/pkg/detect/devices_linux.go

@@ -1,6 +1,7 @@
 package detect
 
 import (
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/jaypipes/ghw"
 	"strings"
@@ -11,6 +12,7 @@ func DevicePrefix() string {
 }
 
 func (d *Detect) Start() ([]models.Device, error) {
+	d.Shell = shell.Create()
 	// call the base/common functionality to get a list of devices
 	detectedDevices, err := d.SmartctlScan()
 	if err != nil {

collector/pkg/detect/devices_windows.go

@@ -1,6 +1,7 @@
 package detect
 
 import (
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"strings"
 )
@@ -10,6 +11,7 @@ func DevicePrefix() string {
 }
 
 func (d *Detect) Start() ([]models.Device, error) {
+	d.Shell = shell.Create()
 	// call the base/common functionality to get a list of devices
 	detectedDevices, err := d.SmartctlScan()
 	if err != nil {

collector/pkg/detect/testdata/smartctl_scan_megaraid.json

@@ -0,0 +1,35 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      1
+    ],
+    "svn_revision": "5022",
+    "platform_info": "x86_64-linux-5.4.0-45-generic",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "--scan"
+    ],
+    "exit_status": 0
+  },
+  "devices": [
+    {
+      "name": "/dev/bus/0",
+      "info_name": "/dev/bus/0 [megaraid_disk_00]",
+      "type": "megaraid,0",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/bus/0",
+      "info_name": "/dev/bus/0 [megaraid_disk_01]",
+      "type": "megaraid,1",
+      "protocol": "SCSI"
+    }
+  ]
+}

collector/pkg/detect/testdata/smartctl_scan_nvme.json

@@ -0,0 +1,29 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-4.19.107-Unraid",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "--scan"
+    ],
+    "exit_status": 0
+  },
+  "devices": [
+    {
+      "name": "/dev/nvme0",
+      "info_name": "/dev/nvme0",
+      "type": "nvme",
+      "protocol": "NVMe"
+    }
+  ]
+}

collector/pkg/detect/testdata/smartctl_scan_simple.json

@@ -0,0 +1,65 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-5.15.32-flatcar",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "--scan",
+      "-j"
+    ],
+    "exit_status": 0
+  },
+  "devices": [
+    {
+      "name": "/dev/sda",
+      "info_name": "/dev/sda",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdb",
+      "info_name": "/dev/sdb",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdc",
+      "info_name": "/dev/sdc",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdd",
+      "info_name": "/dev/sdd",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sde",
+      "info_name": "/dev/sde",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdf",
+      "info_name": "/dev/sdf",
+      "type": "scsi",
+      "protocol": "SCSI"
+    },
+    {
+      "name": "/dev/sdg",
+      "info_name": "/dev/sdg",
+      "type": "scsi",
+      "protocol": "SCSI"
+    }
+  ]
+}

docker/Dockerfile

@@ -16,42 +16,50 @@ FROM node:lts-slim as frontendbuild
 #reduce logging, disable angular-cli analytics for ci environment
 ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
 
-WORKDIR /scrutiny/src
-COPY webapp/frontend /scrutiny/src
+WORKDIR /opt/scrutiny/src
+COPY webapp/frontend /opt/scrutiny/src
 
 RUN npm install -g @angular/cli@9.1.4 && \
     mkdir -p /scrutiny/dist && \
     npm install && \
-    ng build --output-path=/scrutiny/dist --deploy-url="/web/" --base-href="/web/" --prod
+    ng build --output-path=/opt/scrutiny/dist --prod
 
 
 ########
 FROM ubuntu:bionic as runtime
+ARG TARGETARCH
 EXPOSE 8080
-WORKDIR /scrutiny
-ENV PATH="/scrutiny/bin:${PATH}"
+WORKDIR /opt/scrutiny
+ENV PATH="/opt/scrutiny/bin:${PATH}"
+ENV INFLUXD_CONFIG_PATH=/opt/scrutiny/influxdb
 
-ADD https://github.com/dshearer/jobber/releases/download/v1.4.4/jobber_1.4.4-1_amd64.deb /tmp/
-RUN apt install /tmp/jobber_1.4.4-1_amd64.deb
+RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates curl tzdata \
+    && update-ca-certificates \
+    &&  case ${TARGETARCH} in \
+            "amd64")  S6_ARCH=amd64  ;; \
+            "arm64")  S6_ARCH=aarch64  ;; \
+        esac \
+    && curl https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-${S6_ARCH}.tar.gz -L -s --output /tmp/s6-overlay-${S6_ARCH}.tar.gz \
+    && tar xzf /tmp/s6-overlay-${S6_ARCH}.tar.gz -C /
 
-RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates curl && update-ca-certificates
+ADD https://dl.influxdata.com/influxdb/releases/influxdb2-2.2.0-${TARGETARCH}.deb /tmp/
+RUN dpkg -i /tmp/influxdb2-2.2.0-${TARGETARCH}.deb && rm -rf /tmp/influxdb2-2.2.0-${TARGETARCH}.deb
 
-ADD https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-amd64.tar.gz /tmp/
-RUN tar xzf /tmp/s6-overlay-amd64.tar.gz -C /
 COPY /rootfs /
 
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /scrutiny/bin/
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /scrutiny/bin/
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /scrutiny/bin/
-COPY --from=frontendbuild /scrutiny/dist /scrutiny/web
-RUN chmod +x /scrutiny/bin/scrutiny && \
-    chmod +x /scrutiny/bin/scrutiny-collector-selftest && \
-    chmod +x /scrutiny/bin/scrutiny-collector-metrics && \
-    mkdir -p /scrutiny/web && \
-    mkdir -p /scrutiny/config && \
-    mkdir -p /scrutiny/jobber && \
-    chmod -R ugo+rwx /scrutiny/config && \
-    chmod -R ugo+rwx /scrutiny/jobber
+COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /opt/scrutiny/bin/
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
+COPY --from=frontendbuild /opt/scrutiny/dist /opt/scrutiny/web
+RUN chmod +x /opt/scrutiny/bin/scrutiny && \
+    chmod +x /opt/scrutiny/bin/scrutiny-collector-selftest && \
+    chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \
+    chmod 0644 /etc/cron.d/scrutiny && \
+    rm -f /etc/cron.daily/* && \
+    mkdir -p /opt/scrutiny/web && \
+    mkdir -p /opt/scrutiny/config && \
+    chmod -R ugo+rwx /opt/scrutiny/config
 
 
 CMD ["/init"]

docker/Dockerfile.collector

@@ -11,20 +11,19 @@ RUN go mod vendor && \
 
 ########
 FROM ubuntu:bionic as runtime
-EXPOSE 8080
 WORKDIR /scrutiny
-ENV PATH="/scrutiny/bin:${PATH}"
+ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-ADD https://github.com/dshearer/jobber/releases/download/v1.4.4/jobber_1.4.4-1_amd64.deb /tmp/
-RUN apt install /tmp/jobber_1.4.4-1_amd64.deb
+RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates tzdata && update-ca-certificates
 
-RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates && update-ca-certificates
+COPY /docker/entrypoint-collector.sh /entrypoint-collector.sh
+COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /opt/scrutiny/bin/
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/
+RUN chmod +x /opt/scrutiny/bin/scrutiny-collector-selftest && \
+    chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics && \
+    chmod +x /entrypoint-collector.sh && \
+    chmod 0644 /etc/cron.d/scrutiny && \
+    rm -f /etc/cron.daily/apt /etc/cron.daily/dpkg /etc/cron.daily/passwd
 
-COPY /rootfs/scrutiny /scrutiny
-
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /scrutiny/bin/
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /scrutiny/bin/
-RUN chmod +x /scrutiny/bin/scrutiny-collector-selftest && \
-    chmod +x /scrutiny/bin/scrutiny-collector-metrics
-
-CMD ["/usr/lib/x86_64-linux-gnu/jobberrunner", "/scrutiny/jobber/jobber.yaml"]
+CMD ["/entrypoint-collector.sh"]

docker/Dockerfile.web

@@ -14,27 +14,27 @@ FROM node:lts-slim as frontendbuild
 #reduce logging, disable angular-cli analytics for ci environment
 ENV NPM_CONFIG_LOGLEVEL=warn NG_CLI_ANALYTICS=false
 
-WORKDIR /scrutiny/src
-COPY webapp/frontend /scrutiny/src
+WORKDIR /opt/scrutiny/src
+COPY webapp/frontend /opt/scrutiny/src
 
 RUN npm install -g @angular/cli@9.1.4 && \
-    mkdir -p /scrutiny/dist && \
+    mkdir -p /opt/scrutiny/dist && \
     npm install && \
-    ng build --output-path=/scrutiny/dist --deploy-url="/web/" --base-href="/web/" --prod
+    ng build --output-path=/opt/scrutiny/dist --prod
 
 
 ########
 FROM ubuntu:bionic as runtime
 EXPOSE 8080
-WORKDIR /scrutiny
-ENV PATH="/scrutiny/bin:${PATH}"
+WORKDIR /opt/scrutiny
+ENV PATH="/opt/scrutiny/bin:${PATH}"
 
-RUN apt-get update && apt-get install -y ca-certificates curl && update-ca-certificates
+RUN apt-get update && apt-get install -y ca-certificates curl tzdata && update-ca-certificates
 
-COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /scrutiny/bin/
-COPY --from=frontendbuild /scrutiny/dist /scrutiny/web
-RUN chmod +x /scrutiny/bin/scrutiny && \
-    mkdir -p /scrutiny/web && \
-    mkdir -p /scrutiny/config && \
-    chmod -R ugo+rwx /scrutiny/config
-CMD ["/scrutiny/bin/scrutiny", "start"]
+COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /opt/scrutiny/bin/
+COPY --from=frontendbuild /opt/scrutiny/dist /opt/scrutiny/web
+RUN chmod +x /opt/scrutiny/bin/scrutiny && \
+    mkdir -p /opt/scrutiny/web && \
+    mkdir -p /opt/scrutiny/config && \
+    chmod -R ugo+rwx /opt/scrutiny/config
+CMD ["/opt/scrutiny/bin/scrutiny", "start"]

docker/Dockerfile.xgo

@@ -1,4 +1,4 @@
-FROM karalabe/xgo-1.13.x
+FROM techknowlogick/xgo:go-1.13.x
 
 WORKDIR /go/src/github.com/analogj/scrutiny
 

docker/README.md

@@ -0,0 +1 @@
+`rootfs`  is only used by Dockerfile and Dockerfile.collector

docker/Vagrantfile

@@ -1,3 +1,5 @@
+# This vagrant file is only used for local development & testing.
+
 Vagrant.configure("2") do |config|
   config.vm.guest = :freebsd
   config.vm.synced_folder ".", "/vagrant", id: "vagrant-root", disabled: true

docker/docker-compose.yml

@@ -3,14 +3,15 @@ version: '3.5'
 services:
   scrutiny:
     container_name: scrutiny
-    image: analogj/scrutiny
+    image: ghcr.io/analogj/scrutiny:master-omnibus
     cap_add:
       - SYS_RAWIO
     ports:
       - "8080:8080"
     volumes:
       - /run/udev:/run/udev:ro
-      - ./config:/scrutiny/config
+      - ./config:/opt/scrutiny/config
+      - ./influxdb:/var/lib/influxdb2
     devices:
       - "/dev/sda"
       - "/dev/sdb"

docker/entrypoint-collector.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Cron runs in its own isolated environment (usually using only /etc/environment )
+# So when the container starts up, we will do a dump of the runtime environment into a .env file that we
+# will then source into the crontab file (/etc/cron.d/scrutiny.sh)
+
+printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
+
+# adding ability to customize the cron schedule.
+COLLECTOR_CRON_SCHEDULE=${COLLECTOR_CRON_SCHEDULE:-"0 0 * * *"}
+sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron.d/scrutiny
+
+# now that we have the env start cron in the foreground
+echo "starting cron"
+su -c "cron -l 8 -f" root

docs/DOWNSAMPLING.md

@@ -0,0 +1,43 @@
+# Downsampling
+
+Scrutiny collects alot of data, that can cause the database to grow unbounded. 
+
+- Smart data
+- Smart test data
+- Temperature data
+- Disk metrics (capacity/usage)
+- etc
+
+This data must be accurate in the short term, and is useful for doing trend analysis in the long term.
+However, for trend analysis we only need aggregate data, individual data points are not as useful.
+
+Scrutiny will automatically downsample data on a schedule to ensure that the database size stays reasonable, while still
+ensuring historical data is present for comparisons.
+
+
+| Bucket Name | Retention Period | Downsampling Range | Downsampling Aggregation Window | Downsampling Cron | Comments |
+| --- | --- | --- | --- | --- | --- |
+| `metrics` | 15 days | `-2w -1w` | `1w` | main bucket, weekly on Sunday at 1:00am |
+| `metrics_weekly` | 9 weeks | `-2mo -1mo` | `1mo` | monthly on first day of the month at 1:30am
+| `metrics_monthly` | 25 months | `-2y -1y` | `1y` | yearly on the first day of the year at 2:00am
+| `metrics_yearly` | forever | - | - | - | |
+
+
+After 5 months, here's how may data points should exist in each bucket for one disk
+
+| Bucket Name | Datapoints | Comments |
+| --- | --- | --- |
+| `metrics` | 15 | 7 daily datapoints , up to 7 pending data, 1 buffer data point |
+| `metrics_weekly` | 9 | 4 aggregated weekly data points, 4 pending datapoints, 1 buffer data point |
+| `metrics_monthly` | 3 | 3 aggregated monthly data points | 
+| `metrics_yearly` | 0 | |
+
+After 5 years, here's how may data points should exist in each bucket for one disk
+
+| Bucket Name | Datapoints | Comments |
+| --- | --- | --- |
+| `metrics` | - | - |
+| `metrics_weekly` | - | 
+| `metrics_monthly` | - |
+| `metrics_yearly` | - |
+

docs/INSTALL_MANUAL.md

@@ -132,6 +132,8 @@ chmod +x /opt/scrutiny/bin/scrutiny-collector-metrics-linux-amd64
 
 Next, we will manually trigger the collector, to populate the Scrutiny dashboard:
 
+> NOTE: if you need to pass a config file to the scrutiny collector, you can provide it using the `--config` flag.
+
 ```
 /opt/scrutiny/bin/scrutiny-collector-metrics-linux-amd64 run --api-endpoint "http://localhost:8080"
 ```

docs/INSTALL_UNRAID.md

@@ -19,13 +19,13 @@ To install, simply click 'Install'; the configuration parameters should not need
 
 As a docker image can be created using various OS bases, the image choice is entirely the users choice. Recommendations of a specific image from a specific maintainer is beyond the scope of this guide. However, to provide some context given the number of questions posed regarding the various versions available:
 
-- **analogj/scrutiny**
+- **ghcr.io/analogj/scrutiny:master-omnibus**
     - `Image maintained directly by the application author`
     - `Debian based docker image`
-- **linuxserver/scrutiny:**
+- **linuxserver/scrutiny**
     - `Image maintained by the LinuxServer.io group`
     - `Alpine based docker image`
-- **hotio/scrutiny:**
+- **hotio/scrutiny**
     - `Image maintained by hotio`
     - `DETAILS TBD`
 

docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md

@@ -0,0 +1,127 @@
+# Scrutiny <-> SmartMonTools 
+
+Scrutiny uses `smartctl --scan` to detect devices/drives. If your devices are not being detected by Scrutiny, or some 
+data is missing, this is probably due to a `smartctl` issue. 
+The following page will document commonly asked questions and troubleshooting steps for the Scrutiny S.M.A.R.T. data collector.
+
+## WWN vs Device name
+As discussed in [`#117`](https://github.com/AnalogJ/scrutiny/issues/117), `/dev/sd*` device paths are ephemeral. 
+
+> Device paths in Linux aren't guaranteed to be consistent across restarts. Device names consist of major numbers (letters) and minor numbers. When the Linux storage device driver detects a new device, the driver assigns major and minor numbers from the available range to the device. When a device is removed, the device numbers are freed for reuse.
+>
+> The problem occurs because device scanning in Linux is scheduled by the SCSI subsystem to happen asynchronously. As a result, a device path name can vary across restarts.
+>
+> https://docs.microsoft.com/en-us/troubleshoot/azure/virtual-machines/troubleshoot-device-names-problems
+
+While the Docker Scrutiny collector does require devices to attached to the docker container by device name (using `--device=/dev/sd..`), internally 
+Scrutiny stores and references the devices by their `WWN` which is globally unique, and never changes. 
+
+As such, passing devices to the Scrutiny collector container using `/dev/disk/by-id/`, `/dev/disk/by-label/`, `/dev/disk/by-path/` and `/dev/disk/by-uuid/`
+paths are unnecessary, unless you'd like to ensure the docker run command never needs to change.
+
+
+## Device Detection By Smartctl
+
+The first thing you'll want to do is run `smartctl` locally (not in Docker) and make sure the output shows all your drives as expected.
+See the `Drive Types` section below for what this output should look like for `NVMe`/`ATA`/`RAID` drives.
+
+```bash
+smartctl --scan
+
+/dev/sda -d scsi # /dev/sda, SCSI device
+/dev/sdb -d scsi # /dev/sdb, SCSI device
+/dev/sdc -d scsi # /dev/sdc, SCSI device
+/dev/sdd -d scsi # /dev/sdd, SCSI device
+```
+
+Once you've verified that `smartctl` correctly detects your drives, make sure scrutiny is correctly detecting them as well.
+> NOTE: make sure you specify all the devices you'd like scrutiny to process using `--device=` flags.
+
+```bash
+docker run -it --rm \
+  -v /run/udev:/run/udev:ro \
+  --cap-add SYS_RAWIO \
+  --device=/dev/sda \
+  --device=/dev/sdb \
+  ghcr.io/analogj/scrutiny:master-collector smartctl --scan
+```
+
+If the output is the same, your devices will be processed by Scrutiny.
+
+### Collector Config File
+In some cases `--scan` does not correctly detect the device type, returning [incomplete SMART data](https://github.com/AnalogJ/scrutiny/issues/45).
+Scrutiny will supports overriding the detected device type via the config file.
+
+### RAID Controllers (Megaraid/3ware/HBA/Adaptec/HPE/etc)
+Smartctl has support for a large number of [RAID controllers](https://www.smartmontools.org/wiki/Supported_RAID-Controllers), however this 
+support is not automatic, and may require some additional device type hinting. You can provide this information to the Scrutiny collector
+using a collector config file. See [example.collector.yaml](/example.collector.yaml)
+
+> NOTE: If you use docker, you **must** pass though the RAID virtual disk to the container using `--device` (see below)
+>
+> This device may be in `/dev/*` or `/dev/bus/*`. 
+>
+> If you're unsure, run `smartctl --scan` on your host, and pass all listed devices to the container.
+
+```yaml
+# /opt/scrutiny/config/collector.yaml
+devices:
+  # Dell PERC/Broadcom Megaraid example: https://github.com/AnalogJ/scrutiny/issues/30
+  - device: /dev/bus/0
+    type:
+      - megaraid,14
+      - megaraid,15
+      - megaraid,18
+      - megaraid,19
+      - megaraid,20
+      - megaraid,21
+
+  - device: /dev/twa0
+    type:
+      - 3ware,0
+      - 3ware,1
+      - 3ware,2
+      - 3ware,3
+      - 3ware,4
+      - 3ware,5
+  
+  # Adapec RAID: https://github.com/AnalogJ/scrutiny/issues/189
+  - device: /dev/sdb
+    type:
+      - aacraid,0,0,0
+      - aacraid,0,0,1
+  
+  # HPE Smart Array example:  https://github.com/AnalogJ/scrutiny/issues/213
+  - device: /dev/sda
+    type:
+      - 'cciss,0'
+      - 'cciss,1'
+```
+
+### NVMe Drives
+As mentioned in the [README.md](/README.md), NVMe devices require both `--cap-add SYS_RAWIO` and `--cap-add SYS_ADMIN` 
+to allow smartctl permission to query your NVMe device SMART data [#26](https://github.com/AnalogJ/scrutiny/issues/26)
+
+When attaching NVMe devices using `--device=/dev/nvme..`, make sure to provide the device controller (`/dev/nvme0`) 
+instead of the block device (`/dev/nvme0n1`). See [#209](https://github.com/AnalogJ/scrutiny/issues/209).
+
+> The character device /dev/nvme0 is the NVME device controller, and block devices like /dev/nvme0n1 are the NVME storage namespaces: the devices you use for actual storage, which will behave essentially as disks.
+>
+> In enterprise-grade hardware, there might be support for several namespaces, thin provisioning within namespaces and other features. For now, you could think namespaces as sort of meta-partitions with extra features for enterprise use.
+
+### ATA
+
+### Standby/Sleeping Disks
+- https://github.com/AnalogJ/scrutiny/issues/221
+- https://github.com/AnalogJ/scrutiny/issues/157
+
+### Volume Mount All Devices (`/dev`) - Privileged
+
+
+## Hub & Spoke model, with multiple Hosts.
+
+When deploying Scrutiny in a hub & spoke model, it can be difficult to determine exactly which node a set of devices are associated with.
+Thankfully the collector has a special `--host-id` flag (or `COLLECTOR_HOST_ID` env variable) that can be used to associate devices with a friendly host name.
+
+See the [docs/INSTALL_HUB_SPOKE.md](/docs/INSTALL_HUB_SPOKE.md) guide for more information. 
+

docs/dbdiagram.io.txt

@@ -0,0 +1,62 @@
+
+// SQLite Table(s)
+Table device {
+  created_at timestamp
+
+  wwn varchar [pk]
+
+  //user provided
+  label varchar
+  host_id varchar
+
+  // smartctl provided
+  device_name varchar
+  manufacturer varchar
+  model_name varchar
+  interface_type varchar
+  interface_speed varchar
+  serial_number varchar
+  firmware varchar
+  rotational_speed varchar
+  capacity varchar
+  form_factor varchar
+  smart_support varchar
+  device_protocol varchar
+  device_type varchar
+
+}
+
+
+// InfluxDB Tables
+Table device_temperature {
+  //timestamp
+  created_at timestamp
+
+  //tags (indexed & queryable)
+  device_wwn varchar [pk]
+
+  //fields
+  temp bigint
+ }
+
+
+Table smart_ata_results {
+  //timestamp
+  created_at timestamp
+
+  //tags (indexed & queryable)
+  device_wwn varchar [pk]
+  smart_status varchar
+  scrutiny_status varchar
+
+
+
+  //fields
+  temp bigint
+  power_on_hours bigint
+  power_cycle_count bigint
+
+
+ }
+
+Ref: device.wwn < smart_ata_results.device_wwn

example.collector.yaml

@@ -1,6 +1,6 @@
 # Commented Scrutiny Configuration File
 #
-# The default location for this file is /scrutiny/config/collector.yaml.
+# The default location for this file is /opt/scrutiny/config/collector.yaml.
 # In some cases to improve clarity default values are specified,
 # uncommented. Other example values are commented out.
 #
@@ -54,7 +54,12 @@ devices:
 #      - 3ware,4
 #      - 3ware,5
 
-
+#log:
+#  file: '' #absolute or relative paths allowed, eg. web.log
+#  level: INFO
+#
+#api:
+#  endpoint: 'http://localhost:8080'
 
 ########################################################################################################################
 # FEATURES COMING SOON

example.scrutiny.yaml

@@ -1,6 +1,6 @@
 # Commented Scrutiny Configuration File
 #
-# The default location for this file is /scrutiny/config/scrutiny.yaml.
+# The default location for this file is /opt/scrutiny/config/scrutiny.yaml.
 # In some cases to improve clarity default values are specified,
 # uncommented. Other example values are commented out.
 #
@@ -20,13 +20,27 @@ web:
   listen:
     port: 8080
     host: 0.0.0.0
+
+    # if you're using a reverse proxy like apache/nginx, you can override this value to serve scrutiny on a subpath.
+    # eg. http://example.com/scrutiny/* vs http://example.com:8080
+    # see docs/TROUBLESHOOTING_REVERSE_PROXY.md
+    # basepath: `/scrutiny`
+    # leave empty unless behind a path prefixed proxy
+    basepath: ''
   database:
     # can also set absolute path here
-    location: /scrutiny/config/scrutiny.db
+    location: /opt/scrutiny/config/scrutiny.db
   src:
+    # the location on the filesystem where scrutiny javascript + css is located
     frontend:
-      path: /scrutiny/web
-
+      path: /opt/scrutiny/web
+  influxdb:
+    host: 0.0.0.0
+    port: 8086
+#    token: 'my-token'
+#    org: 'my-org'
+#    bucket: 'bucket'
+    retention_policy: true
 
 log:
   file: '' #absolute or relative paths allowed, eg. web.log

go.mod

@@ -4,22 +4,28 @@ go 1.13
 
 require (
 	github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
-	github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a
-	github.com/fatih/color v1.9.0
+	github.com/citilinkru/libudev v1.0.0 // indirect
+	github.com/containrrr/shoutrrr v0.4.4
+	github.com/fatih/color v1.10.0
 	github.com/gin-gonic/gin v1.6.3
+	github.com/go-gormigrate/gormigrate/v2 v2.0.0
 	github.com/golang/mock v1.4.3
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/influxdata/influxdb-client-go/v2 v2.2.3
 	github.com/jaypipes/ghw v0.6.1
+	github.com/jinzhu/gorm v1.9.16
+	github.com/klauspost/compress v1.12.1 // indirect
 	github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect
 	github.com/mattn/go-sqlite3 v1.14.4 // indirect
 	github.com/mitchellh/mapstructure v1.2.2
-	github.com/sirupsen/logrus v1.2.0
+	github.com/onsi/ginkgo v1.16.1 // indirect
+	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/viper v1.7.0
 	github.com/stretchr/testify v1.5.1
 	github.com/urfave/cli/v2 v2.2.0
-	golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
-	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	gopkg.in/yaml.v2 v2.3.0 // indirect
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect
 	gorm.io/driver/sqlite v1.1.3
 	gorm.io/gorm v1.20.2
+	nhooyr.io/websocket v1.8.7 // indirect
 )

go.sum

@@ -1,456 +0,0 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
-github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14 h1:wsrSjiqQtseStRIoLLxS4C5IEtXkazZVEPDHq8jW7r8=
-github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14/go.mod h1:lJQVqFKMV5/oDGYR2bra2OljcF3CvolAoyDRyOA4k4E=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a h1:6ZMiughZYF6fJjFIf2X3D7AfImJeXnTMJ9qC2v75WPw=
-github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a/go.mod h1:z3pUtEhu5zOpu+Q8wZWiEq+ZLL9hM0HiFNhttaI67Ks=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
-github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
-github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
-github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
-github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
-github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
-github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
-github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
-github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
-github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
-github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
-github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
-github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
-github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
-github.com/jaypipes/ghw v0.6.1 h1:Ewt3mdpiyhWotGyzg1ursV/6SnToGcG4215X6rR2af8=
-github.com/jaypipes/ghw v0.6.1/go.mod h1:QOXppNRCLGYR1H+hu09FxZPqjNt09bqUZUnOL3Rcero=
-github.com/jaypipes/pcidb v0.5.0 h1:4W5gZ+G7QxydevI8/MmmKdnIPJpURqJ2JNXTzfLxF5c=
-github.com/jaypipes/pcidb v0.5.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
-github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
-github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 h1:3tLzEnUizyN9YLWFTT9loC30lSBvh2y70LTDcZOTs1s=
-github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0/go.mod h1:8/LTPeDLaklcUjgSQBHbhBF1ibKAFxzS5o+H7USfMSA=
-github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
-github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
-github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA=
-github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
-github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI=
-github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4=
-github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
-github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
-github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
-github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw=
-github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
-github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
-github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
-github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
-github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
-golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa h1:mQTN3ECqfsViCNBgq+A40vdwhkGykrrQlYe3mPj6BoU=
-golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ=
-gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc=
-gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c=
-gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
-gorm.io/gorm v1.20.2 h1:bZzSEnq7NDGsrd+n3evOOedDrY5oLM5QPlCjZJUK2ro=
-gorm.io/gorm v1.20.2/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
-gosrc.io/xmpp v0.1.1 h1:iMtE9W3fx254+4E6rI34AOPJDqWvpfQR6EYaVMzhJ4s=
-gosrc.io/xmpp v0.1.1/go.mod h1:4JgaXzw4MnEv2sGltONtK3GMhj+h9gpQ7cO8nwbFJLU=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
-howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

rootfs/etc/cont-init.d/01-timezone

@@ -0,0 +1,7 @@
+#!/usr/bin/with-contenv bash
+
+if [ -n "${TZ}" ]
+then
+    ln -snf "/usr/share/zoneinfo/${TZ}" /etc/localtime
+    echo "${TZ}" > /etc/timezone
+fi

rootfs/etc/cont-init.d/50-config

@@ -0,0 +1,4 @@
+#!/usr/bin/with-contenv bash
+
+COLLECTOR_CRON_SCHEDULE=${COLLECTOR_CRON_SCHEDULE:-"0 0 * * *"}
+sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron.d/scrutiny

rootfs/etc/cron.d/scrutiny

@@ -0,0 +1,15 @@
+MAILTO=""
+# Example of job definition:
+# .---------------- minute (0 - 59)
+# |  .------------- hour (0 - 23)
+# |  |  .---------- day of month (1 - 31)
+# |  |  |  .------- month (1 - 12) OR jan,feb,mar,apr ...
+# |  |  |  |  .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat
+# |  |  |  |  |
+# *  *  *  *  * user-name command to be executed
+
+# correctly route collector logs (STDOUT & STDERR) to Cron foreground (collectable by Docker STDOUT)
+# cron schedule to run daily at midnight:  '0 0 * * *'
+# System environmental variables are stripped by cron, source our dump of the docker environmental variables before each command (/env.sh)
+{COLLECTOR_CRON_SCHEDULE} root . /env.sh; /opt/scrutiny/bin/scrutiny-collector-metrics run >/proc/1/fd/1 2>/proc/1/fd/2
+# An empty line is required at the end of this file for a valid cron file.

rootfs/etc/services.d/collector-once/run

@@ -7,8 +7,7 @@ s6-svwait -u /var/run/s6/services/scrutiny
 s6-svc -O /var/run/s6/services/collector-once
 
 # wait until scrutiny is "Ready"
-while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:8080/api/health)" != "200" ]]; do sleep 5; done
-
+until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done
 
 echo "starting scrutiny collector"
-/scrutiny/bin/scrutiny-collector-metrics run
+/opt/scrutiny/bin/scrutiny-collector-metrics run

rootfs/etc/services.d/cron/finish

@@ -1,4 +1,4 @@
 #!/usr/bin/execlineb -S0
 
-echo "jobber/cron exiting"
+echo "cron exiting"
 s6-svscanctl -t /var/run/s6/services

rootfs/etc/services.d/cron/run

@@ -0,0 +1,10 @@
+#!/usr/bin/with-contenv bash
+
+# Cron runs in its own isolated environment (usually using only /etc/environment )
+# So when the container starts up, we will do a dump of the runtime environment into a .env file that we
+# will then source into the crontab file (/etc/cron.d/scrutiny.sh)
+
+printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
+
+echo "starting cron"
+cron -f

rootfs/etc/services.d/influxdb/run

@@ -0,0 +1,17 @@
+#!/usr/bin/with-contenv bash
+
+mkdir -p /opt/scrutiny/influxdb/
+
+if [ -f "/opt/scrutiny/influxdb/config.yaml" ]; then
+    echo "influxdb config file already exists. skipping."
+else
+cat << 'EOF' > /opt/scrutiny/influxdb/config.yaml
+bolt-path: /opt/scrutiny/influxdb/influxd.bolt
+engine-path: /opt/scrutiny/influxdb/engine
+http-bind-address: ":8086"
+reporting-disabled: true
+EOF
+fi
+
+echo "starting influxdb"
+influxd run

rootfs/etc/services.d/jobber/run

@@ -1,5 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-echo "starting jobber/cron"
-
-su -c "/usr/lib/x86_64-linux-gnu/jobberrunner /scrutiny/jobber/jobber.yaml" root

rootfs/etc/services.d/scrutiny/run

@@ -1,5 +1,7 @@
 #!/usr/bin/with-contenv bash
 
-echo "starting scrutiny"
+echo "waiting for influxdb"
+until $(curl --output /dev/null --silent --head --fail http://localhost:8086/health); do echo "influxdb not ready" && sleep 5; done
 
+echo "starting scrutiny"
 scrutiny start

rootfs/scrutiny/jobber/jobber.yaml

@@ -1,30 +0,0 @@
-version: 1.4
-
-prefs:
-  logPath: /scrutiny/jobber/log.log
-  runLog:
-    type: file
-    path: /scrutiny/jobber/runlog
-    maxFileLen: 100m
-    maxHistories: 2
-
-resultSinks:
-  - &filesystemSink
-    type: filesystem
-    path: /scrutiny/jobber
-    data:
-      - stdout
-      - stderr
-    maxAgeDays: 10
-
-jobs:
-  MetricsJob:
-    cmd: /scrutiny/bin/scrutiny-collector-metrics run
-    # run daily at midnight.
-    time: '0 0 0 * * *'
-    onError: Backoff
-    notifyOnSuccess:
-      - *filesystemSink
-    notifyOnFailure:
-      - *filesystemSink
-

webapp/backend/cmd/scrutiny/scrutiny.go

@@ -27,8 +27,8 @@ func main() {
 	}
 
 	//we're going to load the config file manually, since we need to validate it.
-	err = config.ReadConfig("/scrutiny/config/scrutiny.yaml") // Find and read the config file
-	if _, ok := err.(errors.ConfigFileMissingError); ok {     // Handle errors reading the config file
+	err = config.ReadConfig("/opt/scrutiny/config/scrutiny.yaml") // Find and read the config file
+	if _, ok := err.(errors.ConfigFileMissingError); ok {         // Handle errors reading the config file
 		//ignore "could not find config file"
 	} else if err != nil {
 		os.Exit(1)

webapp/backend/pkg/config/config.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/viper"
 	"log"
 	"os"
+	"strings"
 )
 
 // When initializing this class the following methods must be called:
@@ -29,20 +30,29 @@ func (c *configuration) Init() error {
 	//set defaults
 	c.SetDefault("web.listen.port", "8080")
 	c.SetDefault("web.listen.host", "0.0.0.0")
-	c.SetDefault("web.src.frontend.path", "/scrutiny/web")
-	c.SetDefault("web.database.location", "/scrutiny/config/scrutiny.db")
+	c.SetDefault("web.listen.basepath", "")
+	c.SetDefault("web.src.frontend.path", "/opt/scrutiny/web")
+	c.SetDefault("web.database.location", "/opt/scrutiny/config/scrutiny.db")
 
 	c.SetDefault("log.level", "INFO")
 	c.SetDefault("log.file", "")
 
 	c.SetDefault("notify.urls", []string{})
 
+	c.SetDefault("web.influxdb.host", "0.0.0.0")
+	c.SetDefault("web.influxdb.port", "8086")
+	c.SetDefault("web.influxdb.org", "scrutiny")
+	c.SetDefault("web.influxdb.bucket", "metrics")
+	c.SetDefault("web.influxdb.init_username", "admin")
+	c.SetDefault("web.influxdb.init_password", "password12345")
+	c.SetDefault("web.influxdb.retention_policy", true)
+
 	//c.SetDefault("disks.include", []string{})
 	//c.SetDefault("disks.exclude", []string{})
 
-	//c.SetDefault("notify.metric.script", "/scrutiny/config/notify-metrics.sh")
-	//c.SetDefault("notify.long.script", "/scrutiny/config/notify-long-test.sh")
-	//c.SetDefault("notify.short.script", "/scrutiny/config/notify-short-test.sh")
+	//c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh")
+	//c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh")
+	//c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh")
 
 	//c.SetDefault("collect.metric.enable", true)
 	//c.SetDefault("collect.metric.command", "-a -o on -S on")
@@ -56,11 +66,19 @@ func (c *configuration) Init() error {
 	//c.SetConfigName("drawbridge")
 	//c.AddConfigPath("$HOME/")
 
+	//configure env variable parsing.
+	c.SetEnvPrefix("SCRUTINY")
+	c.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
+	c.AutomaticEnv()
+
 	//CLI options will be added via the `Set()` function
 	return nil
 }
 
 func (c *configuration) ReadConfig(configFilePath string) error {
+	//make sure that we specify that this is the correct config path (for eventual WriteConfig() calls)
+	c.SetConfigFile(configFilePath)
+
 	configFilePath, err := utils.ExpandPath(configFilePath)
 	if err != nil {
 		return err

webapp/backend/pkg/config/interface.go

@@ -9,6 +9,7 @@ import (
 type Interface interface {
 	Init() error
 	ReadConfig(configFilePath string) error
+	WriteConfig() error
 	Set(key string, value interface{})
 	SetDefault(key string, value interface{})
 

webapp/backend/pkg/config/mock/mock_config.go

@@ -5,87 +5,36 @@
 package mock_config
 
 import (
+	reflect "reflect"
+
 	gomock "github.com/golang/mock/gomock"
 	viper "github.com/spf13/viper"
-	reflect "reflect"
 )
 
-// MockInterface is a mock of Interface interface
+// MockInterface is a mock of Interface interface.
 type MockInterface struct {
 	ctrl     *gomock.Controller
 	recorder *MockInterfaceMockRecorder
 }
 
-// MockInterfaceMockRecorder is the mock recorder for MockInterface
+// MockInterfaceMockRecorder is the mock recorder for MockInterface.
 type MockInterfaceMockRecorder struct {
 	mock *MockInterface
 }
 
-// NewMockInterface creates a new mock instance
+// NewMockInterface creates a new mock instance.
 func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
 	mock := &MockInterface{ctrl: ctrl}
 	mock.recorder = &MockInterfaceMockRecorder{mock}
 	return mock
 }
 
-// EXPECT returns an object that allows the caller to indicate expected use
+// EXPECT returns an object that allows the caller to indicate expected use.
 func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
-// Init mocks base method
-func (m *MockInterface) Init() error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Init")
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Init indicates an expected call of Init
-func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
-}
-
-// ReadConfig mocks base method
-func (m *MockInterface) ReadConfig(configFilePath string) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// ReadConfig indicates an expected call of ReadConfig
-func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
-}
-
-// Set mocks base method
-func (m *MockInterface) Set(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Set", key, value)
-}
-
-// Set indicates an expected call of Set
-func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
-}
-
-// SetDefault mocks base method
-func (m *MockInterface) SetDefault(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "SetDefault", key, value)
-}
-
-// SetDefault indicates an expected call of SetDefault
-func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
-}
-
-// AllSettings mocks base method
+// AllSettings mocks base method.
 func (m *MockInterface) AllSettings() map[string]interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "AllSettings")
@@ -93,27 +42,13 @@ func (m *MockInterface) AllSettings() map[string]interface{} {
 	return ret0
 }
 
-// AllSettings indicates an expected call of AllSettings
+// AllSettings indicates an expected call of AllSettings.
 func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings))
 }
 
-// IsSet mocks base method
-func (m *MockInterface) IsSet(key string) bool {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "IsSet", key)
-	ret0, _ := ret[0].(bool)
-	return ret0
-}
-
-// IsSet indicates an expected call of IsSet
-func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
-}
-
-// Get mocks base method
+// Get mocks base method.
 func (m *MockInterface) Get(key string) interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Get", key)
@@ -121,13 +56,13 @@ func (m *MockInterface) Get(key string) interface{} {
 	return ret0
 }
 
-// Get indicates an expected call of Get
+// Get indicates an expected call of Get.
 func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key)
 }
 
-// GetBool mocks base method
+// GetBool mocks base method.
 func (m *MockInterface) GetBool(key string) bool {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetBool", key)
@@ -135,13 +70,13 @@ func (m *MockInterface) GetBool(key string) bool {
 	return ret0
 }
 
-// GetBool indicates an expected call of GetBool
+// GetBool indicates an expected call of GetBool.
 func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key)
 }
 
-// GetInt mocks base method
+// GetInt mocks base method.
 func (m *MockInterface) GetInt(key string) int {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetInt", key)
@@ -149,13 +84,13 @@ func (m *MockInterface) GetInt(key string) int {
 	return ret0
 }
 
-// GetInt indicates an expected call of GetInt
+// GetInt indicates an expected call of GetInt.
 func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
 }
 
-// GetString mocks base method
+// GetString mocks base method.
 func (m *MockInterface) GetString(key string) string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetString", key)
@@ -163,13 +98,13 @@ func (m *MockInterface) GetString(key string) string {
 	return ret0
 }
 
-// GetString indicates an expected call of GetString
+// GetString indicates an expected call of GetString.
 func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key)
 }
 
-// GetStringSlice mocks base method
+// GetStringSlice mocks base method.
 func (m *MockInterface) GetStringSlice(key string) []string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetStringSlice", key)
@@ -177,13 +112,79 @@ func (m *MockInterface) GetStringSlice(key string) []string {
 	return ret0
 }
 
-// GetStringSlice indicates an expected call of GetStringSlice
+// GetStringSlice indicates an expected call of GetStringSlice.
 func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key)
 }
 
-// UnmarshalKey mocks base method
+// Init mocks base method.
+func (m *MockInterface) Init() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Init")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Init indicates an expected call of Init.
+func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
+}
+
+// IsSet mocks base method.
+func (m *MockInterface) IsSet(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsSet", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsSet indicates an expected call of IsSet.
+func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
+}
+
+// ReadConfig mocks base method.
+func (m *MockInterface) ReadConfig(configFilePath string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReadConfig indicates an expected call of ReadConfig.
+func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
+}
+
+// Set mocks base method.
+func (m *MockInterface) Set(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Set", key, value)
+}
+
+// Set indicates an expected call of Set.
+func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
+}
+
+// SetDefault mocks base method.
+func (m *MockInterface) SetDefault(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetDefault", key, value)
+}
+
+// SetDefault indicates an expected call of SetDefault.
+func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
+}
+
+// UnmarshalKey mocks base method.
 func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{key, rawVal}
@@ -195,9 +196,23 @@ func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts
 	return ret0
 }
 
-// UnmarshalKey indicates an expected call of UnmarshalKey
+// UnmarshalKey indicates an expected call of UnmarshalKey.
 func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{key, rawVal}, decoderOpts...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...)
 }
+
+// WriteConfig mocks base method.
+func (m *MockInterface) WriteConfig() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "WriteConfig")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// WriteConfig indicates an expected call of WriteConfig.
+func (mr *MockInterfaceMockRecorder) WriteConfig() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteConfig", reflect.TypeOf((*MockInterface)(nil).WriteConfig))
+}

webapp/backend/pkg/constants.go

@@ -0,0 +1,28 @@
+package pkg
+
+const DeviceProtocolAta = "ATA"
+const DeviceProtocolScsi = "SCSI"
+const DeviceProtocolNvme = "NVMe"
+
+const SmartAttributeStatusPassed = 0
+const SmartAttributeStatusFailed = 1
+const SmartAttributeStatusWarning = 2
+
+const SmartWhenFailedFailingNow = "FAILING_NOW"
+const SmartWhenFailedInThePast = "IN_THE_PAST"
+
+//const SmartStatusPassed = "passed"
+//const SmartStatusFailed = "failed"
+
+type DeviceStatus int
+
+const (
+	DeviceStatusPassed         DeviceStatus = 0
+	DeviceStatusFailedSmart    DeviceStatus = iota
+	DeviceStatusFailedScrutiny DeviceStatus = iota
+)
+
+func Set(b, flag DeviceStatus) DeviceStatus    { return b | flag }
+func Clear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
+func Toggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
+func Has(b, flag DeviceStatus) bool            { return b&flag != 0 }

webapp/backend/pkg/database/interface.go

@@ -0,0 +1,30 @@
+package database
+
+import (
+	"context"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+)
+
+type DeviceRepo interface {
+	Close() error
+
+	//GetSettings()
+	//SaveSetting()
+
+	RegisterDevice(ctx context.Context, dev models.Device) error
+	GetDevices(ctx context.Context) ([]models.Device, error)
+	UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error)
+	UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error)
+	GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error)
+
+	SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error)
+	GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error)
+
+	SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error
+
+	GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
+	GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)
+}

webapp/backend/pkg/database/migrations/m20201107210306/device.go

@@ -0,0 +1,47 @@
+package m20201107210306
+
+import (
+	"time"
+)
+
+// Deprecated: m20201107210306.Device is deprecated, only used by db migrations
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN    string `json:"wwn" gorm:"primary_key"`
+	HostId string `json:"host_id"`
+
+	DeviceName     string  `json:"device_name"`
+	Manufacturer   string  `json:"manufacturer"`
+	ModelName      string  `json:"model_name"`
+	InterfaceType  string  `json:"interface_type"`
+	InterfaceSpeed string  `json:"interface_speed"`
+	SerialNumber   string  `json:"serial_number"`
+	Firmware       string  `json:"firmware"`
+	RotationSpeed  int     `json:"rotational_speed"`
+	Capacity       int64   `json:"capacity"`
+	FormFactor     string  `json:"form_factor"`
+	SmartSupport   bool    `json:"smart_support"`
+	DeviceProtocol string  `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string  `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+	SmartResults   []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
+}
+
+const DeviceProtocolAta = "ATA"
+const DeviceProtocolScsi = "SCSI"
+const DeviceProtocolNvme = "NVMe"
+
+func (dv *Device) IsAta() bool {
+	return dv.DeviceProtocol == DeviceProtocolAta
+}
+
+func (dv *Device) IsScsi() bool {
+	return dv.DeviceProtocol == DeviceProtocolScsi
+}
+
+func (dv *Device) IsNvme() bool {
+	return dv.DeviceProtocol == DeviceProtocolNvme
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart.go

@@ -0,0 +1,26 @@
+package m20201107210306
+
+import (
+	"gorm.io/gorm"
+	"time"
+)
+
+// Deprecated: m20201107210306.Smart is deprecated, only used by db migrations
+type Smart struct {
+	gorm.Model
+
+	DeviceWWN string `json:"device_wwn"`
+	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
+
+	TestDate    time.Time `json:"date"`
+	SmartStatus string    `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed
+
+	//Metrics
+	Temp            int64 `json:"temp"`
+	PowerOnHours    int64 `json:"power_on_hours"`
+	PowerCycleCount int64 `json:"power_cycle_count"`
+
+	AtaAttributes  []SmartAtaAttribute  `json:"ata_attributes" gorm:"foreignkey:SmartId"`
+	NvmeAttributes []SmartNvmeAttribute `json:"nvme_attributes" gorm:"foreignkey:SmartId"`
+	ScsiAttributes []SmartScsiAttribute `json:"scsi_attributes" gorm:"foreignkey:SmartId"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go

@@ -0,0 +1,26 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartAtaAttribute is deprecated, only used by db migrations
+type SmartAtaAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId int    `json:"attribute_id"`
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Worst       int    `json:"worst"`
+	Threshold   int    `json:"thresh"`
+	RawValue    int64  `json:"raw_value"`
+	RawString   string `json:"raw_string"`
+	WhenFailed  string `json:"when_failed"`
+
+	TransformedValue int64               `json:"transformed_value"`
+	Status           string              `gorm:"-" json:"status,omitempty"`
+	StatusReason     string              `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64             `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartAtaAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go

@@ -0,0 +1,22 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartNvmeAttribute is deprecated, only used by db migrations
+type SmartNvmeAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Threshold   int    `json:"thresh"`
+
+	TransformedValue int64                `json:"transformed_value"`
+	Status           string               `gorm:"-" json:"status,omitempty"`
+	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartNvmeAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go

@@ -0,0 +1,22 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartScsiAttribute is deprecated, only used by db migrations
+type SmartScsiAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Threshold   int    `json:"thresh"`
+
+	TransformedValue int64                `json:"transformed_value"`
+	Status           string               `gorm:"-" json:"status,omitempty"`
+	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartScsiAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/scrutiny_repository.go

@@ -0,0 +1,407 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api"
+	"github.com/influxdata/influxdb-client-go/v2/domain"
+	"github.com/sirupsen/logrus"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"time"
+)
+
+const (
+	// 60seconds * 60minutes * 24hours * 15 days
+	RETENTION_PERIOD_15_DAYS_IN_SECONDS = 1_296_000
+
+	// 60seconds * 60minutes * 24hours * 7 days * 9 weeks
+	RETENTION_PERIOD_9_WEEKS_IN_SECONDS = 5_443_200
+
+	// 60seconds * 60minutes * 24hours * 7 days * (52 + 52 + 4)weeks
+	RETENTION_PERIOD_25_MONTHS_IN_SECONDS = 65_318_400
+
+	DURATION_KEY_WEEK    = "week"
+	DURATION_KEY_MONTH   = "month"
+	DURATION_KEY_YEAR    = "year"
+	DURATION_KEY_FOREVER = "forever"
+)
+
+//// GormLogger is a custom logger for Gorm, making it use logrus.
+//type GormLogger struct{ Logger logrus.FieldLogger }
+//
+//// Print handles log events from Gorm for the custom logger.
+//func (gl *GormLogger) Print(v ...interface{}) {
+//	switch v[0] {
+//	case "sql":
+//		gl.Logger.WithFields(
+//			logrus.Fields{
+//				"module":  "gorm",
+//				"type":    "sql",
+//				"rows":    v[5],
+//				"src_ref": v[1],
+//				"values":  v[4],
+//			},
+//		).Debug(v[3])
+//	case "log":
+//		gl.Logger.WithFields(logrus.Fields{"module": "gorm", "type": "log"}).Print(v[2])
+//	}
+//}
+
+func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.FieldLogger) (DeviceRepo, error) {
+	backgroundContext := context.Background()
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// Gorm/SQLite setup
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
+	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
+		//TODO: figure out how to log database queries again.
+		//Logger: logger
+		DisableForeignKeyConstraintWhenMigrating: true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Failed to connect to database! - %v", err)
+	}
+	globalLogger.Infof("Successfully connected to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
+
+	//database.SetLogger()
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// InfluxDB setup
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+	// Create a new client using an InfluxDB server base URL and an authentication token
+	influxdbUrl := fmt.Sprintf("http://%s:%s", appConfig.GetString("web.influxdb.host"), appConfig.GetString("web.influxdb.port"))
+	globalLogger.Debugf("InfluxDB url: %s", influxdbUrl)
+
+	client := influxdb2.NewClient(influxdbUrl, appConfig.GetString("web.influxdb.token"))
+
+	if !appConfig.IsSet("web.influxdb.token") {
+		globalLogger.Debugf("No influxdb token found, running first-time setup...")
+
+		// if no token is provided, but we have a valid server, we're going to assume this is the first setup of our server.
+		// we will initialize with a predetermined username & password, that you should change.
+
+		// metrics bucket will have a retention period of 8 days (since it will be down-sampled once a week)
+		// in seconds (60seconds * 60minutes * 24hours * 15 days) = 1_296_000 (see EnsureBucket() function)
+		onboardingResponse, err := client.Setup(
+			backgroundContext,
+			appConfig.GetString("web.influxdb.init_username"),
+			appConfig.GetString("web.influxdb.init_password"),
+			appConfig.GetString("web.influxdb.org"),
+			appConfig.GetString("web.influxdb.bucket"),
+			0)
+		if err != nil {
+			return nil, err
+		}
+
+		appConfig.Set("web.influxdb.token", *onboardingResponse.Auth.Token)
+		// we should write the config file out here. Ignore failures.
+		err = appConfig.WriteConfig()
+		if err != nil {
+			globalLogger.Infof("ignoring error while writing influxdb info to config: %v", err)
+		}
+	}
+
+	// Use blocking write client for writes to desired bucket
+	writeAPI := client.WriteAPIBlocking(appConfig.GetString("web.influxdb.org"), appConfig.GetString("web.influxdb.bucket"))
+
+	// Get query client
+	queryAPI := client.QueryAPI(appConfig.GetString("web.influxdb.org"))
+
+	// Get task client
+	taskAPI := client.TasksAPI()
+
+	if writeAPI == nil || queryAPI == nil || taskAPI == nil {
+		return nil, fmt.Errorf("Failed to connect to influxdb!")
+	}
+
+	deviceRepo := scrutinyRepository{
+		appConfig:      appConfig,
+		logger:         globalLogger,
+		influxClient:   client,
+		influxWriteApi: writeAPI,
+		influxQueryApi: queryAPI,
+		influxTaskApi:  taskAPI,
+		gormClient:     database,
+	}
+
+	orgInfo, err := client.OrganizationsAPI().FindOrganizationByName(backgroundContext, appConfig.GetString("web.influxdb.org"))
+	if err != nil {
+		return nil, err
+	}
+
+	// Initialize Buckets (if necessary)
+	err = deviceRepo.EnsureBuckets(backgroundContext, orgInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	// Initialize Background Tasks
+	err = deviceRepo.EnsureTasks(backgroundContext, *orgInfo.Id)
+	if err != nil {
+		return nil, err
+	}
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// InfluxDB & SQLite migrations
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	//database.AutoMigrate(&models.Device{})
+	err = deviceRepo.Migrate(backgroundContext)
+	if err != nil {
+		return nil, err
+	}
+
+	return &deviceRepo, nil
+}
+
+type scrutinyRepository struct {
+	appConfig config.Interface
+	logger    logrus.FieldLogger
+
+	influxWriteApi api.WriteAPIBlocking
+	influxQueryApi api.QueryAPI
+	influxTaskApi  api.TasksAPI
+	influxClient   influxdb2.Client
+
+	gormClient *gorm.DB
+}
+
+func (sr *scrutinyRepository) Close() error {
+	sr.influxClient.Close()
+	return nil
+}
+
+func (sr *scrutinyRepository) EnsureBuckets(ctx context.Context, org *domain.Organization) error {
+
+	var mainBucketRetentionRule domain.RetentionRule
+	var weeklyBucketRetentionRule domain.RetentionRule
+	var monthlyBucketRetentionRule domain.RetentionRule
+	if sr.appConfig.GetBool("web.influxdb.retention_policy") {
+
+		// in tests, we may not want to set a retention policy. If "false", we can set data with old timestamps,
+		// then manually run the down sampling scripts. This should be true for production environments.
+		mainBucketRetentionRule = domain.RetentionRule{EverySeconds: RETENTION_PERIOD_15_DAYS_IN_SECONDS}
+		weeklyBucketRetentionRule = domain.RetentionRule{EverySeconds: RETENTION_PERIOD_9_WEEKS_IN_SECONDS}
+		monthlyBucketRetentionRule = domain.RetentionRule{EverySeconds: RETENTION_PERIOD_25_MONTHS_IN_SECONDS}
+	}
+
+	mainBucket := sr.appConfig.GetString("web.influxdb.bucket")
+	if foundMainBucket, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, mainBucket); foundErr != nil {
+		// metrics bucket will have a retention period of 15 days (since it will be down-sampled once a week)
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, mainBucket, mainBucketRetentionRule)
+		if err != nil {
+			return err
+		}
+	} else if sr.appConfig.GetBool("web.influxdb.retention_policy") {
+		//correctly set the retention period for the main bucket (cant do it during setup/creation)
+		foundMainBucket.RetentionRules = domain.RetentionRules{mainBucketRetentionRule}
+		sr.influxClient.BucketsAPI().UpdateBucket(ctx, foundMainBucket)
+	}
+
+	//create buckets (used for downsampling)
+	weeklyBucket := fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, weeklyBucket); foundErr != nil {
+		// metrics_weekly bucket will have a retention period of 8+1 weeks (since it will be down-sampled once a month)
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, weeklyBucket, weeklyBucketRetentionRule)
+		if err != nil {
+			return err
+		}
+	}
+
+	monthlyBucket := fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, monthlyBucket); foundErr != nil {
+		// metrics_monthly bucket will have a retention period of 24+1 months (since it will be down-sampled once a year)
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, monthlyBucket, monthlyBucketRetentionRule)
+		if err != nil {
+			return err
+		}
+	}
+
+	yearlyBucket := fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
+	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, yearlyBucket); foundErr != nil {
+		// metrics_yearly bucket will have an infinite retention period
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, yearlyBucket)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// DeviceSummary
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// get a map of all devices and associated SMART data
+func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) {
+	devices, err := sr.GetDevices(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	summaries := map[string]*models.DeviceSummary{}
+
+	for _, device := range devices {
+		summaries[device.WWN] = &models.DeviceSummary{Device: device}
+	}
+
+	// Get parser flux query result
+	//appConfig.GetString("web.influxdb.bucket")
+	queryStr := fmt.Sprintf(`
+  	import "influxdata/influxdb/schema"
+  	bucketBaseName = "%s"
+
+	dailyData = from(bucket: bucketBaseName)
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	weeklyData = from(bucket: bucketBaseName + "_weekly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	monthlyData = from(bucket: bucketBaseName + "_monthly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	yearlyData = from(bucket: bucketBaseName + "_yearly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	union(tables: [dailyData, weeklyData, monthlyData, yearlyData])
+	|> sort(columns: ["_time"], desc: false)
+	|> group(columns: ["device_wwn"])
+	|> last(column: "device_wwn")
+	|> yield(name: "last")
+		`,
+		sr.appConfig.GetString("web.influxdb.bucket"),
+	)
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+			// Observe when there is new grouping key producing new table
+			if result.TableChanged() {
+				//fmt.Printf("table: %s\n", result.TableMetadata().String())
+			}
+			// read result
+
+			//get summary data from Influxdb.
+			//result.Record().Values()
+			if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
+
+				//ensure summaries is intialized for this wwn
+				if _, exists := summaries[deviceWWN.(string)]; !exists {
+					summaries[deviceWWN.(string)] = &models.DeviceSummary{}
+				}
+
+				summaries[deviceWWN.(string)].SmartResults = &models.SmartSummary{
+					Temp:          result.Record().Values()["temp"].(int64),
+					PowerOnHours:  result.Record().Values()["power_on_hours"].(int64),
+					CollectorDate: result.Record().Values()["_time"].(time.Time),
+				}
+			}
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+
+	deviceTempHistory, err := sr.GetSmartTemperatureHistory(ctx, DURATION_KEY_FOREVER)
+	if err != nil {
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("Error: %v", err)
+	}
+	for wwn, tempHistory := range deviceTempHistory {
+		summaries[wwn].TempHistory = tempHistory
+	}
+
+	return summaries, nil
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Helper Methods
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+func (sr *scrutinyRepository) lookupBucketName(durationKey string) string {
+	switch durationKey {
+	case DURATION_KEY_WEEK:
+		//data stored in the last week
+		return sr.appConfig.GetString("web.influxdb.bucket")
+	case DURATION_KEY_MONTH:
+		// data stored in the last month (after the first week)
+		return fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+	case DURATION_KEY_YEAR:
+		// data stored in the last year (after the first month)
+		return fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+	case DURATION_KEY_FOREVER:
+		//data stored before the last year
+		return fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
+	}
+	return sr.appConfig.GetString("web.influxdb.bucket")
+}
+
+func (sr *scrutinyRepository) lookupDuration(durationKey string) []string {
+
+	switch durationKey {
+	case DURATION_KEY_WEEK:
+		//data stored in the last week
+		return []string{"-1w", "now()"}
+	case DURATION_KEY_MONTH:
+		// data stored in the last month (after the first week)
+		return []string{"-1mo", "-1w"}
+	case DURATION_KEY_YEAR:
+		// data stored in the last year (after the first month)
+		return []string{"-1y", "-1mo"}
+	case DURATION_KEY_FOREVER:
+		//data stored before the last year
+		return []string{"-10y", "-1y"}
+	}
+	return []string{"-1w", "now()"}
+}
+
+func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []string {
+	switch durationKey {
+	case DURATION_KEY_WEEK:
+		//all data is stored in a single bucket
+		return []string{DURATION_KEY_WEEK}
+	case DURATION_KEY_MONTH:
+		//data is stored in the week bucket and the month bucket
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH}
+	case DURATION_KEY_YEAR:
+		// data stored in the last year (after the first month)
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR}
+	case DURATION_KEY_FOREVER:
+		//data stored before the last year
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR, DURATION_KEY_FOREVER}
+	}
+	return []string{DURATION_KEY_WEEK}
+}

webapp/backend/pkg/database/scrutiny_repository_device.go

@@ -0,0 +1,74 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"gorm.io/gorm/clause"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Device
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+//insert device into DB (and update specified columns if device is already registered)
+// update device fields that may change: (DeviceType, HostID)
+func (sr *scrutinyRepository) RegisterDevice(ctx context.Context, dev models.Device) error {
+	if err := sr.gormClient.WithContext(ctx).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "wwn"}},
+		DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}),
+	}).Create(&dev).Error; err != nil {
+		return err
+	}
+	return nil
+}
+
+// get a list of all devices (only device metadata, no SMART data)
+func (sr *scrutinyRepository) GetDevices(ctx context.Context) ([]models.Device, error) {
+	//Get a list of all the active devices.
+	devices := []models.Device{}
+	if err := sr.gormClient.WithContext(ctx).Find(&devices).Error; err != nil {
+		return nil, fmt.Errorf("Could not get device summary from DB: %v", err)
+	}
+	return devices, nil
+}
+
+// update device (only metadata) from collector
+func (sr *scrutinyRepository) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) {
+	var device models.Device
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return device, fmt.Errorf("Could not get device from DB: %v", err)
+	}
+
+	//TODO catch GormClient err
+	err := device.UpdateFromCollectorSmartInfo(collectorSmartData)
+	if err != nil {
+		return device, err
+	}
+	return device, sr.gormClient.Model(&device).Updates(device).Error
+}
+
+//Update Device Status
+func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) {
+	var device models.Device
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return device, fmt.Errorf("Could not get device from DB: %v", err)
+	}
+
+	device.DeviceStatus = pkg.Set(device.DeviceStatus, status)
+	return device, sr.gormClient.Model(&device).Updates(device).Error
+}
+
+func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) {
+	var device models.Device
+
+	fmt.Println("GetDeviceDetails from GORM")
+
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return models.Device{}, err
+	}
+
+	return device, nil
+}

webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go

@@ -0,0 +1,169 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api"
+	log "github.com/sirupsen/logrus"
+	"strings"
+	"time"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// SMART
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
+	deviceSmartData := measurements.Smart{}
+	err := deviceSmartData.FromCollectorSmartInfo(wwn, collectorSmartData)
+	if err != nil {
+		sr.logger.Errorln("Could not process SMART metrics", err)
+		return measurements.Smart{}, err
+	}
+
+	tags, fields := deviceSmartData.Flatten()
+
+	// write point immediately
+	return deviceSmartData, sr.saveDatapoint(sr.influxWriteApi, "smart", tags, fields, deviceSmartData.Date, ctx)
+}
+
+func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) {
+	// Get SMartResults from InfluxDB
+
+	//TODO: change the filter startrange to a real number.
+
+	// Get parser flux query result
+	//appConfig.GetString("web.influxdb.bucket")
+	queryStr := sr.aggregateSmartAttributesQuery(wwn, durationKey)
+	log.Infoln(queryStr)
+
+	smartResults := []measurements.Smart{}
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+			// Observe when there is new grouping key producing new table
+			if result.TableChanged() {
+				//fmt.Printf("table: %s\n", result.TableMetadata().String())
+			}
+
+			smartData, err := measurements.NewSmartFromInfluxDB(result.Record().Values())
+			if err != nil {
+				return nil, err
+			}
+			smartResults = append(smartResults, *smartData)
+
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+
+	return smartResults, nil
+
+	//if err := device.SquashHistory(); err != nil {
+	//	logger.Errorln("An error occurred while squashing device history", err)
+	//	c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+	//	return
+	//}
+	//
+	//if err := device.ApplyMetadataRules(); err != nil {
+	//	logger.Errorln("An error occurred while applying scrutiny thresholds & rules", err)
+	//	c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+	//	return
+	//}
+
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Helper Methods
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, measurement string, tags map[string]string, fields map[string]interface{}, date time.Time, ctx context.Context) error {
+	//sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields))
+	p := influxdb2.NewPoint(measurement,
+		tags,
+		fields,
+		date)
+
+	// write point immediately
+	return influxWriteApi.WritePoint(ctx, p)
+}
+
+func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string) string {
+
+	/*
+
+		import "influxdata/influxdb/schema"
+		weekData = from(bucket: "metrics")
+		|> range(start: -1w, stop: now())
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		monthData = from(bucket: "metrics_weekly")
+		|> range(start: -1mo, stop: -1w)
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		yearData = from(bucket: "metrics_monthly")
+		|> range(start: -1y, stop: -1mo)
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		foreverData = from(bucket: "metrics_yearly")
+		|> range(start: -10y, stop: -1y)
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		union(tables: [weekData, monthData, yearData, foreverData])
+		|> sort(columns: ["_time"], desc: false)
+		|> yield(name: "last")
+
+	*/
+
+	partialQueryStr := []string{
+		`import "influxdata/influxdb/schema"`,
+	}
+
+	nestedDurationKeys := sr.lookupNestedDurationKeys(durationKey)
+
+	subQueryNames := []string{}
+	for _, nestedDurationKey := range nestedDurationKeys {
+		bucketName := sr.lookupBucketName(nestedDurationKey)
+		durationRange := sr.lookupDuration(nestedDurationKey)
+
+		subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey))
+		partialQueryStr = append(partialQueryStr, []string{
+			fmt.Sprintf(`%sData = from(bucket: "%s")`, nestedDurationKey, bucketName),
+			fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
+			`|> filter(fn: (r) => r["_measurement"] == "smart" )`,
+			fmt.Sprintf(`|> filter(fn: (r) => r["device_wwn"] == "%s" )`, wwn),
+			"|> schema.fieldsAsCols()",
+		}...)
+	}
+
+	if len(subQueryNames) == 1 {
+		//there's only one bucket being queried, no need to union, just aggregate the dataset and return
+		partialQueryStr = append(partialQueryStr, []string{
+			subQueryNames[0],
+			`|> yield()`,
+		}...)
+	} else {
+		partialQueryStr = append(partialQueryStr, []string{
+			fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")),
+			`|> sort(columns: ["_time"], desc: false)`,
+			`|> yield(name: "last")`,
+		}...)
+	}
+
+	return strings.Join(partialQueryStr, "\n")
+}

webapp/backend/pkg/database/scrutiny_repository_migrations.go

@@ -0,0 +1,451 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"github.com/go-gormigrate/gormigrate/v2"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"gorm.io/gorm"
+	"strconv"
+	"time"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// SQLite migrations
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+//database.AutoMigrate(&models.Device{})
+
+func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
+
+	sr.logger.Infoln("Database migration starting")
+
+	m := gormigrate.New(sr.gormClient, gormigrate.DefaultOptions, []*gormigrate.Migration{
+		{
+			ID: "20201107210306", // v0.3.13 (pre-influxdb schema). 9fac3c6308dc6cb6cd5bbc43a68cd93e8fb20b87
+			Migrate: func(tx *gorm.DB) error {
+				// it's a good practice to copy the struct inside the function,
+
+				return tx.AutoMigrate(
+					&m20201107210306.Device{},
+					&m20201107210306.Smart{},
+					&m20201107210306.SmartAtaAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+				)
+			},
+			Rollback: func(tx *gorm.DB) error {
+				return tx.Migrator().DropTable(
+					&m20201107210306.Device{},
+					&m20201107210306.Smart{},
+					&m20201107210306.SmartAtaAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+					"self_tests",
+				)
+			},
+		},
+		{
+			ID: "20220503113100", // backwards compatible - influxdb schema
+			Migrate: func(tx *gorm.DB) error {
+				// delete unnecessary table.
+				err := tx.Migrator().DropTable("self_tests")
+				if err != nil {
+					return err
+				}
+
+				//add columns to the Device schema, so we can start adding data to the database & influxdb
+				err = tx.Migrator().AddColumn(&models.Device{}, "Label") //Label  string `json:"label"`
+				if err != nil {
+					return err
+				}
+				err = tx.Migrator().AddColumn(&models.Device{}, "DeviceStatus") //DeviceStatus pkg.DeviceStatus `json:"device_status"`
+				if err != nil {
+					return err
+				}
+
+				//TODO: migrate the data from GORM to influxdb.
+				//get a list of all devices:
+				//	get a list of all smart scans in the last 2 weeks:
+				//		get a list of associated smart attribute data:
+				//			translate to a measurements.Smart{} object
+				//			call CUSTOM INFLUXDB SAVE FUNCTION (taking bucket as parameter)
+				//	get a list of all smart scans in the last 9 weeks:
+				//		do same as above (select 1 scan per week)
+				//	get a list of all smart scans in the last 25 months:
+				//		do same as above (select 1 scan per month)
+				//	get a list of all smart scans:
+				//		do same as above (select 1 scan per year)
+
+				preDevices := []m20201107210306.Device{} //pre-migration device information
+				if err = tx.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
+					return db.Order("smarts.created_at ASC") //OLD: .Limit(devicesCount)
+				}).Find(&preDevices).Error; err != nil {
+					sr.logger.Errorln("Could not get device summary from DB", err)
+					return err
+				}
+
+				//calculate bucket oldest dates
+				today := time.Now()
+				dailyBucketMax := today.Add(-RETENTION_PERIOD_15_DAYS_IN_SECONDS * time.Second)     //15 days
+				weeklyBucketMax := today.Add(-RETENTION_PERIOD_9_WEEKS_IN_SECONDS * time.Second)    //9 weeks
+				monthlyBucketMax := today.Add(-RETENTION_PERIOD_25_MONTHS_IN_SECONDS * time.Second) //25 weeks
+
+				for _, preDevice := range preDevices {
+					sr.logger.Debugf("====================================")
+					sr.logger.Infof("begin processing device: %s", preDevice.WWN)
+
+					//weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary.
+					weeklyLookup := map[string]bool{}
+					monthlyLookup := map[string]bool{}
+					yearlyLookup := map[string]bool{}
+					for _, preSmartResult := range preDevice.SmartResults { //pre-migration smart results
+
+						//we're looping in ASC mode, so from oldest entry to most current.
+
+						err, postSmartResults := m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(tx, preDevice, preSmartResult)
+						if err != nil {
+							return err
+						}
+						smartTags, smartFields := postSmartResults.Flatten()
+
+						err, postSmartTemp := m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice, preSmartResult)
+						if err != nil {
+							return err
+						}
+						tempTags, tempFields := postSmartTemp.Flatten()
+						tempTags["device_wwn"] = preDevice.WWN
+
+						year, week := postSmartResults.Date.ISOWeek()
+						month := postSmartResults.Date.Month()
+
+						yearStr := strconv.Itoa(year)
+						yearMonthStr := fmt.Sprintf("%d-%d", year, month)
+						yearWeekStr := fmt.Sprintf("%d-%d", year, week)
+
+						//write data to daily bucket if in the last 15 days
+						if postSmartResults.Date.After(dailyBucketMax) {
+							sr.logger.Debugf("device (%s) smart data added to bucket: daily", preDevice.WWN)
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+
+						//write data to the weekly bucket if in the last 9 weeks, and week has not been processed yet
+						if _, weekExists := weeklyLookup[yearWeekStr]; !weekExists && postSmartResults.Date.After(weeklyBucketMax) {
+							sr.logger.Debugf("device (%s) smart data added to bucket: weekly", preDevice.WWN)
+
+							//this week/year pair has not been processed
+							weeklyLookup[yearWeekStr] = true
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+
+						//write data to the monthly bucket if in the last 9 weeks, and week has not been processed yet
+						if _, monthExists := monthlyLookup[yearMonthStr]; !monthExists && postSmartResults.Date.After(monthlyBucketMax) {
+							sr.logger.Debugf("device (%s) smart data added to bucket: monthly", preDevice.WWN)
+
+							//this month/year pair has not been processed
+							monthlyLookup[yearMonthStr] = true
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+
+						if _, yearExists := yearlyLookup[yearStr]; !yearExists && year != today.Year() {
+							sr.logger.Debugf("device (%s) smart data added to bucket: yearly", preDevice.WWN)
+
+							//this year has not been processed
+							yearlyLookup[yearStr] = true
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+					}
+					sr.logger.Infof("finished processing device %s. weekly: %d, monthly: %d, yearly: %d", preDevice.WWN, len(weeklyLookup), len(monthlyLookup), len(yearlyLookup))
+
+				}
+
+				return nil
+			},
+		},
+		{
+			ID: "20220503120000", // cleanup - v0.4.0 - influxdb schema
+			Migrate: func(tx *gorm.DB) error {
+				// delete unnecessary tables.
+				err := tx.Migrator().DropTable(
+					&m20201107210306.Smart{},
+					&m20201107210306.SmartAtaAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+					&m20201107210306.SmartScsiAttribute{},
+				)
+				if err != nil {
+					return err
+				}
+
+				//migrate the device database to the final version
+				return tx.AutoMigrate(models.Device{})
+			},
+		},
+	})
+
+	if err := m.Migrate(); err != nil {
+		sr.logger.Errorf("Database migration failed with error: %w", err)
+		return err
+	}
+	sr.logger.Infoln("Database migration completed successfully")
+	return nil
+}
+
+// Deprecated
+func m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.SmartTemperature) {
+	//extract temperature data for every datapoint
+	postSmartTemp := measurements.SmartTemperature{
+		Date: preSmartResult.TestDate,
+		Temp: preSmartResult.Temp,
+	}
+
+	return nil, postSmartTemp
+}
+
+// Deprecated
+func m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(database *gorm.DB, preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.Smart) {
+	//create a measurements.Smart object (which we will then push to the InfluxDB)
+	postDeviceSmartData := measurements.Smart{
+		Date:            preSmartResult.TestDate,
+		DeviceWWN:       preDevice.WWN,
+		DeviceProtocol:  preDevice.DeviceProtocol,
+		Temp:            preSmartResult.Temp,
+		PowerOnHours:    preSmartResult.PowerOnHours,
+		PowerCycleCount: preSmartResult.PowerCycleCount,
+
+		// this needs to be populated using measurements.Smart.ProcessAtaSmartInfo, ProcessScsiSmartInfo or ProcessNvmeSmartInfo
+		// because those functions will take into account thresholds (which we didn't consider correctly previously)
+		Attributes: map[string]measurements.SmartAttribute{},
+	}
+
+	result := database.Preload("AtaAttributes").Preload("NvmeAttributes").Preload("ScsiAttributes").Find(&preSmartResult)
+	if result.Error != nil {
+		return result.Error, postDeviceSmartData
+	}
+
+	if preDevice.IsAta() {
+		preAtaSmartAttributesTable := []collector.AtaSmartAttributesTableItem{}
+		for _, preAtaAttribute := range preSmartResult.AtaAttributes {
+			preAtaSmartAttributesTable = append(preAtaSmartAttributesTable, collector.AtaSmartAttributesTableItem{
+				ID:         preAtaAttribute.AttributeId,
+				Name:       preAtaAttribute.Name,
+				Value:      int64(preAtaAttribute.Value),
+				Worst:      int64(preAtaAttribute.Worst),
+				Thresh:     int64(preAtaAttribute.Threshold),
+				WhenFailed: preAtaAttribute.WhenFailed,
+				Flags: struct {
+					Value         int    `json:"value"`
+					String        string `json:"string"`
+					Prefailure    bool   `json:"prefailure"`
+					UpdatedOnline bool   `json:"updated_online"`
+					Performance   bool   `json:"performance"`
+					ErrorRate     bool   `json:"error_rate"`
+					EventCount    bool   `json:"event_count"`
+					AutoKeep      bool   `json:"auto_keep"`
+				}{
+					Value:         0,
+					String:        "",
+					Prefailure:    false,
+					UpdatedOnline: false,
+					Performance:   false,
+					ErrorRate:     false,
+					EventCount:    false,
+					AutoKeep:      false,
+				},
+				Raw: struct {
+					Value  int64  `json:"value"`
+					String string `json:"string"`
+				}{
+					Value:  preAtaAttribute.RawValue,
+					String: preAtaAttribute.RawString,
+				},
+			})
+		}
+
+		postDeviceSmartData.ProcessAtaSmartInfo(preAtaSmartAttributesTable)
+
+	} else if preDevice.IsNvme() {
+		//info collector.SmartInfo
+		postNvmeSmartHealthInformation := collector.NvmeSmartHealthInformationLog{}
+
+		for _, preNvmeAttribute := range preSmartResult.NvmeAttributes {
+			switch preNvmeAttribute.AttributeId {
+			case "critical_warning":
+				postNvmeSmartHealthInformation.CriticalWarning = int64(preNvmeAttribute.Value)
+			case "temperature":
+				postNvmeSmartHealthInformation.Temperature = int64(preNvmeAttribute.Value)
+			case "available_spare":
+				postNvmeSmartHealthInformation.AvailableSpare = int64(preNvmeAttribute.Value)
+			case "available_spare_threshold":
+				postNvmeSmartHealthInformation.AvailableSpareThreshold = int64(preNvmeAttribute.Value)
+			case "percentage_used":
+				postNvmeSmartHealthInformation.PercentageUsed = int64(preNvmeAttribute.Value)
+			case "data_units_read":
+				postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value)
+			case "data_units_written":
+				postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value)
+			case "host_reads":
+				postNvmeSmartHealthInformation.HostReads = int64(preNvmeAttribute.Value)
+			case "host_writes":
+				postNvmeSmartHealthInformation.HostWrites = int64(preNvmeAttribute.Value)
+			case "controller_busy_time":
+				postNvmeSmartHealthInformation.ControllerBusyTime = int64(preNvmeAttribute.Value)
+			case "power_cycles":
+				postNvmeSmartHealthInformation.PowerCycles = int64(preNvmeAttribute.Value)
+			case "power_on_hours":
+				postNvmeSmartHealthInformation.PowerOnHours = int64(preNvmeAttribute.Value)
+			case "unsafe_shutdowns":
+				postNvmeSmartHealthInformation.UnsafeShutdowns = int64(preNvmeAttribute.Value)
+			case "media_errors":
+				postNvmeSmartHealthInformation.MediaErrors = int64(preNvmeAttribute.Value)
+			case "num_err_log_entries":
+				postNvmeSmartHealthInformation.NumErrLogEntries = int64(preNvmeAttribute.Value)
+			case "warning_temp_time":
+				postNvmeSmartHealthInformation.WarningTempTime = int64(preNvmeAttribute.Value)
+			case "critical_comp_time":
+				postNvmeSmartHealthInformation.CriticalCompTime = int64(preNvmeAttribute.Value)
+			}
+		}
+
+		postDeviceSmartData.ProcessNvmeSmartInfo(postNvmeSmartHealthInformation)
+
+	} else if preDevice.IsScsi() {
+		//info collector.SmartInfo
+		var postScsiGrownDefectList int64
+		postScsiErrorCounterLog := collector.ScsiErrorCounterLog{
+			Read: struct {
+				ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+				ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+				ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+				TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+				CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+				GigabytesProcessed               string `json:"gigabytes_processed"`
+				TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+			}{},
+			Write: struct {
+				ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+				ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+				ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+				TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+				CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+				GigabytesProcessed               string `json:"gigabytes_processed"`
+				TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+			}{},
+		}
+
+		for _, preScsiAttribute := range preSmartResult.ScsiAttributes {
+			switch preScsiAttribute.AttributeId {
+			case "scsi_grown_defect_list":
+				postScsiGrownDefectList = int64(preScsiAttribute.Value)
+			case "read.errors_corrected_by_eccfast":
+				postScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value)
+			case "read.errors_corrected_by_eccdelayed":
+				postScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value)
+			case "read.errors_corrected_by_rereads_rewrites":
+				postScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value)
+			case "read.total_errors_corrected":
+				postScsiErrorCounterLog.Read.TotalErrorsCorrected = int64(preScsiAttribute.Value)
+			case "read.correction_algorithm_invocations":
+				postScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value)
+			case "read.total_uncorrected_errors":
+				postScsiErrorCounterLog.Read.TotalUncorrectedErrors = int64(preScsiAttribute.Value)
+			case "write.errors_corrected_by_eccfast":
+				postScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value)
+			case "write.errors_corrected_by_eccdelayed":
+				postScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value)
+			case "write.errors_corrected_by_rereads_rewrites":
+				postScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value)
+			case "write.total_errors_corrected":
+				postScsiErrorCounterLog.Write.TotalErrorsCorrected = int64(preScsiAttribute.Value)
+			case "write.correction_algorithm_invocations":
+				postScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value)
+			case "write.total_uncorrected_errors":
+				postScsiErrorCounterLog.Write.TotalUncorrectedErrors = int64(preScsiAttribute.Value)
+			}
+		}
+		postDeviceSmartData.ProcessScsiSmartInfo(postScsiGrownDefectList, postScsiErrorCounterLog)
+	} else {
+		return fmt.Errorf("Unknown device protocol: %s", preDevice.DeviceProtocol), postDeviceSmartData
+	}
+
+	return nil, postDeviceSmartData
+}

webapp/backend/pkg/database/scrutiny_repository_tasks.go

@@ -0,0 +1,122 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/influxdata/influxdb-client-go/v2/api"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Tasks
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) error {
+	weeklyTaskName := "tsk-weekly-aggr"
+	if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: weeklyTaskName}); findErr == nil && len(found) == 0 {
+		//weekly on Sunday at 1:00am
+		_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, weeklyTaskName, sr.DownsampleScript("weekly"), "0 1 * * 0", orgID)
+		if err != nil {
+			return err
+		}
+	}
+
+	monthlyTaskName := "tsk-monthly-aggr"
+	if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: monthlyTaskName}); findErr == nil && len(found) == 0 {
+		//monthly on first day of the month at 1:30am
+		_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, monthlyTaskName, sr.DownsampleScript("monthly"), "30 1 1 * *", orgID)
+		if err != nil {
+			return err
+		}
+	}
+
+	yearlyTaskName := "tsk-yearly-aggr"
+	if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: yearlyTaskName}); findErr == nil && len(found) == 0 {
+		//yearly on the first day of the year at 2:00am
+		_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, yearlyTaskName, sr.DownsampleScript("yearly"), "0 2 1 1 *", orgID)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
+	var sourceBucket string // the source of the data
+	var destBucket string   // the destination for the aggregated data
+	var rangeStart string
+	var rangeEnd string
+	var aggWindow string
+	switch aggregationType {
+	case "weekly":
+		sourceBucket = sr.appConfig.GetString("web.influxdb.bucket")
+		destBucket = fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+		rangeStart = "-2w"
+		rangeEnd = "-1w"
+		aggWindow = "1w"
+	case "monthly":
+		sourceBucket = fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+		destBucket = fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+		rangeStart = "-2mo"
+		rangeEnd = "-1mo"
+		aggWindow = "1mo"
+	case "yearly":
+		sourceBucket = fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+		destBucket = fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
+		rangeStart = "-2y"
+		rangeEnd = "-1y"
+		aggWindow = "1y"
+	}
+
+	// TODO: using "last" function for aggregation. This should eventually be replaced with a more accurate represenation
+	/*
+	  import "types"
+	  smart_data = from(bucket: sourceBucket)
+	  |> range(start: rangeStart, stop: rangeEnd)
+	  |> filter(fn: (r) => r["_measurement"] == "smart" )
+	  |> group(columns: ["device_wwn", "_field"])
+
+	  non_numeric_smart_data = smart_data
+	    |> filter(fn: (r) => types.isType(v: r._value, type: "string") or types.isType(v: r._value, type: "bool"))
+	    |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
+
+	  numeric_smart_data = smart_data
+	    |> filter(fn: (r) => types.isType(v: r._value, type: "int") or types.isType(v: r._value, type: "float"))
+	    |> aggregateWindow(every: aggWindow, fn: mean, createEmpty: false)
+
+	  union(tables: [non_numeric_smart_data, numeric_smart_data])
+	  |> to(bucket: destBucket, org: destOrg)
+
+	*/
+
+	return fmt.Sprintf(`
+  sourceBucket = "%s"
+  rangeStart = %s
+  rangeEnd = %s
+  aggWindow = %s
+  destBucket = "%s"
+  destOrg = "%s"
+
+  from(bucket: sourceBucket)
+  |> range(start: rangeStart, stop: rangeEnd)
+  |> filter(fn: (r) => r["_measurement"] == "smart" )
+  |> group(columns: ["device_wwn", "_field"])
+  |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
+  |> to(bucket: destBucket, org: destOrg)
+
+  temp_data = from(bucket: sourceBucket)
+  |> range(start: rangeStart, stop: rangeEnd)
+  |> filter(fn: (r) => r["_measurement"] == "temp")
+  |> group(columns: ["device_wwn"])
+  |> toInt()
+
+  temp_data
+  |> aggregateWindow(fn: mean, every: aggWindow)
+  |> to(bucket: destBucket, org: destOrg)
+		`,
+		sourceBucket,
+		rangeStart,
+		rangeEnd,
+		aggWindow,
+		destBucket,
+		sr.appConfig.GetString("web.influxdb.org"),
+	)
+}

webapp/backend/pkg/database/scrutiny_repository_temperature.go

@@ -0,0 +1,167 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"strings"
+	"time"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Temperature Data
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error {
+	if len(collectorSmartData.AtaSctTemperatureHistory.Table) > 0 {
+
+		for ndx, temp := range collectorSmartData.AtaSctTemperatureHistory.Table {
+
+			minutesOffset := collectorSmartData.AtaSctTemperatureHistory.LoggingIntervalMinutes * int64(ndx) * 60
+			smartTemp := measurements.SmartTemperature{
+				Date: time.Unix(collectorSmartData.LocalTime.TimeT-minutesOffset, 0),
+				Temp: temp,
+			}
+
+			tags, fields := smartTemp.Flatten()
+			tags["device_wwn"] = wwn
+			p := influxdb2.NewPoint("temp",
+				tags,
+				fields,
+				smartTemp.Date)
+			err := sr.influxWriteApi.WritePoint(ctx, p)
+			if err != nil {
+				return err
+			}
+		}
+		// also add the current temperature.
+	} else {
+
+		smartTemp := measurements.SmartTemperature{
+			Date: time.Unix(collectorSmartData.LocalTime.TimeT, 0),
+			Temp: collectorSmartData.Temperature.Current,
+		}
+
+		tags, fields := smartTemp.Flatten()
+		tags["device_wwn"] = wwn
+		p := influxdb2.NewPoint("temp",
+			tags,
+			fields,
+			smartTemp.Date)
+		return sr.influxWriteApi.WritePoint(ctx, p)
+	}
+	return nil
+}
+
+func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) {
+	//we can get temp history for "week", "month", DURATION_KEY_YEAR, "forever"
+
+	deviceTempHistory := map[string][]measurements.SmartTemperature{}
+
+	//TODO: change the query range to a variable.
+	queryStr := sr.aggregateTempQuery(durationKey)
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+
+			if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
+
+				//check if deviceWWN has been seen and initialized already
+				if _, ok := deviceTempHistory[deviceWWN.(string)]; !ok {
+					deviceTempHistory[deviceWWN.(string)] = []measurements.SmartTemperature{}
+				}
+
+				currentTempHistory := deviceTempHistory[deviceWWN.(string)]
+				smartTemp := measurements.SmartTemperature{}
+
+				for key, val := range result.Record().Values() {
+					smartTemp.Inflate(key, val)
+				}
+				smartTemp.Date = result.Record().Values()["_time"].(time.Time)
+				currentTempHistory = append(currentTempHistory, smartTemp)
+				deviceTempHistory[deviceWWN.(string)] = currentTempHistory
+			}
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+	return deviceTempHistory, nil
+
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Helper Methods
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+func (sr *scrutinyRepository) aggregateTempQuery(durationKey string) string {
+
+	/*
+		import "influxdata/influxdb/schema"
+		weekData = from(bucket: "metrics")
+		  |> range(start: -1w, stop: now())
+		  |> filter(fn: (r) => r["_measurement"] == "temp" )
+		  |> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+		  |> group(columns: ["device_wwn"])
+		  |> toInt()
+
+		monthData = from(bucket: "metrics_weekly")
+		  |> range(start: -1mo, stop: now())
+		  |> filter(fn: (r) => r["_measurement"] == "temp" )
+		  |> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+		  |> group(columns: ["device_wwn"])
+		  |> toInt()
+
+		union(tables: [weekData, monthData])
+		  |> group(columns: ["device_wwn"])
+		  |> sort(columns: ["_time"], desc: false)
+		  |> schema.fieldsAsCols()
+
+	*/
+
+	partialQueryStr := []string{
+		`import "influxdata/influxdb/schema"`,
+	}
+
+	nestedDurationKeys := sr.lookupNestedDurationKeys(durationKey)
+
+	subQueryNames := []string{}
+	for _, nestedDurationKey := range nestedDurationKeys {
+		bucketName := sr.lookupBucketName(nestedDurationKey)
+		durationRange := sr.lookupDuration(nestedDurationKey)
+
+		subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey))
+		partialQueryStr = append(partialQueryStr, []string{
+			fmt.Sprintf(`%sData = from(bucket: "%s")`, nestedDurationKey, bucketName),
+			fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
+			`|> filter(fn: (r) => r["_measurement"] == "temp" )`,
+			`|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)`,
+			`|> group(columns: ["device_wwn"])`,
+			`|> toInt()`,
+			"",
+		}...)
+	}
+
+	if len(subQueryNames) == 1 {
+		//there's only one bucket being queried, no need to union, just aggregate the dataset and return
+		partialQueryStr = append(partialQueryStr, []string{
+			subQueryNames[0],
+			"|> schema.fieldsAsCols()",
+			"|> yield()",
+		}...)
+	} else {
+		partialQueryStr = append(partialQueryStr, []string{
+			fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")),
+			`|> group(columns: ["device_wwn"])`,
+			`|> sort(columns: ["_time"], desc: false)`,
+			"|> schema.fieldsAsCols()",
+		}...)
+	}
+
+	return strings.Join(partialQueryStr, "\n")
+}

webapp/backend/pkg/models/collector/smart.go

@@ -119,30 +119,23 @@ type SmartInfo struct {
 		FeatureControlSupported       bool `json:"feature_control_supported"`
 		DataTableSupported            bool `json:"data_table_supported"`
 	} `json:"ata_sct_capabilities"`
+	AtaSctTemperatureHistory struct {
+		Version                int   `json:"version"`
+		SamplingPeriodMinutes  int64 `json:"sampling_period_minutes"`
+		LoggingIntervalMinutes int64 `json:"logging_interval_minutes"`
+		Temperature            struct {
+			OpLimitMin int `json:"op_limit_min"`
+			OpLimitMax int `json:"op_limit_max"`
+			LimitMin   int `json:"limit_min"`
+			LimitMax   int `json:"limit_max"`
+		} `json:"temperature"`
+		Size  int     `json:"size"`
+		Index int     `json:"index"`
+		Table []int64 `json:"table"`
+	} `json:"ata_sct_temperature_history"`
 	AtaSmartAttributes struct {
-		Revision int `json:"revision"`
-		Table    []struct {
-			ID         int    `json:"id"`
-			Name       string `json:"name"`
-			Value      int    `json:"value"`
-			Worst      int    `json:"worst"`
-			Thresh     int    `json:"thresh"`
-			WhenFailed string `json:"when_failed"`
-			Flags      struct {
-				Value         int    `json:"value"`
-				String        string `json:"string"`
-				Prefailure    bool   `json:"prefailure"`
-				UpdatedOnline bool   `json:"updated_online"`
-				Performance   bool   `json:"performance"`
-				ErrorRate     bool   `json:"error_rate"`
-				EventCount    bool   `json:"event_count"`
-				AutoKeep      bool   `json:"auto_keep"`
-			} `json:"flags"`
-			Raw struct {
-				Value  int64  `json:"value"`
-				String string `json:"string"`
-			} `json:"raw"`
-		} `json:"table"`
+		Revision int                           `json:"revision"`
+		Table    []AtaSmartAttributesTableItem `json:"table"`
 	} `json:"ata_smart_attributes"`
 	AtaSmartErrorLog struct {
 		Summary struct {
@@ -236,49 +229,77 @@ type SmartInfo struct {
 		} `json:"utilization"`
 		FormattedLbaSize int `json:"formatted_lba_size"`
 	} `json:"nvme_namespaces"`
-	NvmeSmartHealthInformationLog struct {
-		CriticalWarning         int `json:"critical_warning"`
-		Temperature             int `json:"temperature"`
-		AvailableSpare          int `json:"available_spare"`
-		AvailableSpareThreshold int `json:"available_spare_threshold"`
-		PercentageUsed          int `json:"percentage_used"`
-		DataUnitsRead           int `json:"data_units_read"`
-		DataUnitsWritten        int `json:"data_units_written"`
-		HostReads               int `json:"host_reads"`
-		HostWrites              int `json:"host_writes"`
-		ControllerBusyTime      int `json:"controller_busy_time"`
-		PowerCycles             int `json:"power_cycles"`
-		PowerOnHours            int `json:"power_on_hours"`
-		UnsafeShutdowns         int `json:"unsafe_shutdowns"`
-		MediaErrors             int `json:"media_errors"`
-		NumErrLogEntries        int `json:"num_err_log_entries"`
-		WarningTempTime         int `json:"warning_temp_time"`
-		CriticalCompTime        int `json:"critical_comp_time"`
-	} `json:"nvme_smart_health_information_log"`
+	NvmeSmartHealthInformationLog NvmeSmartHealthInformationLog `json:"nvme_smart_health_information_log"`
 
 	// SCSI Protocol Specific Fields
-	Vendor              string `json:"vendor"`
-	Product             string `json:"product"`
-	ScsiVersion         string `json:"scsi_version"`
-	ScsiGrownDefectList int    `json:"scsi_grown_defect_list"`
-	ScsiErrorCounterLog struct {
-		Read struct {
-			ErrorsCorrectedByEccfast         int    `json:"errors_corrected_by_eccfast"`
-			ErrorsCorrectedByEccdelayed      int    `json:"errors_corrected_by_eccdelayed"`
-			ErrorsCorrectedByRereadsRewrites int    `json:"errors_corrected_by_rereads_rewrites"`
-			TotalErrorsCorrected             int    `json:"total_errors_corrected"`
-			CorrectionAlgorithmInvocations   int    `json:"correction_algorithm_invocations"`
-			GigabytesProcessed               string `json:"gigabytes_processed"`
-			TotalUncorrectedErrors           int    `json:"total_uncorrected_errors"`
-		} `json:"read"`
-		Write struct {
-			ErrorsCorrectedByEccfast         int    `json:"errors_corrected_by_eccfast"`
-			ErrorsCorrectedByEccdelayed      int    `json:"errors_corrected_by_eccdelayed"`
-			ErrorsCorrectedByRereadsRewrites int    `json:"errors_corrected_by_rereads_rewrites"`
-			TotalErrorsCorrected             int    `json:"total_errors_corrected"`
-			CorrectionAlgorithmInvocations   int    `json:"correction_algorithm_invocations"`
-			GigabytesProcessed               string `json:"gigabytes_processed"`
-			TotalUncorrectedErrors           int    `json:"total_uncorrected_errors"`
-		} `json:"write"`
-	} `json:"scsi_error_counter_log"`
+	Vendor              string              `json:"vendor"`
+	Product             string              `json:"product"`
+	ScsiVersion         string              `json:"scsi_version"`
+	ScsiGrownDefectList int64               `json:"scsi_grown_defect_list"`
+	ScsiErrorCounterLog ScsiErrorCounterLog `json:"scsi_error_counter_log"`
+}
+
+//Primary Attribute Structs
+type AtaSmartAttributesTableItem struct {
+	ID         int    `json:"id"`
+	Name       string `json:"name"`
+	Value      int64  `json:"value"`
+	Worst      int64  `json:"worst"`
+	Thresh     int64  `json:"thresh"`
+	WhenFailed string `json:"when_failed"`
+	Flags      struct {
+		Value         int    `json:"value"`
+		String        string `json:"string"`
+		Prefailure    bool   `json:"prefailure"`
+		UpdatedOnline bool   `json:"updated_online"`
+		Performance   bool   `json:"performance"`
+		ErrorRate     bool   `json:"error_rate"`
+		EventCount    bool   `json:"event_count"`
+		AutoKeep      bool   `json:"auto_keep"`
+	} `json:"flags"`
+	Raw struct {
+		Value  int64  `json:"value"`
+		String string `json:"string"`
+	} `json:"raw"`
+}
+
+type NvmeSmartHealthInformationLog struct {
+	CriticalWarning         int64 `json:"critical_warning"`
+	Temperature             int64 `json:"temperature"`
+	AvailableSpare          int64 `json:"available_spare"`
+	AvailableSpareThreshold int64 `json:"available_spare_threshold"`
+	PercentageUsed          int64 `json:"percentage_used"`
+	DataUnitsRead           int64 `json:"data_units_read"`
+	DataUnitsWritten        int64 `json:"data_units_written"`
+	HostReads               int64 `json:"host_reads"`
+	HostWrites              int64 `json:"host_writes"`
+	ControllerBusyTime      int64 `json:"controller_busy_time"`
+	PowerCycles             int64 `json:"power_cycles"`
+	PowerOnHours            int64 `json:"power_on_hours"`
+	UnsafeShutdowns         int64 `json:"unsafe_shutdowns"`
+	MediaErrors             int64 `json:"media_errors"`
+	NumErrLogEntries        int64 `json:"num_err_log_entries"`
+	WarningTempTime         int64 `json:"warning_temp_time"`
+	CriticalCompTime        int64 `json:"critical_comp_time"`
+}
+
+type ScsiErrorCounterLog struct {
+	Read struct {
+		ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+		ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+		ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+		TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+		CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+		GigabytesProcessed               string `json:"gigabytes_processed"`
+		TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+	} `json:"read"`
+	Write struct {
+		ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+		ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+		ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+		TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+		CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+		GigabytesProcessed               string `json:"gigabytes_processed"`
+		TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+	} `json:"write"`
 }

webapp/backend/pkg/models/db/device.go

@@ -1,160 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"time"
-)
-
-type DeviceWrapper struct {
-	Success bool     `json:"success"`
-	Errors  []error  `json:"errors"`
-	Data    []Device `json:"data"`
-}
-
-const DeviceProtocolAta = "ATA"
-const DeviceProtocolScsi = "SCSI"
-const DeviceProtocolNvme = "NVMe"
-
-type Device struct {
-	//GORM attributes, see: http://gorm.io/docs/conventions.html
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt *time.Time
-
-	WWN    string `json:"wwn" gorm:"primary_key"`
-	HostId string `json:"host_id"`
-
-	DeviceName     string  `json:"device_name"`
-	Manufacturer   string  `json:"manufacturer"`
-	ModelName      string  `json:"model_name"`
-	InterfaceType  string  `json:"interface_type"`
-	InterfaceSpeed string  `json:"interface_speed"`
-	SerialNumber   string  `json:"serial_number"`
-	Firmware       string  `json:"firmware"`
-	RotationSpeed  int     `json:"rotational_speed"`
-	Capacity       int64   `json:"capacity"`
-	FormFactor     string  `json:"form_factor"`
-	SmartSupport   bool    `json:"smart_support"`
-	DeviceProtocol string  `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
-	DeviceType     string  `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
-	SmartResults   []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
-}
-
-func (dv *Device) IsAta() bool {
-	return dv.DeviceProtocol == DeviceProtocolAta
-}
-
-func (dv *Device) IsScsi() bool {
-	return dv.DeviceProtocol == DeviceProtocolScsi
-}
-
-func (dv *Device) IsNvme() bool {
-	return dv.DeviceProtocol == DeviceProtocolNvme
-}
-
-//This method requires a device with an array of SmartResults.
-//It will remove all SmartResults other than the first (the latest one)
-//All removed SmartResults, will be processed, grouping SmartAtaAttribute by attribute_id
-// and adding theme to an array called History.
-func (dv *Device) SquashHistory() error {
-	if len(dv.SmartResults) <= 1 {
-		return nil //no ataHistory found. ignore
-	}
-
-	latestSmartResultSlice := dv.SmartResults[0:1]
-	historicalSmartResultSlice := dv.SmartResults[1:]
-
-	//re-assign the latest slice to the SmartResults field
-	dv.SmartResults = latestSmartResultSlice
-
-	//process the historical slice for ATA data
-	if len(dv.SmartResults[0].AtaAttributes) > 0 {
-		ataHistory := map[int][]SmartAtaAttribute{}
-		for _, smartResult := range historicalSmartResultSlice {
-			for _, smartAttribute := range smartResult.AtaAttributes {
-				if _, ok := ataHistory[smartAttribute.AttributeId]; !ok {
-					ataHistory[smartAttribute.AttributeId] = []SmartAtaAttribute{}
-				}
-				ataHistory[smartAttribute.AttributeId] = append(ataHistory[smartAttribute.AttributeId], smartAttribute)
-			}
-		}
-
-		//now assign the historical slices to the AtaAttributes in the latest SmartResults
-		for sandx, smartAttribute := range dv.SmartResults[0].AtaAttributes {
-			if attributeHistory, ok := ataHistory[smartAttribute.AttributeId]; ok {
-				dv.SmartResults[0].AtaAttributes[sandx].History = attributeHistory
-			}
-		}
-	}
-
-	//process the historical slice for Nvme data
-	if len(dv.SmartResults[0].NvmeAttributes) > 0 {
-		nvmeHistory := map[string][]SmartNvmeAttribute{}
-		for _, smartResult := range historicalSmartResultSlice {
-			for _, smartAttribute := range smartResult.NvmeAttributes {
-				if _, ok := nvmeHistory[smartAttribute.AttributeId]; !ok {
-					nvmeHistory[smartAttribute.AttributeId] = []SmartNvmeAttribute{}
-				}
-				nvmeHistory[smartAttribute.AttributeId] = append(nvmeHistory[smartAttribute.AttributeId], smartAttribute)
-			}
-		}
-
-		//now assign the historical slices to the AtaAttributes in the latest SmartResults
-		for sandx, smartAttribute := range dv.SmartResults[0].NvmeAttributes {
-			if attributeHistory, ok := nvmeHistory[smartAttribute.AttributeId]; ok {
-				dv.SmartResults[0].NvmeAttributes[sandx].History = attributeHistory
-			}
-		}
-	}
-	//process the historical slice for Scsi data
-	if len(dv.SmartResults[0].ScsiAttributes) > 0 {
-		scsiHistory := map[string][]SmartScsiAttribute{}
-		for _, smartResult := range historicalSmartResultSlice {
-			for _, smartAttribute := range smartResult.ScsiAttributes {
-				if _, ok := scsiHistory[smartAttribute.AttributeId]; !ok {
-					scsiHistory[smartAttribute.AttributeId] = []SmartScsiAttribute{}
-				}
-				scsiHistory[smartAttribute.AttributeId] = append(scsiHistory[smartAttribute.AttributeId], smartAttribute)
-			}
-		}
-
-		//now assign the historical slices to the AtaAttributes in the latest SmartResults
-		for sandx, smartAttribute := range dv.SmartResults[0].ScsiAttributes {
-			if attributeHistory, ok := scsiHistory[smartAttribute.AttributeId]; ok {
-				dv.SmartResults[0].ScsiAttributes[sandx].History = attributeHistory
-			}
-		}
-	}
-	return nil
-}
-
-func (dv *Device) ApplyMetadataRules() error {
-
-	//embed metadata in the latest smart attributes object
-	if len(dv.SmartResults) > 0 {
-		for ndx, attr := range dv.SmartResults[0].AtaAttributes {
-			attr.PopulateAttributeStatus()
-			dv.SmartResults[0].AtaAttributes[ndx] = attr
-		}
-
-		for ndx, attr := range dv.SmartResults[0].NvmeAttributes {
-			attr.PopulateAttributeStatus()
-			dv.SmartResults[0].NvmeAttributes[ndx] = attr
-
-		}
-
-		for ndx, attr := range dv.SmartResults[0].ScsiAttributes {
-			attr.PopulateAttributeStatus()
-			dv.SmartResults[0].ScsiAttributes[ndx] = attr
-
-		}
-	}
-	return nil
-}
-
-// This function is called every time the collector sends SMART data to the API.
-// It can be used to update device data that can change over time.
-func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
-	dv.Firmware = info.FirmwareVersion
-	return nil
-}

webapp/backend/pkg/models/db/selftest.go

@@ -1,15 +0,0 @@
-package db
-
-import "time"
-
-type SelfTest struct {
-	//GORM attributes, see: http://gorm.io/docs/conventions.html
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt *time.Time
-
-	DeviceWWN string
-	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
-
-	Date time.Time
-}

webapp/backend/pkg/models/db/smart.go

@@ -1,127 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"gorm.io/gorm"
-	"time"
-)
-
-const SmartWhenFailedFailingNow = "FAILING_NOW"
-const SmartWhenFailedInThePast = "IN_THE_PAST"
-
-const SmartStatusPassed = "passed"
-const SmartStatusFailed = "failed"
-
-type Smart struct {
-	gorm.Model
-
-	DeviceWWN string `json:"device_wwn"`
-	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
-
-	TestDate    time.Time `json:"date"`
-	SmartStatus string    `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed
-
-	//Metrics
-	Temp            int64 `json:"temp"`
-	PowerOnHours    int64 `json:"power_on_hours"`
-	PowerCycleCount int64 `json:"power_cycle_count"`
-
-	AtaAttributes  []SmartAtaAttribute  `json:"ata_attributes" gorm:"foreignkey:SmartId"`
-	NvmeAttributes []SmartNvmeAttribute `json:"nvme_attributes" gorm:"foreignkey:SmartId"`
-	ScsiAttributes []SmartScsiAttribute `json:"scsi_attributes" gorm:"foreignkey:SmartId"`
-}
-
-//Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries)
-func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error {
-	sm.DeviceWWN = wwn
-	sm.TestDate = time.Unix(info.LocalTime.TimeT, 0)
-
-	//smart metrics
-	sm.Temp = info.Temperature.Current
-	sm.PowerCycleCount = info.PowerCycleCount
-	sm.PowerOnHours = info.PowerOnTime.Hours
-
-	// process ATA/NVME/SCSI protocol data
-	if info.Device.Protocol == DeviceProtocolAta {
-		sm.ProcessAtaSmartInfo(info)
-	} else if info.Device.Protocol == DeviceProtocolNvme {
-		sm.ProcessNvmeSmartInfo(info)
-	} else if info.Device.Protocol == DeviceProtocolScsi {
-		sm.ProcessScsiSmartInfo(info)
-	}
-
-	if info.SmartStatus.Passed {
-		sm.SmartStatus = SmartStatusPassed
-	} else {
-		sm.SmartStatus = SmartStatusFailed
-	}
-	return nil
-}
-
-//generate SmartAtaAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) {
-	sm.AtaAttributes = []SmartAtaAttribute{}
-	for _, collectorAttr := range info.AtaSmartAttributes.Table {
-		attrModel := SmartAtaAttribute{
-			AttributeId: collectorAttr.ID,
-			Name:        collectorAttr.Name,
-			Value:       collectorAttr.Value,
-			Worst:       collectorAttr.Worst,
-			Threshold:   collectorAttr.Thresh,
-			RawValue:    collectorAttr.Raw.Value,
-			RawString:   collectorAttr.Raw.String,
-			WhenFailed:  collectorAttr.WhenFailed,
-		}
-
-		//now that we've parsed the data from the smartctl response, lets match it against our metadata rules and add additional Scrutiny specific data.
-		if smartMetadata, ok := metadata.AtaMetadata[collectorAttr.ID]; ok {
-			attrModel.Name = smartMetadata.DisplayName
-			if smartMetadata.Transform != nil {
-				attrModel.TransformedValue = smartMetadata.Transform(attrModel.Value, attrModel.RawValue, attrModel.RawString)
-			}
-		}
-		sm.AtaAttributes = append(sm.AtaAttributes, attrModel)
-	}
-}
-
-//generate SmartNvmeAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) {
-	sm.NvmeAttributes = []SmartNvmeAttribute{
-		{AttributeId: "critical_warning", Name: "Critical Warning", Value: info.NvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0},
-		{AttributeId: "temperature", Name: "Temperature", Value: info.NvmeSmartHealthInformationLog.Temperature, Threshold: -1},
-		{AttributeId: "available_spare", Name: "Available Spare", Value: info.NvmeSmartHealthInformationLog.AvailableSpare, Threshold: info.NvmeSmartHealthInformationLog.AvailableSpareThreshold},
-		{AttributeId: "percentage_used", Name: "Percentage Used", Value: info.NvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100},
-		{AttributeId: "data_units_read", Name: "Data Units Read", Value: info.NvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1},
-		{AttributeId: "data_units_written", Name: "Data Units Written", Value: info.NvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1},
-		{AttributeId: "host_reads", Name: "Host Reads", Value: info.NvmeSmartHealthInformationLog.HostReads, Threshold: -1},
-		{AttributeId: "host_writes", Name: "Host Writes", Value: info.NvmeSmartHealthInformationLog.HostWrites, Threshold: -1},
-		{AttributeId: "controller_busy_time", Name: "Controller Busy Time", Value: info.NvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1},
-		{AttributeId: "power_cycles", Name: "Power Cycles", Value: info.NvmeSmartHealthInformationLog.PowerCycles, Threshold: -1},
-		{AttributeId: "power_on_hours", Name: "Power on Hours", Value: info.NvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1},
-		{AttributeId: "unsafe_shutdowns", Name: "Unsafe Shutdowns", Value: info.NvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1},
-		{AttributeId: "media_errors", Name: "Media Errors", Value: info.NvmeSmartHealthInformationLog.MediaErrors, Threshold: 0},
-		{AttributeId: "num_err_log_entries", Name: "Numb Err Log Entries", Value: info.NvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0},
-		{AttributeId: "warning_temp_time", Name: "Warning Temp Time", Value: info.NvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1},
-		{AttributeId: "critical_comp_time", Name: "Critical CompTime", Value: info.NvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1},
-	}
-}
-
-//generate SmartScsiAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessScsiSmartInfo(info collector.SmartInfo) {
-	sm.ScsiAttributes = []SmartScsiAttribute{
-		{AttributeId: "scsi_grown_defect_list", Name: "Grown Defect List", Value: info.ScsiGrownDefectList, Threshold: 0},
-		{AttributeId: "read.errors_corrected_by_eccfast", Name: "Read Errors Corrected by ECC Fast", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1},
-		{AttributeId: "read.errors_corrected_by_eccdelayed", Name: "Read Errors Corrected by ECC Delayed", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1},
-		{AttributeId: "read.errors_corrected_by_rereads_rewrites", Name: "Read Errors Corrected by ReReads/ReWrites", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0},
-		{AttributeId: "read.total_errors_corrected", Name: "Read Total Errors Corrected", Value: info.ScsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1},
-		{AttributeId: "read.correction_algorithm_invocations", Name: "Read Correction Algorithm Invocations", Value: info.ScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1},
-		{AttributeId: "read.total_uncorrected_errors", Name: "Read Total Uncorrected Errors", Value: info.ScsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0},
-		{AttributeId: "write.errors_corrected_by_eccfast", Name: "Write Errors Corrected by ECC Fast", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1},
-		{AttributeId: "write.errors_corrected_by_eccdelayed", Name: "Write Errors Corrected by ECC Delayed", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1},
-		{AttributeId: "write.errors_corrected_by_rereads_rewrites", Name: "Write Errors Corrected by ReReads/ReWrites", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0},
-		{AttributeId: "write.total_errors_corrected", Name: "Write Total Errors Corrected", Value: info.ScsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1},
-		{AttributeId: "write.correction_algorithm_invocations", Name: "Write Correction Algorithm Invocations", Value: info.ScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1},
-		{AttributeId: "write.total_uncorrected_errors", Name: "Write Total Uncorrected Errors", Value: info.ScsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0},
-	}
-}

webapp/backend/pkg/models/db/smart_ata_attribute.go

@@ -1,111 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"gorm.io/gorm"
-	"strings"
-)
-
-const SmartAttributeStatusPassed = "passed"
-const SmartAttributeStatusFailed = "failed"
-const SmartAttributeStatusWarning = "warn"
-
-type SmartAtaAttribute struct {
-	gorm.Model
-
-	SmartId int    `json:"smart_id"`
-	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
-
-	AttributeId int    `json:"attribute_id"`
-	Name        string `json:"name"`
-	Value       int    `json:"value"`
-	Worst       int    `json:"worst"`
-	Threshold   int    `json:"thresh"`
-	RawValue    int64  `json:"raw_value"`
-	RawString   string `json:"raw_string"`
-	WhenFailed  string `json:"when_failed"`
-
-	TransformedValue int64               `json:"transformed_value"`
-	Status           string              `gorm:"-" json:"status,omitempty"`
-	StatusReason     string              `gorm:"-" json:"status_reason,omitempty"`
-	FailureRate      float64             `gorm:"-" json:"failure_rate,omitempty"`
-	History          []SmartAtaAttribute `gorm:"-" json:"history,omitempty"`
-}
-
-//populate attribute status, using SMART Thresholds & Observed Metadata
-func (sa *SmartAtaAttribute) PopulateAttributeStatus() {
-	if strings.ToUpper(sa.WhenFailed) == SmartWhenFailedFailingNow {
-		//this attribute has previously failed
-		sa.Status = SmartAttributeStatusFailed
-		sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
-
-	} else if strings.ToUpper(sa.WhenFailed) == SmartWhenFailedInThePast {
-		sa.Status = SmartAttributeStatusWarning
-		sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
-	}
-
-	if smartMetadata, ok := metadata.AtaMetadata[sa.AttributeId]; ok {
-		sa.MetadataObservedThresholdStatus(smartMetadata)
-	}
-
-	//check if status is blank, set to "passed"
-	if len(sa.Status) == 0 {
-		sa.Status = SmartAttributeStatusPassed
-	}
-}
-
-// compare the attribute (raw, normalized, transformed) value to observed thresholds, and update status if necessary
-func (sa *SmartAtaAttribute) MetadataObservedThresholdStatus(smartMetadata metadata.AtaAttributeMetadata) {
-	//TODO: multiple rules
-	// try to predict the failure rates for observed thresholds that have 0 failure rate and error bars.
-	// - if the attribute is critical
-	//		- the failure rate is over 10 - set to failed
-	//		- the attribute does not match any threshold, set to warn
-	// - if the attribute is not critical
-	//		- if failure rate is above 20 - set to failed
-	// 		- if failure rate is above 10 but below 20 - set to warn
-
-	//update the smart attribute status based on Observed thresholds.
-	var value int64
-	if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeNormalized {
-		value = int64(sa.Value)
-	} else if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeTransformed {
-		value = sa.TransformedValue
-	} else {
-		value = sa.RawValue
-	}
-
-	for _, obsThresh := range smartMetadata.ObservedThresholds {
-
-		//check if "value" is in this bucket
-		if ((obsThresh.Low == obsThresh.High) && value == obsThresh.Low) ||
-			(obsThresh.Low < value && value <= obsThresh.High) {
-			sa.FailureRate = obsThresh.AnnualFailureRate
-
-			if smartMetadata.Critical {
-				if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = SmartAttributeStatusFailed
-					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
-				}
-			} else {
-				if obsThresh.AnnualFailureRate >= 0.20 {
-					sa.Status = SmartAttributeStatusFailed
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
-				} else if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = SmartAttributeStatusWarning
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
-				}
-			}
-
-			//we've found the correct bucket, we can drop out of this loop
-			return
-		}
-	}
-	// no bucket found
-	if smartMetadata.Critical {
-		sa.Status = SmartAttributeStatusWarning
-		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
-	}
-
-	return
-}

webapp/backend/pkg/models/db/smart_nvme_attribute.go

@@ -1,46 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"gorm.io/gorm"
-)
-
-type SmartNvmeAttribute struct {
-	gorm.Model
-
-	SmartId int    `json:"smart_id"`
-	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
-
-	AttributeId string `json:"attribute_id"` //json string from smartctl
-	Name        string `json:"name"`
-	Value       int    `json:"value"`
-	Threshold   int    `json:"thresh"`
-
-	TransformedValue int64                `json:"transformed_value"`
-	Status           string               `gorm:"-" json:"status,omitempty"`
-	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
-	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
-	History          []SmartNvmeAttribute `gorm:"-" json:"history,omitempty"`
-}
-
-//populate attribute status, using SMART Thresholds & Observed Metadata
-func (sa *SmartNvmeAttribute) PopulateAttributeStatus() {
-
-	//-1 is a special number meaning no threshold.
-	if sa.Threshold != -1 {
-		if smartMetadata, ok := metadata.NmveMetadata[sa.AttributeId]; ok {
-			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
-			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
-				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = SmartAttributeStatusFailed
-				sa.StatusReason = "Attribute is failing recommended SMART threshold"
-			}
-		}
-	}
-	//TODO: eventually figure out the critical_warning bits and determine correct error messages here.
-
-	//check if status is blank, set to "passed"
-	if len(sa.Status) == 0 {
-		sa.Status = SmartAttributeStatusPassed
-	}
-}

webapp/backend/pkg/models/db/smart_scsci_attribute.go

@@ -1,45 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"gorm.io/gorm"
-)
-
-type SmartScsiAttribute struct {
-	gorm.Model
-
-	SmartId int    `json:"smart_id"`
-	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
-
-	AttributeId string `json:"attribute_id"` //json string from smartctl
-	Name        string `json:"name"`
-	Value       int    `json:"value"`
-	Threshold   int    `json:"thresh"`
-
-	TransformedValue int64                `json:"transformed_value"`
-	Status           string               `gorm:"-" json:"status,omitempty"`
-	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
-	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
-	History          []SmartScsiAttribute `gorm:"-" json:"history,omitempty"`
-}
-
-//populate attribute status, using SMART Thresholds & Observed Metadata
-func (sa *SmartScsiAttribute) PopulateAttributeStatus() {
-
-	//-1 is a special number meaning no threshold.
-	if sa.Threshold != -1 {
-		if smartMetadata, ok := metadata.NmveMetadata[sa.AttributeId]; ok {
-			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
-			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
-				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = SmartAttributeStatusFailed
-				sa.StatusReason = "Attribute is failing recommended SMART threshold"
-			}
-		}
-	}
-
-	//check if status is blank, set to "passed"
-	if len(sa.Status) == 0 {
-		sa.Status = SmartAttributeStatusPassed
-	}
-}

webapp/backend/pkg/models/db/smart_test.go

@@ -1,155 +0,0 @@
-package db_test
-
-import (
-	"encoding/json"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
-	"github.com/stretchr/testify/require"
-	"io/ioutil"
-	"os"
-	"testing"
-)
-
-func TestFromCollectorSmartInfo(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-ata.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "passed", smartMdl.SmartStatus)
-	require.Equal(t, 18, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-
-	//check that temperature was correctly parsed
-	for _, attr := range smartMdl.AtaAttributes {
-		if attr.AttributeId == 194 {
-			require.Equal(t, int64(163210330144), attr.RawValue)
-			require.Equal(t, int64(32), attr.TransformedValue)
-		}
-	}
-}
-
-func TestFromCollectorSmartInfo_Fail(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-fail.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "failed", smartMdl.SmartStatus)
-	require.Equal(t, 0, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-}
-
-func TestFromCollectorSmartInfo_Fail2(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-fail2.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "failed", smartMdl.SmartStatus)
-	require.Equal(t, 17, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-}
-
-func TestFromCollectorSmartInfo_Nvme(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-nvme.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "passed", smartMdl.SmartStatus)
-	require.Equal(t, 0, len(smartMdl.AtaAttributes))
-	require.Equal(t, 16, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-
-	require.Equal(t, 111303174, smartMdl.NvmeAttributes[6].Value)
-	require.Equal(t, 83170961, smartMdl.NvmeAttributes[7].Value)
-}
-
-func TestFromCollectorSmartInfo_Scsi(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-scsi.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "passed", smartMdl.SmartStatus)
-	require.Equal(t, 0, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 13, len(smartMdl.ScsiAttributes))
-
-	require.Equal(t, 56, smartMdl.ScsiAttributes[0].Value)
-	require.Equal(t, 300357663, smartMdl.ScsiAttributes[4].Value) //total_errors_corrected
-}

webapp/backend/pkg/models/device.go

@@ -0,0 +1,169 @@
+package models
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"time"
+)
+
+type DeviceWrapper struct {
+	Success bool     `json:"success"`
+	Errors  []error  `json:"errors"`
+	Data    []Device `json:"data"`
+}
+
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN string `json:"wwn" gorm:"primary_key"`
+
+	DeviceName     string `json:"device_name"`
+	Manufacturer   string `json:"manufacturer"`
+	ModelName      string `json:"model_name"`
+	InterfaceType  string `json:"interface_type"`
+	InterfaceSpeed string `json:"interface_speed"`
+	SerialNumber   string `json:"serial_number"`
+	Firmware       string `json:"firmware"`
+	RotationSpeed  int    `json:"rotational_speed"`
+	Capacity       int64  `json:"capacity"`
+	FormFactor     string `json:"form_factor"`
+	SmartSupport   bool   `json:"smart_support"`
+	DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string `json:"label"`
+	HostId string `json:"host_id"`
+
+	// Data set by Scrutiny
+	DeviceStatus pkg.DeviceStatus `json:"device_status"`
+}
+
+func (dv *Device) IsAta() bool {
+	return dv.DeviceProtocol == pkg.DeviceProtocolAta
+}
+
+func (dv *Device) IsScsi() bool {
+	return dv.DeviceProtocol == pkg.DeviceProtocolScsi
+}
+
+func (dv *Device) IsNvme() bool {
+	return dv.DeviceProtocol == pkg.DeviceProtocolNvme
+}
+
+//
+////This method requires a device with an array of SmartResults.
+////It will remove all SmartResults other than the first (the latest one)
+////All removed SmartResults, will be processed, grouping SmartAtaAttribute by attribute_id
+//// and adding theme to an array called History.
+//func (dv *Device) SquashHistory() error {
+//	if len(dv.SmartResults) <= 1 {
+//		return nil //no ataHistory found. ignore
+//	}
+//
+//	latestSmartResultSlice := dv.SmartResults[0:1]
+//	historicalSmartResultSlice := dv.SmartResults[1:]
+//
+//	//re-assign the latest slice to the SmartResults field
+//	dv.SmartResults = latestSmartResultSlice
+//
+//	//process the historical slice for ATA data
+//	if len(dv.SmartResults[0].AtaAttributes) > 0 {
+//		ataHistory := map[int][]SmartAtaAttribute{}
+//		for _, smartResult := range historicalSmartResultSlice {
+//			for _, smartAttribute := range smartResult.AtaAttributes {
+//				if _, ok := ataHistory[smartAttribute.AttributeId]; !ok {
+//					ataHistory[smartAttribute.AttributeId] = []SmartAtaAttribute{}
+//				}
+//				ataHistory[smartAttribute.AttributeId] = append(ataHistory[smartAttribute.AttributeId], smartAttribute)
+//			}
+//		}
+//
+//		//now assign the historical slices to the AtaAttributes in the latest SmartResults
+//		for sandx, smartAttribute := range dv.SmartResults[0].AtaAttributes {
+//			if attributeHistory, ok := ataHistory[smartAttribute.AttributeId]; ok {
+//				dv.SmartResults[0].AtaAttributes[sandx].History = attributeHistory
+//			}
+//		}
+//	}
+//
+//	//process the historical slice for Nvme data
+//	if len(dv.SmartResults[0].NvmeAttributes) > 0 {
+//		nvmeHistory := map[string][]SmartNvmeAttribute{}
+//		for _, smartResult := range historicalSmartResultSlice {
+//			for _, smartAttribute := range smartResult.NvmeAttributes {
+//				if _, ok := nvmeHistory[smartAttribute.AttributeId]; !ok {
+//					nvmeHistory[smartAttribute.AttributeId] = []SmartNvmeAttribute{}
+//				}
+//				nvmeHistory[smartAttribute.AttributeId] = append(nvmeHistory[smartAttribute.AttributeId], smartAttribute)
+//			}
+//		}
+//
+//		//now assign the historical slices to the AtaAttributes in the latest SmartResults
+//		for sandx, smartAttribute := range dv.SmartResults[0].NvmeAttributes {
+//			if attributeHistory, ok := nvmeHistory[smartAttribute.AttributeId]; ok {
+//				dv.SmartResults[0].NvmeAttributes[sandx].History = attributeHistory
+//			}
+//		}
+//	}
+//	//process the historical slice for Scsi data
+//	if len(dv.SmartResults[0].ScsiAttributes) > 0 {
+//		scsiHistory := map[string][]SmartScsiAttribute{}
+//		for _, smartResult := range historicalSmartResultSlice {
+//			for _, smartAttribute := range smartResult.ScsiAttributes {
+//				if _, ok := scsiHistory[smartAttribute.AttributeId]; !ok {
+//					scsiHistory[smartAttribute.AttributeId] = []SmartScsiAttribute{}
+//				}
+//				scsiHistory[smartAttribute.AttributeId] = append(scsiHistory[smartAttribute.AttributeId], smartAttribute)
+//			}
+//		}
+//
+//		//now assign the historical slices to the AtaAttributes in the latest SmartResults
+//		for sandx, smartAttribute := range dv.SmartResults[0].ScsiAttributes {
+//			if attributeHistory, ok := scsiHistory[smartAttribute.AttributeId]; ok {
+//				dv.SmartResults[0].ScsiAttributes[sandx].History = attributeHistory
+//			}
+//		}
+//	}
+//	return nil
+//}
+//
+//func (dv *Device) ApplyMetadataRules() error {
+//
+//	//embed metadata in the latest smart attributes object
+//	if len(dv.SmartResults) > 0 {
+//		for ndx, attr := range dv.SmartResults[0].AtaAttributes {
+//			attr.PopulateAttributeStatus()
+//			dv.SmartResults[0].AtaAttributes[ndx] = attr
+//		}
+//
+//		for ndx, attr := range dv.SmartResults[0].NvmeAttributes {
+//			attr.PopulateAttributeStatus()
+//			dv.SmartResults[0].NvmeAttributes[ndx] = attr
+//
+//		}
+//
+//		for ndx, attr := range dv.SmartResults[0].ScsiAttributes {
+//			attr.PopulateAttributeStatus()
+//			dv.SmartResults[0].ScsiAttributes[ndx] = attr
+//
+//		}
+//	}
+//	return nil
+//}
+
+// This function is called every time the collector sends SMART data to the API.
+// It can be used to update device data that can change over time.
+func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
+	dv.Firmware = info.FirmwareVersion
+	dv.DeviceProtocol = info.Device.Protocol
+
+	if !info.SmartStatus.Passed {
+		dv.DeviceStatus = pkg.Set(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
+	}
+
+	return nil
+}

webapp/backend/pkg/models/device_summary.go

@@ -0,0 +1,27 @@
+package models
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"time"
+)
+
+type DeviceSummaryWrapper struct {
+	Success bool    `json:"success"`
+	Errors  []error `json:"errors"`
+	Data    struct {
+		Summary map[string]*DeviceSummary `json:"summary"`
+	} `json:"data"`
+}
+
+type DeviceSummary struct {
+	Device Device `json:"device"`
+
+	SmartResults *SmartSummary                   `json:"smart,omitempty"`
+	TempHistory  []measurements.SmartTemperature `json:"temp_history,omitempty"`
+}
+type SmartSummary struct {
+	// Collector Summary Data
+	CollectorDate time.Time `json:"collector_date,omitempty"`
+	Temp          int64     `json:"temp,omitempty"`
+	PowerOnHours  int64     `json:"power_on_hours,omitempty"`
+}

webapp/backend/pkg/models/measurements/smart.go

@@ -0,0 +1,211 @@
+package measurements
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
+	"log"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Smart struct {
+	Date           time.Time `json:"date"`
+	DeviceWWN      string    `json:"device_wwn"` //(tag)
+	DeviceProtocol string    `json:"device_protocol"`
+
+	//Metrics (fields)
+	Temp            int64 `json:"temp"`
+	PowerOnHours    int64 `json:"power_on_hours"`
+	PowerCycleCount int64 `json:"power_cycle_count"`
+
+	//Attributes (fields)
+	Attributes map[string]SmartAttribute `json:"attrs"`
+
+	//status
+	Status pkg.DeviceStatus
+}
+
+func (sm *Smart) Flatten() (tags map[string]string, fields map[string]interface{}) {
+	tags = map[string]string{
+		"device_wwn":      sm.DeviceWWN,
+		"device_protocol": sm.DeviceProtocol,
+	}
+
+	fields = map[string]interface{}{
+		"temp":              sm.Temp,
+		"power_on_hours":    sm.PowerOnHours,
+		"power_cycle_count": sm.PowerCycleCount,
+	}
+
+	for _, attr := range sm.Attributes {
+		for attrKey, attrVal := range attr.Flatten() {
+			fields[attrKey] = attrVal
+		}
+	}
+
+	return tags, fields
+}
+
+func NewSmartFromInfluxDB(attrs map[string]interface{}) (*Smart, error) {
+	//go though the massive map returned from influxdb. If a key is associated with the Smart struct, assign it. If it starts with "attr.*" group it by attributeId, and pass to attribute inflate.
+
+	sm := Smart{
+		//required fields
+		Date:           attrs["_time"].(time.Time),
+		DeviceWWN:      attrs["device_wwn"].(string),
+		DeviceProtocol: attrs["device_protocol"].(string),
+
+		Attributes: map[string]SmartAttribute{},
+	}
+
+	for key, val := range attrs {
+		switch key {
+		case "temp":
+			sm.Temp = val.(int64)
+		case "power_on_hours":
+			sm.PowerOnHours = val.(int64)
+		case "power_cycle_count":
+			sm.PowerCycleCount = val.(int64)
+		default:
+			// this key is unknown.
+			if !strings.HasPrefix(key, "attr.") {
+				continue
+			}
+			//this is a attribute, lets group it with its related "siblings", populating a SmartAttribute object
+			keyParts := strings.Split(key, ".")
+			attributeId := keyParts[1]
+			if _, ok := sm.Attributes[attributeId]; !ok {
+				// init the attribute group
+				if sm.DeviceProtocol == pkg.DeviceProtocolAta {
+					sm.Attributes[attributeId] = &SmartAtaAttribute{}
+				} else if sm.DeviceProtocol == pkg.DeviceProtocolNvme {
+					sm.Attributes[attributeId] = &SmartNvmeAttribute{}
+				} else if sm.DeviceProtocol == pkg.DeviceProtocolScsi {
+					sm.Attributes[attributeId] = &SmartScsiAttribute{}
+				} else {
+					return nil, fmt.Errorf("Unknown Device Protocol: %s", sm.DeviceProtocol)
+				}
+			}
+
+			sm.Attributes[attributeId].Inflate(key, val)
+		}
+
+	}
+
+	log.Printf("Found Smart Device (%s) Attributes (%v)", sm.DeviceWWN, len(sm.Attributes))
+
+	return &sm, nil
+}
+
+//Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries)
+func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error {
+	sm.DeviceWWN = wwn
+	sm.Date = time.Unix(info.LocalTime.TimeT, 0)
+
+	//smart metrics
+	sm.Temp = info.Temperature.Current
+	sm.PowerCycleCount = info.PowerCycleCount
+	sm.PowerOnHours = info.PowerOnTime.Hours
+	if !info.SmartStatus.Passed {
+		sm.Status = pkg.DeviceStatusFailedSmart
+	}
+
+	sm.DeviceProtocol = info.Device.Protocol
+	// process ATA/NVME/SCSI protocol data
+	sm.Attributes = map[string]SmartAttribute{}
+	if sm.DeviceProtocol == pkg.DeviceProtocolAta {
+		sm.ProcessAtaSmartInfo(info.AtaSmartAttributes.Table)
+	} else if sm.DeviceProtocol == pkg.DeviceProtocolNvme {
+		sm.ProcessNvmeSmartInfo(info.NvmeSmartHealthInformationLog)
+	} else if sm.DeviceProtocol == pkg.DeviceProtocolScsi {
+		sm.ProcessScsiSmartInfo(info.ScsiGrownDefectList, info.ScsiErrorCounterLog)
+	}
+
+	return nil
+}
+
+//generate SmartAtaAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTableItem) {
+	for _, collectorAttr := range tableItems {
+		attrModel := SmartAtaAttribute{
+			AttributeId: collectorAttr.ID,
+			Value:       collectorAttr.Value,
+			Worst:       collectorAttr.Worst,
+			Threshold:   collectorAttr.Thresh,
+			RawValue:    collectorAttr.Raw.Value,
+			RawString:   collectorAttr.Raw.String,
+			WhenFailed:  collectorAttr.WhenFailed,
+		}
+
+		//now that we've parsed the data from the smartctl response, lets match it against our metadata rules and add additional Scrutiny specific data.
+		if smartMetadata, ok := thresholds.AtaMetadata[collectorAttr.ID]; ok {
+			if smartMetadata.Transform != nil {
+				attrModel.TransformedValue = smartMetadata.Transform(attrModel.Value, attrModel.RawValue, attrModel.RawString)
+			}
+		}
+		attrModel.PopulateAttributeStatus()
+		sm.Attributes[strconv.Itoa(collectorAttr.ID)] = &attrModel
+		if attrModel.Status == pkg.SmartAttributeStatusFailed {
+			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		}
+	}
+}
+
+//generate SmartNvmeAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.NvmeSmartHealthInformationLog) {
+
+	sm.Attributes = map[string]SmartAttribute{
+		"critical_warning":     (&SmartNvmeAttribute{AttributeId: "critical_warning", Value: nvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0}).PopulateAttributeStatus(),
+		"temperature":          (&SmartNvmeAttribute{AttributeId: "temperature", Value: nvmeSmartHealthInformationLog.Temperature, Threshold: -1}).PopulateAttributeStatus(),
+		"available_spare":      (&SmartNvmeAttribute{AttributeId: "available_spare", Value: nvmeSmartHealthInformationLog.AvailableSpare, Threshold: nvmeSmartHealthInformationLog.AvailableSpareThreshold}).PopulateAttributeStatus(),
+		"percentage_used":      (&SmartNvmeAttribute{AttributeId: "percentage_used", Value: nvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100}).PopulateAttributeStatus(),
+		"data_units_read":      (&SmartNvmeAttribute{AttributeId: "data_units_read", Value: nvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1}).PopulateAttributeStatus(),
+		"data_units_written":   (&SmartNvmeAttribute{AttributeId: "data_units_written", Value: nvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1}).PopulateAttributeStatus(),
+		"host_reads":           (&SmartNvmeAttribute{AttributeId: "host_reads", Value: nvmeSmartHealthInformationLog.HostReads, Threshold: -1}).PopulateAttributeStatus(),
+		"host_writes":          (&SmartNvmeAttribute{AttributeId: "host_writes", Value: nvmeSmartHealthInformationLog.HostWrites, Threshold: -1}).PopulateAttributeStatus(),
+		"controller_busy_time": (&SmartNvmeAttribute{AttributeId: "controller_busy_time", Value: nvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1}).PopulateAttributeStatus(),
+		"power_cycles":         (&SmartNvmeAttribute{AttributeId: "power_cycles", Value: nvmeSmartHealthInformationLog.PowerCycles, Threshold: -1}).PopulateAttributeStatus(),
+		"power_on_hours":       (&SmartNvmeAttribute{AttributeId: "power_on_hours", Value: nvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1}).PopulateAttributeStatus(),
+		"unsafe_shutdowns":     (&SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Value: nvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1}).PopulateAttributeStatus(),
+		"media_errors":         (&SmartNvmeAttribute{AttributeId: "media_errors", Value: nvmeSmartHealthInformationLog.MediaErrors, Threshold: 0}).PopulateAttributeStatus(),
+		"num_err_log_entries":  (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: nvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0}).PopulateAttributeStatus(),
+		"warning_temp_time":    (&SmartNvmeAttribute{AttributeId: "warning_temp_time", Value: nvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1}).PopulateAttributeStatus(),
+		"critical_comp_time":   (&SmartNvmeAttribute{AttributeId: "critical_comp_time", Value: nvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1}).PopulateAttributeStatus(),
+	}
+
+	//find analyzed attribute status
+	for _, val := range sm.Attributes {
+		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
+			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		}
+	}
+}
+
+//generate SmartScsiAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog collector.ScsiErrorCounterLog) {
+	sm.Attributes = map[string]SmartAttribute{
+		"scsi_grown_defect_list":                     (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: defectGrownList, Threshold: 0}).PopulateAttributeStatus(),
+		"read_errors_corrected_by_eccfast":           (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(),
+		"read_errors_corrected_by_eccdelayed":        (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(),
+		"read_errors_corrected_by_rereads_rewrites":  (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(),
+		"read_total_errors_corrected":                (&SmartScsiAttribute{AttributeId: "read_total_errors_corrected", Value: scsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(),
+		"read_correction_algorithm_invocations":      (&SmartScsiAttribute{AttributeId: "read_correction_algorithm_invocations", Value: scsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(),
+		"read_total_uncorrected_errors":              (&SmartScsiAttribute{AttributeId: "read_total_uncorrected_errors", Value: scsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(),
+		"write_errors_corrected_by_eccfast":          (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(),
+		"write_errors_corrected_by_eccdelayed":       (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(),
+		"write_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(),
+		"write_total_errors_corrected":               (&SmartScsiAttribute{AttributeId: "write_total_errors_corrected", Value: scsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(),
+		"write_correction_algorithm_invocations":     (&SmartScsiAttribute{AttributeId: "write_correction_algorithm_invocations", Value: scsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(),
+		"write_total_uncorrected_errors":             (&SmartScsiAttribute{AttributeId: "write_total_uncorrected_errors", Value: scsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(),
+	}
+
+	//find analyzed attribute status
+	for _, val := range sm.Attributes {
+		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
+			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		}
+	}
+}

webapp/backend/pkg/models/measurements/smart_ata_attribute.go

@@ -0,0 +1,163 @@
+package measurements
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
+	"strconv"
+	"strings"
+)
+
+type SmartAtaAttribute struct {
+	AttributeId int    `json:"attribute_id"`
+	Value       int64  `json:"value"`
+	Threshold   int64  `json:"thresh"`
+	Worst       int64  `json:"worst"`
+	RawValue    int64  `json:"raw_value"`
+	RawString   string `json:"raw_string"`
+	WhenFailed  string `json:"when_failed"`
+
+	//Generated data
+	TransformedValue int64   `json:"transformed_value"`
+	Status           int64   `json:"status"`
+	StatusReason     string  `json:"status_reason,omitempty"`
+	FailureRate      float64 `json:"failure_rate,omitempty"`
+}
+
+func (sa *SmartAtaAttribute) GetStatus() int64 {
+	return sa.Status
+}
+
+func (sa *SmartAtaAttribute) Flatten() map[string]interface{} {
+
+	idString := strconv.Itoa(sa.AttributeId)
+
+	return map[string]interface{}{
+		fmt.Sprintf("attr.%s.attribute_id", idString): idString,
+		fmt.Sprintf("attr.%s.value", idString):        sa.Value,
+		fmt.Sprintf("attr.%s.worst", idString):        sa.Worst,
+		fmt.Sprintf("attr.%s.thresh", idString):       sa.Threshold,
+		fmt.Sprintf("attr.%s.raw_value", idString):    sa.RawValue,
+		fmt.Sprintf("attr.%s.raw_string", idString):   sa.RawString,
+		fmt.Sprintf("attr.%s.when_failed", idString):  sa.WhenFailed,
+
+		//Generated Data
+		fmt.Sprintf("attr.%s.transformed_value", idString): sa.TransformedValue,
+		fmt.Sprintf("attr.%s.status", idString):            sa.Status,
+		fmt.Sprintf("attr.%s.status_reason", idString):     sa.StatusReason,
+		fmt.Sprintf("attr.%s.failure_rate", idString):      sa.FailureRate,
+	}
+}
+func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+	keyParts := strings.Split(key, ".")
+
+	switch keyParts[2] {
+	case "attribute_id":
+		attrId, err := strconv.Atoi(val.(string))
+		if err == nil {
+			sa.AttributeId = attrId
+		}
+	case "value":
+		sa.Value = val.(int64)
+	case "worst":
+		sa.Worst = val.(int64)
+	case "thresh":
+		sa.Threshold = val.(int64)
+	case "raw_value":
+		sa.RawValue = val.(int64)
+	case "raw_string":
+		sa.RawString = val.(string)
+	case "when_failed":
+		sa.WhenFailed = val.(string)
+
+	//generated
+	case "transformed_value":
+		sa.TransformedValue = val.(int64)
+	case "status":
+		sa.Status = val.(int64)
+	case "status_reason":
+		sa.StatusReason = val.(string)
+	case "failure_rate":
+		sa.FailureRate = val.(float64)
+
+	}
+}
+
+//populate attribute status, using SMART Thresholds & Observed Metadata
+// Chainable
+func (sa *SmartAtaAttribute) PopulateAttributeStatus() *SmartAtaAttribute {
+	if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedFailingNow {
+		//this attribute has previously failed
+		sa.Status = pkg.SmartAttributeStatusFailed
+		sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
+
+	} else if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedInThePast {
+		sa.Status = pkg.SmartAttributeStatusWarning
+		sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
+	}
+
+	if smartMetadata, ok := thresholds.AtaMetadata[sa.AttributeId]; ok {
+		sa.ValidateThreshold(smartMetadata)
+	}
+
+	return sa
+}
+
+// compare the attribute (raw, normalized, transformed) value to observed thresholds, and update status if necessary
+func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttributeMetadata) {
+	//TODO: multiple rules
+	// try to predict the failure rates for observed thresholds that have 0 failure rate and error bars.
+	// - if the attribute is critical
+	//		- the failure rate is over 10 - set to failed
+	//		- the attribute does not match any threshold, set to warn
+	// - if the attribute is not critical
+	//		- if failure rate is above 20 - set to failed
+	// 		- if failure rate is above 10 but below 20 - set to warn
+
+	//update the smart attribute status based on Observed thresholds.
+	var value int64
+	if smartMetadata.DisplayType == thresholds.AtaSmartAttributeDisplayTypeNormalized {
+		value = int64(sa.Value)
+	} else if smartMetadata.DisplayType == thresholds.AtaSmartAttributeDisplayTypeTransformed {
+		value = sa.TransformedValue
+	} else {
+		value = sa.RawValue
+	}
+
+	for _, obsThresh := range smartMetadata.ObservedThresholds {
+
+		//check if "value" is in this bucket
+		if ((obsThresh.Low == obsThresh.High) && value == obsThresh.Low) ||
+			(obsThresh.Low < value && value <= obsThresh.High) {
+			sa.FailureRate = obsThresh.AnnualFailureRate
+
+			if smartMetadata.Critical {
+				if obsThresh.AnnualFailureRate >= 0.10 {
+					sa.Status = pkg.SmartAttributeStatusFailed
+					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
+				}
+			} else {
+				if obsThresh.AnnualFailureRate >= 0.20 {
+					sa.Status = pkg.SmartAttributeStatusFailed
+					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
+				} else if obsThresh.AnnualFailureRate >= 0.10 {
+					sa.Status = pkg.SmartAttributeStatusWarning
+					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
+				}
+			}
+
+			//we've found the correct bucket, we can drop out of this loop
+			return
+		}
+	}
+	// no bucket found
+	if smartMetadata.Critical {
+		sa.Status = pkg.SmartAttributeStatusWarning
+		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
+	}
+
+	return
+}

webapp/backend/pkg/models/measurements/smart_attribute.go

@@ -0,0 +1,7 @@
+package measurements
+
+type SmartAttribute interface {
+	Flatten() (fields map[string]interface{})
+	Inflate(key string, val interface{})
+	GetStatus() int64
+}

webapp/backend/pkg/models/measurements/smart_nvme_attribute.go

@@ -0,0 +1,83 @@
+package measurements
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
+	"strings"
+)
+
+type SmartNvmeAttribute struct {
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Value       int64  `json:"value"`
+	Threshold   int64  `json:"thresh"`
+
+	TransformedValue int64   `json:"transformed_value"`
+	Status           int64   `json:"status"`
+	StatusReason     string  `json:"status_reason,omitempty"`
+	FailureRate      float64 `json:"failure_rate,omitempty"`
+}
+
+func (sa *SmartNvmeAttribute) GetStatus() int64 {
+	return sa.Status
+}
+
+func (sa *SmartNvmeAttribute) Flatten() map[string]interface{} {
+	return map[string]interface{}{
+		fmt.Sprintf("attr.%s.attribute_id", sa.AttributeId): sa.AttributeId,
+		fmt.Sprintf("attr.%s.value", sa.AttributeId):        sa.Value,
+		fmt.Sprintf("attr.%s.thresh", sa.AttributeId):       sa.Threshold,
+
+		//Generated Data
+		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
+		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
+	}
+}
+func (sa *SmartNvmeAttribute) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+
+	keyParts := strings.Split(key, ".")
+
+	switch keyParts[2] {
+	case "attribute_id":
+		sa.AttributeId = val.(string)
+	case "value":
+		sa.Value = val.(int64)
+	case "thresh":
+		sa.Threshold = val.(int64)
+
+	//generated
+	case "transformed_value":
+		sa.TransformedValue = val.(int64)
+	case "status":
+		sa.Status = val.(int64)
+	case "status_reason":
+		sa.StatusReason = val.(string)
+	case "failure_rate":
+		sa.FailureRate = val.(float64)
+	}
+}
+
+//populate attribute status, using SMART Thresholds & Observed Metadata
+// Chainable
+func (sa *SmartNvmeAttribute) PopulateAttributeStatus() *SmartNvmeAttribute {
+
+	//-1 is a special number meaning no threshold.
+	if sa.Threshold != -1 {
+		if smartMetadata, ok := thresholds.NmveMetadata[sa.AttributeId]; ok {
+			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
+			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
+				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
+				sa.Status = pkg.SmartAttributeStatusFailed
+				sa.StatusReason = "Attribute is failing recommended SMART threshold"
+			}
+		}
+	}
+	//TODO: eventually figure out the critical_warning bits and determine correct error messages here.
+
+	return sa
+}

webapp/backend/pkg/models/measurements/smart_scsci_attribute.go

@@ -0,0 +1,83 @@
+package measurements
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
+	"strings"
+)
+
+type SmartScsiAttribute struct {
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Value       int64  `json:"value"`
+	Threshold   int64  `json:"thresh"`
+
+	TransformedValue int64   `json:"transformed_value"`
+	Status           int64   `json:"status"`
+	StatusReason     string  `json:"status_reason,omitempty"`
+	FailureRate      float64 `json:"failure_rate,omitempty"`
+}
+
+func (sa *SmartScsiAttribute) GetStatus() int64 {
+	return sa.Status
+}
+
+func (sa *SmartScsiAttribute) Flatten() map[string]interface{} {
+	return map[string]interface{}{
+		fmt.Sprintf("attr.%s.attribute_id", sa.AttributeId): sa.AttributeId,
+		fmt.Sprintf("attr.%s.value", sa.AttributeId):        sa.Value,
+		fmt.Sprintf("attr.%s.thresh", sa.AttributeId):       sa.Threshold,
+
+		//Generated Data
+		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
+		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
+	}
+}
+func (sa *SmartScsiAttribute) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+
+	keyParts := strings.Split(key, ".")
+
+	switch keyParts[2] {
+	case "attribute_id":
+		sa.AttributeId = val.(string)
+	case "value":
+		sa.Value = val.(int64)
+	case "thresh":
+		sa.Threshold = val.(int64)
+
+	//generated
+	case "transformed_value":
+		sa.TransformedValue = val.(int64)
+	case "status":
+		sa.Status = val.(int64)
+	case "status_reason":
+		sa.StatusReason = val.(string)
+	case "failure_rate":
+		sa.FailureRate = val.(float64)
+	}
+}
+
+//
+//populate attribute status, using SMART Thresholds & Observed Metadata
+//Chainable
+func (sa *SmartScsiAttribute) PopulateAttributeStatus() *SmartScsiAttribute {
+
+	//-1 is a special number meaning no threshold.
+	if sa.Threshold != -1 {
+		if smartMetadata, ok := thresholds.NmveMetadata[sa.AttributeId]; ok {
+			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
+			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
+				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
+				sa.Status = pkg.SmartAttributeStatusFailed
+				sa.StatusReason = "Attribute is failing recommended SMART threshold"
+			}
+		}
+	}
+
+	return sa
+}

webapp/backend/pkg/models/measurements/smart_temperature.go

@@ -0,0 +1,34 @@
+package measurements
+
+import (
+	"time"
+)
+
+type SmartTemperature struct {
+	Date time.Time `json:"date"`
+	Temp int64     `json:"temp"`
+}
+
+func (st *SmartTemperature) Flatten() (tags map[string]string, fields map[string]interface{}) {
+	fields = map[string]interface{}{
+		"temp": st.Temp,
+	}
+	tags = map[string]string{}
+
+	return tags, fields
+}
+
+func (st *SmartTemperature) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+
+	if key == "temp" {
+		switch t := val.(type) {
+		case int64:
+			st.Temp = t
+		case float64:
+			st.Temp = int64(t)
+		}
+	}
+}

webapp/backend/pkg/models/measurements/smart_test.go

@@ -0,0 +1,436 @@
+package measurements_test
+
+import (
+	"encoding/json"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"github.com/stretchr/testify/require"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestSmart_Flatten(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	smart := measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  pkg.DeviceProtocolAta,
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Attributes:      nil,
+		Status:          0,
+	}
+
+	//test
+	tags, fields := smart.Flatten()
+
+	//assert
+	require.Equal(t, map[string]string{"device_protocol": "ATA", "device_wwn": "test-wwn"}, tags)
+	require.Equal(t, map[string]interface{}{"power_cycle_count": int64(10), "power_on_hours": int64(10), "temp": int64(50)}, fields)
+}
+
+func TestSmart_Flatten_ATA(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	smart := measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  pkg.DeviceProtocolAta,
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Status:          0,
+		Attributes: map[string]measurements.SmartAttribute{
+			"1": &measurements.SmartAtaAttribute{
+				AttributeId: 1,
+				Value:       100,
+				Threshold:   1,
+				Worst:       100,
+				RawValue:    0,
+				RawString:   "0",
+				WhenFailed:  "",
+			},
+			"2": &measurements.SmartAtaAttribute{
+				AttributeId: 2,
+				Value:       135,
+				Threshold:   54,
+				Worst:       135,
+				RawValue:    108,
+				RawString:   "108",
+				WhenFailed:  "",
+			},
+		},
+	}
+
+	//test
+	tags, fields := smart.Flatten()
+
+	//assert
+	require.Equal(t, map[string]string{"device_protocol": "ATA", "device_wwn": "test-wwn"}, tags)
+	require.Equal(t, map[string]interface{}{
+		"attr.1.attribute_id":      "1",
+		"attr.1.failure_rate":      float64(0),
+		"attr.1.raw_string":        "0",
+		"attr.1.raw_value":         int64(0),
+		"attr.1.status":            int64(0),
+		"attr.1.status_reason":     "",
+		"attr.1.thresh":            int64(1),
+		"attr.1.transformed_value": int64(0),
+		"attr.1.value":             int64(100),
+		"attr.1.when_failed":       "",
+		"attr.1.worst":             int64(100),
+
+		"attr.2.attribute_id":      "2",
+		"attr.2.failure_rate":      float64(0),
+		"attr.2.raw_string":        "108",
+		"attr.2.raw_value":         int64(108),
+		"attr.2.status":            int64(0),
+		"attr.2.status_reason":     "",
+		"attr.2.thresh":            int64(54),
+		"attr.2.transformed_value": int64(0),
+		"attr.2.value":             int64(135),
+		"attr.2.when_failed":       "",
+		"attr.2.worst":             int64(135),
+
+		"power_cycle_count": int64(10),
+		"power_on_hours":    int64(10),
+		"temp":              int64(50),
+	}, fields)
+}
+
+func TestSmart_Flatten_SCSI(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	smart := measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  pkg.DeviceProtocolScsi,
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Status:          0,
+		Attributes: map[string]measurements.SmartAttribute{
+			"read_errors_corrected_by_eccfast": &measurements.SmartScsiAttribute{
+				AttributeId: "read_errors_corrected_by_eccfast",
+				Value:       int64(300357663),
+			},
+		},
+	}
+
+	//test
+	tags, fields := smart.Flatten()
+
+	//assert
+	require.Equal(t, map[string]string{"device_protocol": "SCSI", "device_wwn": "test-wwn"}, tags)
+	require.Equal(t, map[string]interface{}{
+		"attr.read_errors_corrected_by_eccfast.attribute_id":      "read_errors_corrected_by_eccfast",
+		"attr.read_errors_corrected_by_eccfast.failure_rate":      float64(0),
+		"attr.read_errors_corrected_by_eccfast.status":            int64(0),
+		"attr.read_errors_corrected_by_eccfast.status_reason":     "",
+		"attr.read_errors_corrected_by_eccfast.thresh":            int64(0),
+		"attr.read_errors_corrected_by_eccfast.transformed_value": int64(0),
+		"attr.read_errors_corrected_by_eccfast.value":             int64(300357663),
+		"power_cycle_count": int64(10),
+		"power_on_hours":    int64(10),
+		"temp":              int64(50)},
+		fields)
+}
+
+func TestSmart_Flatten_NVMe(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	smart := measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  pkg.DeviceProtocolNvme,
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Status:          0,
+		Attributes: map[string]measurements.SmartAttribute{
+			"available_spare": &measurements.SmartNvmeAttribute{
+				AttributeId: "available_spare",
+				Value:       int64(100),
+			},
+		},
+	}
+
+	//test
+	tags, fields := smart.Flatten()
+
+	//assert
+	require.Equal(t, map[string]string{"device_protocol": "NVMe", "device_wwn": "test-wwn"}, tags)
+	require.Equal(t, map[string]interface{}{
+		"attr.available_spare.attribute_id":      "available_spare",
+		"attr.available_spare.failure_rate":      float64(0),
+		"attr.available_spare.status":            int64(0),
+		"attr.available_spare.status_reason":     "",
+		"attr.available_spare.thresh":            int64(0),
+		"attr.available_spare.transformed_value": int64(0),
+		"attr.available_spare.value":             int64(100),
+		"power_cycle_count":                      int64(10),
+		"power_on_hours":                         int64(10),
+		"temp":                                   int64(50)}, fields)
+}
+
+func TestNewSmartFromInfluxDB_ATA(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	attrs := map[string]interface{}{
+		"_time":                    timeNow,
+		"device_wwn":               "test-wwn",
+		"device_protocol":          pkg.DeviceProtocolAta,
+		"attr.1.attribute_id":      "1",
+		"attr.1.failure_rate":      float64(0),
+		"attr.1.raw_string":        "108",
+		"attr.1.raw_value":         int64(108),
+		"attr.1.status":            int64(0),
+		"attr.1.status_reason":     "",
+		"attr.1.thresh":            int64(54),
+		"attr.1.transformed_value": int64(0),
+		"attr.1.value":             int64(135),
+		"attr.1.when_failed":       "",
+		"attr.1.worst":             int64(135),
+		"power_cycle_count":        int64(10),
+		"power_on_hours":           int64(10),
+		"temp":                     int64(50),
+	}
+
+	//test
+	smart, err := measurements.NewSmartFromInfluxDB(attrs)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, &measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  "ATA",
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Attributes: map[string]measurements.SmartAttribute{
+			"1": &measurements.SmartAtaAttribute{
+				AttributeId: 1,
+				Value:       135,
+				Threshold:   54,
+				Worst:       135,
+				RawValue:    108,
+				RawString:   "108",
+				WhenFailed:  "",
+			},
+		}, Status: 0}, smart)
+}
+
+func TestNewSmartFromInfluxDB_NVMe(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	attrs := map[string]interface{}{
+		"_time":                                  timeNow,
+		"device_wwn":                             "test-wwn",
+		"device_protocol":                        pkg.DeviceProtocolNvme,
+		"attr.available_spare.attribute_id":      "available_spare",
+		"attr.available_spare.failure_rate":      float64(0),
+		"attr.available_spare.status":            int64(0),
+		"attr.available_spare.status_reason":     "",
+		"attr.available_spare.thresh":            int64(0),
+		"attr.available_spare.transformed_value": int64(0),
+		"attr.available_spare.value":             int64(100),
+		"power_cycle_count":                      int64(10),
+		"power_on_hours":                         int64(10),
+		"temp":                                   int64(50),
+	}
+
+	//test
+	smart, err := measurements.NewSmartFromInfluxDB(attrs)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, &measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  "NVMe",
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Attributes: map[string]measurements.SmartAttribute{
+			"available_spare": &measurements.SmartNvmeAttribute{
+				AttributeId: "available_spare",
+				Value:       int64(100),
+			},
+		}, Status: 0}, smart)
+}
+
+func TestNewSmartFromInfluxDB_SCSI(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	attrs := map[string]interface{}{
+		"_time":           timeNow,
+		"device_wwn":      "test-wwn",
+		"device_protocol": pkg.DeviceProtocolScsi,
+		"attr.read_errors_corrected_by_eccfast.attribute_id":      "read_errors_corrected_by_eccfast",
+		"attr.read_errors_corrected_by_eccfast.failure_rate":      float64(0),
+		"attr.read_errors_corrected_by_eccfast.status":            int64(0),
+		"attr.read_errors_corrected_by_eccfast.status_reason":     "",
+		"attr.read_errors_corrected_by_eccfast.thresh":            int64(0),
+		"attr.read_errors_corrected_by_eccfast.transformed_value": int64(0),
+		"attr.read_errors_corrected_by_eccfast.value":             int64(300357663),
+		"power_cycle_count": int64(10),
+		"power_on_hours":    int64(10),
+		"temp":              int64(50),
+	}
+
+	//test
+	smart, err := measurements.NewSmartFromInfluxDB(attrs)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, &measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  "SCSI",
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Attributes: map[string]measurements.SmartAttribute{
+			"read_errors_corrected_by_eccfast": &measurements.SmartScsiAttribute{
+				AttributeId: "read_errors_corrected_by_eccfast",
+				Value:       int64(300357663),
+			},
+		}, Status: 0}, smart)
+}
+
+func TestFromCollectorSmartInfo(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-ata.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusPassed, smartMdl.Status)
+	require.Equal(t, 18, len(smartMdl.Attributes))
+
+	//check that temperature was correctly parsed
+
+	require.Equal(t, int64(163210330144), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).RawValue)
+	require.Equal(t, int64(32), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).TransformedValue)
+}
+
+func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-fail.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusFailedSmart, smartMdl.Status)
+	require.Equal(t, 0, len(smartMdl.Attributes))
+}
+
+func TestFromCollectorSmartInfo_Fail_ScrutinySmart(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-fail2.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusFailedScrutiny|pkg.DeviceStatusFailedSmart, smartMdl.Status)
+	require.Equal(t, 17, len(smartMdl.Attributes))
+}
+
+func TestFromCollectorSmartInfo_Nvme(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-nvme.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusPassed, smartMdl.Status)
+	require.Equal(t, 16, len(smartMdl.Attributes))
+
+	require.Equal(t, int64(111303174), smartMdl.Attributes["host_reads"].(*measurements.SmartNvmeAttribute).Value)
+	require.Equal(t, int64(83170961), smartMdl.Attributes["host_writes"].(*measurements.SmartNvmeAttribute).Value)
+}
+
+func TestFromCollectorSmartInfo_Scsi(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-scsi.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusPassed, smartMdl.Status)
+	require.Equal(t, 13, len(smartMdl.Attributes))
+
+	require.Equal(t, int64(56), smartMdl.Attributes["scsi_grown_defect_list"].(*measurements.SmartScsiAttribute).Value)
+	require.Equal(t, int64(300357663), smartMdl.Attributes["read_errors_corrected_by_eccfast"].(*measurements.SmartScsiAttribute).Value) //total_errors_corrected
+}

webapp/backend/pkg/models/setting.go

@@ -0,0 +1,5 @@
+package models
+
+// Temperature Format
+// Date Format
+// Device History window

webapp/backend/pkg/models/testdata/helper.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+func main() {
+
+	//webapp/backend/pkg/web/testdata/register-devices-req.json
+	devices := "webapp/backend/pkg/web/testdata/register-devices-req.json"
+
+	smartData := map[string][]string{
+		"0x5000cca264eb01d7": {"webapp/backend/pkg/models/testdata/smart-ata.json", "webapp/backend/pkg/models/testdata/smart-ata-date.json", "webapp/backend/pkg/models/testdata/smart-ata-date2.json"},
+		"0x5000cca264ec3183": {"webapp/backend/pkg/models/testdata/smart-fail2.json"},
+		"0x5002538e40a22954": {"webapp/backend/pkg/models/testdata/smart-nvme.json"},
+		"0x5000cca252c859cc": {"webapp/backend/pkg/models/testdata/smart-scsi.json"},
+		"0x5000cca264ebc248": {"webapp/backend/pkg/models/testdata/smart-scsi2.json"},
+	}
+
+	// send a post request to register devices
+	file, err := os.Open(devices)
+	if err != nil {
+		log.Fatalf("ERROR %v", err)
+	}
+	defer file.Close()
+	_, err = SendPostRequest("http://localhost:9090/api/devices/register", file)
+	if err != nil {
+		log.Fatalf("ERROR %v", err)
+	}
+	//
+
+	for diskId, smartDataFileNames := range smartData {
+		for _, smartDataFileName := range smartDataFileNames {
+			for daysToSubtract := 0; daysToSubtract <= 30; daysToSubtract++ { //add 4 weeks worth of data
+				smartDataReader, err := readSmartDataFileFixTimestamp(daysToSubtract, smartDataFileName)
+				if err != nil {
+					log.Fatalf("ERROR %v", err)
+				}
+
+				_, err = SendPostRequest(fmt.Sprintf("http://localhost:9090/api/device/%s/smart", diskId), smartDataReader)
+				if err != nil {
+					log.Fatalf("ERROR %v", err)
+				}
+			}
+
+		}
+
+	}
+
+}
+
+func SendPostRequest(url string, file io.Reader) ([]byte, error) {
+	response, err := http.Post(url, "application/json", file)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	log.Printf("%v\n", response.Status)
+
+	return ioutil.ReadAll(response.Body)
+}
+
+// InfluxDB will throw an error/ignore any submitted data with a timestamp older than the
+// retention period. Lets fix this by opening test files, modifying the timestamp and returning an io.Reader
+func readSmartDataFileFixTimestamp(daysToSubtract int, smartDataFilepath string) (io.Reader, error) {
+	metricsfile, err := os.Open(smartDataFilepath)
+	if err != nil {
+		return nil, err
+	}
+
+	metricsFileData, err := ioutil.ReadAll(metricsfile)
+	if err != nil {
+		return nil, err
+	}
+	//unmarshal because we need to change the timestamp
+	var smartData collector.SmartInfo
+	err = json.Unmarshal(metricsFileData, &smartData)
+	if err != nil {
+		return nil, err
+	}
+
+	daysToSubtractInHours := time.Duration(-1 * 24 * daysToSubtract)
+	smartData.LocalTime.TimeT = time.Now().Add(daysToSubtractInHours * time.Hour).Unix()
+	updatedSmartDataBytes, err := json.Marshal(smartData)
+
+	return bytes.NewReader(updatedSmartDataBytes), nil
+}

webapp/backend/pkg/models/testdata/smart-ata-date.json

@@ -0,0 +1,846 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-4.19.128-flatcar",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "-a",
+      "/dev/sdb"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/sdb",
+    "info_name": "/dev/sdb [SAT]",
+    "type": "sat",
+    "protocol": "ATA"
+  },
+  "model_name": "WDC WD140EDFZ-11A0VA0",
+  "serial_number": "9RK1XXXX",
+  "wwn": {
+    "naa": 5,
+    "oui": 3274,
+    "id": 10283057623
+  },
+  "firmware_version": "81.00A81",
+  "user_capacity": {
+    "blocks": 27344764928,
+    "bytes": 14000519643136
+  },
+  "logical_block_size": 512,
+  "physical_block_size": 4096,
+  "rotation_rate": 5400,
+  "form_factor": {
+    "ata_value": 2,
+    "name": "3.5 inches"
+  },
+  "in_smartctl_database": false,
+  "ata_version": {
+    "string": "ACS-2, ATA8-ACS T13/1699-D revision 4",
+    "major_value": 1020,
+    "minor_value": 41
+  },
+  "sata_version": {
+    "string": "SATA 3.2",
+    "value": 255
+  },
+  "interface_speed": {
+    "max": {
+      "sata_value": 14,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    },
+    "current": {
+      "sata_value": 3,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    }
+  },
+  "local_time": {
+    "time_t": 1637039918,
+    "asctime": "Sun Jun 30 00:03:30 2021 UTC"
+  },
+  "smart_status": {
+    "passed": true
+  },
+  "ata_smart_data": {
+    "offline_data_collection": {
+      "status": {
+        "value": 130,
+        "string": "was completed without error",
+        "passed": true
+      },
+      "completion_seconds": 101
+    },
+    "self_test": {
+      "status": {
+        "value": 241,
+        "string": "in progress, 10% remaining",
+        "remaining_percent": 10
+      },
+      "polling_minutes": {
+        "short": 2,
+        "extended": 1479
+      }
+    },
+    "capabilities": {
+      "values": [
+        91,
+        3
+      ],
+      "exec_offline_immediate_supported": true,
+      "offline_is_aborted_upon_new_cmd": false,
+      "offline_surface_scan_supported": true,
+      "self_tests_supported": true,
+      "conveyance_self_test_supported": false,
+      "selective_self_test_supported": true,
+      "attribute_autosave_enabled": true,
+      "error_logging_supported": true,
+      "gp_logging_supported": true
+    }
+  },
+  "ata_sct_capabilities": {
+    "value": 61,
+    "error_recovery_control_supported": true,
+    "feature_control_supported": true,
+    "data_table_supported": true
+  },
+  "ata_smart_attributes": {
+    "revision": 16,
+    "table": [
+      {
+        "id": 1,
+        "name": "Raw_Read_Error_Rate",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 11,
+          "string": "PO-R-- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 2,
+        "name": "Throughput_Performance",
+        "value": 135,
+        "worst": 135,
+        "thresh": 54,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 108,
+          "string": "108"
+        }
+      },
+      {
+        "id": 3,
+        "name": "Spin_Up_Time",
+        "value": 81,
+        "worst": 81,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 7,
+          "string": "POS--- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 30089675132,
+          "string": "380 (Average 380)"
+        }
+      },
+      {
+        "id": 4,
+        "name": "Start_Stop_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 9,
+          "string": "9"
+        }
+      },
+      {
+        "id": 5,
+        "name": "Reallocated_Sector_Ct",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 51,
+          "string": "PO--CK ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 7,
+        "name": "Seek_Error_Rate",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 8,
+        "name": "Seek_Time_Performance",
+        "value": 133,
+        "worst": 133,
+        "thresh": 20,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 18,
+          "string": "18"
+        }
+      },
+      {
+        "id": 9,
+        "name": "Power_On_Hours",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 1730,
+          "string": "1730"
+        }
+      },
+      {
+        "id": 10,
+        "name": "Spin_Retry_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 12,
+        "name": "Power_Cycle_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 9,
+          "string": "9"
+        }
+      },
+      {
+        "id": 22,
+        "name": "Unknown_Attribute",
+        "value": 100,
+        "worst": 100,
+        "thresh": 25,
+        "when_failed": "",
+        "flags": {
+          "value": 35,
+          "string": "PO---K ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 100,
+          "string": "100"
+        }
+      },
+      {
+        "id": 192,
+        "name": "Power-Off_Retract_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 329,
+          "string": "329"
+        }
+      },
+      {
+        "id": 193,
+        "name": "Load_Cycle_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 329,
+          "string": "329"
+        }
+      },
+      {
+        "id": 194,
+        "name": "Temperature_Celsius",
+        "value": 51,
+        "worst": 51,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 2,
+          "string": "-O---- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 163210330144,
+          "string": "32 (Min/Max 24/38)"
+        }
+      },
+      {
+        "id": 196,
+        "name": "Reallocated_Event_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 197,
+        "name": "Current_Pending_Sector",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 34,
+          "string": "-O---K ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 198,
+        "name": "Offline_Uncorrectable",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 8,
+          "string": "---R-- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 199,
+        "name": "UDMA_CRC_Error_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      }
+    ]
+  },
+  "power_on_time": {
+    "hours": 1730
+  },
+  "power_cycle_count": 9,
+  "temperature": {
+    "current": 32
+  },
+  "ata_smart_error_log": {
+    "summary": {
+      "revision": 1,
+      "count": 0
+    }
+  },
+  "ata_smart_self_test_log": {
+    "standard": {
+      "revision": 1,
+      "table": [
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1708
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1684
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1661
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1636
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1624
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1541
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1517
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1493
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1469
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1445
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1439
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1373
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1349
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1325
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1301
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1277
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1253
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1252
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1205
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1181
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1157
+        }
+      ],
+      "count": 21,
+      "error_count_total": 0,
+      "error_count_outdated": 0
+    }
+  },
+  "ata_smart_selective_self_test_log": {
+    "revision": 1,
+    "table": [
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      }
+    ],
+    "flags": {
+      "value": 0,
+      "remainder_scan_enabled": false
+    },
+    "power_up_scan_resume_minutes": 0
+  }
+}

webapp/backend/pkg/models/testdata/smart-ata-date2.json

@@ -0,0 +1,846 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-4.19.128-flatcar",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "-a",
+      "/dev/sdb"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/sdb",
+    "info_name": "/dev/sdb [SAT]",
+    "type": "sat",
+    "protocol": "ATA"
+  },
+  "model_name": "WDC WD140EDFZ-11A0VA0",
+  "serial_number": "9RK1XXXX",
+  "wwn": {
+    "naa": 5,
+    "oui": 3274,
+    "id": 10283057623
+  },
+  "firmware_version": "81.00A81",
+  "user_capacity": {
+    "blocks": 27344764928,
+    "bytes": 14000519643136
+  },
+  "logical_block_size": 512,
+  "physical_block_size": 4096,
+  "rotation_rate": 5400,
+  "form_factor": {
+    "ata_value": 2,
+    "name": "3.5 inches"
+  },
+  "in_smartctl_database": false,
+  "ata_version": {
+    "string": "ACS-2, ATA8-ACS T13/1699-D revision 4",
+    "major_value": 1020,
+    "minor_value": 41
+  },
+  "sata_version": {
+    "string": "SATA 3.2",
+    "value": 255
+  },
+  "interface_speed": {
+    "max": {
+      "sata_value": 14,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    },
+    "current": {
+      "sata_value": 3,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    }
+  },
+  "local_time": {
+    "time_t": 1637039918,
+    "asctime": "Tue Feb 23 00:03:30 2021 UTC"
+  },
+  "smart_status": {
+    "passed": true
+  },
+  "ata_smart_data": {
+    "offline_data_collection": {
+      "status": {
+        "value": 130,
+        "string": "was completed without error",
+        "passed": true
+      },
+      "completion_seconds": 101
+    },
+    "self_test": {
+      "status": {
+        "value": 241,
+        "string": "in progress, 10% remaining",
+        "remaining_percent": 10
+      },
+      "polling_minutes": {
+        "short": 2,
+        "extended": 1479
+      }
+    },
+    "capabilities": {
+      "values": [
+        91,
+        3
+      ],
+      "exec_offline_immediate_supported": true,
+      "offline_is_aborted_upon_new_cmd": false,
+      "offline_surface_scan_supported": true,
+      "self_tests_supported": true,
+      "conveyance_self_test_supported": false,
+      "selective_self_test_supported": true,
+      "attribute_autosave_enabled": true,
+      "error_logging_supported": true,
+      "gp_logging_supported": true
+    }
+  },
+  "ata_sct_capabilities": {
+    "value": 61,
+    "error_recovery_control_supported": true,
+    "feature_control_supported": true,
+    "data_table_supported": true
+  },
+  "ata_smart_attributes": {
+    "revision": 16,
+    "table": [
+      {
+        "id": 1,
+        "name": "Raw_Read_Error_Rate",
+        "value": 90,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 11,
+          "string": "PO-R-- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 2,
+        "name": "Throughput_Performance",
+        "value": 125,
+        "worst": 135,
+        "thresh": 54,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 118,
+          "string": "108"
+        }
+      },
+      {
+        "id": 3,
+        "name": "Spin_Up_Time",
+        "value": 71,
+        "worst": 81,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 7,
+          "string": "POS--- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 30089675142,
+          "string": "380 (Average 380)"
+        }
+      },
+      {
+        "id": 4,
+        "name": "Start_Stop_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 19,
+          "string": "9"
+        }
+      },
+      {
+        "id": 5,
+        "name": "Reallocated_Sector_Ct",
+        "value": 90,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 51,
+          "string": "PO--CK ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 7,
+        "name": "Seek_Error_Rate",
+        "value": 90,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 8,
+        "name": "Seek_Time_Performance",
+        "value": 123,
+        "worst": 133,
+        "thresh": 20,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 28,
+          "string": "18"
+        }
+      },
+      {
+        "id": 9,
+        "name": "Power_On_Hours",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 1740,
+          "string": "1730"
+        }
+      },
+      {
+        "id": 10,
+        "name": "Spin_Retry_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 12,
+        "name": "Power_Cycle_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 19,
+          "string": "9"
+        }
+      },
+      {
+        "id": 22,
+        "name": "Unknown_Attribute",
+        "value": 90,
+        "worst": 100,
+        "thresh": 25,
+        "when_failed": "",
+        "flags": {
+          "value": 35,
+          "string": "PO---K ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 110,
+          "string": "100"
+        }
+      },
+      {
+        "id": 192,
+        "name": "Power-Off_Retract_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 339,
+          "string": "329"
+        }
+      },
+      {
+        "id": 193,
+        "name": "Load_Cycle_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 339,
+          "string": "329"
+        }
+      },
+      {
+        "id": 194,
+        "name": "Temperature_Celsius",
+        "value": 41,
+        "worst": 51,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 2,
+          "string": "-O---- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 163210330154,
+          "string": "32 (Min/Max 24/38)"
+        }
+      },
+      {
+        "id": 196,
+        "name": "Reallocated_Event_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 197,
+        "name": "Current_Pending_Sector",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 34,
+          "string": "-O---K ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 198,
+        "name": "Offline_Uncorrectable",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 8,
+          "string": "---R-- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 199,
+        "name": "UDMA_CRC_Error_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      }
+    ]
+  },
+  "power_on_time": {
+    "hours": 3030
+  },
+  "power_cycle_count": 9,
+  "temperature": {
+    "current": 62
+  },
+  "ata_smart_error_log": {
+    "summary": {
+      "revision": 1,
+      "count": 0
+    }
+  },
+  "ata_smart_self_test_log": {
+    "standard": {
+      "revision": 1,
+      "table": [
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1708
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1684
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1661
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1636
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1624
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1541
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1517
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1493
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1469
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1445
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1439
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1373
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1349
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1325
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1301
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1277
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1253
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1252
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1205
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1181
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1157
+        }
+      ],
+      "count": 21,
+      "error_count_total": 0,
+      "error_count_outdated": 0
+    }
+  },
+  "ata_smart_selective_self_test_log": {
+    "revision": 1,
+    "table": [
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      }
+    ],
+    "flags": {
+      "value": 0,
+      "remainder_scan_enabled": false
+    },
+    "power_up_scan_resume_minutes": 0
+  }
+}

webapp/backend/pkg/models/testdata/smart-ata-full.json

@@ -70,7 +70,7 @@
     }
   },
   "local_time": {
-    "time_t": 1600014563,
+    "time_t": 1637039918,
     "asctime": "Sun Sep 13 16:29:23 2020 UTC"
   },
   "read_lookahead": {

webapp/backend/pkg/models/testdata/smart-ata.json

@@ -69,7 +69,7 @@
     }
   },
   "local_time": {
-    "time_t": 1592697810,
+    "time_t": 1637039918,
     "asctime": "Sun Jun 21 00:03:30 2020 UTC"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-ata2.json

@@ -66,7 +66,7 @@
     }
   },
   "local_time": {
-    "time_t": 1564664713,
+    "time_t": 1637039918,
     "asctime": "Thu Aug 01 15:05:13 2019 WEDT"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-fail2.json

@@ -70,7 +70,7 @@
     }
   },
   "local_time": {
-    "time_t": 1594216103,
+    "time_t": 1637039918,
     "asctime": "Wed Jul  8 15:48:23 2020 CEST"
   },
   "smart_status": {