!!!!WIP!!!!

adding InfluxDB

- influxdb added to dockerfile
- influxdb s6 service
- influxdb config
- adding defaults to config
- creating a DeviceRepo interface (multiple db backends)
- implemented DeviceRepo interface as ScruitnyRepository

CONTRIBUTING.md

@@ -71,6 +71,33 @@ go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
 Now visit http://localhost:8080
 
 
+If you'd like to populate the database with some test data,  you can run the following commands:
+
+```
+docker run -p 8086:8086 --rm influxdb:2.0
+
+
+docker run -p 8086:8086 \
+      -e DOCKER_INFLUXDB_INIT_USERNAME=admin \
+      -e DOCKER_INFLUXDB_INIT_PASSWORD=12345678 \
+      -e DOCKER_INFLUXDB_INIT_ORG=my-org \
+      -e DOCKER_INFLUXDB_INIT_BUCKET=bucket \
+      influxdb:2.0
+
+
+curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/web/testdata/register-devices-req.json localhost:8080/api/devices/register
+
+curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata-date.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-fail2.json localhost:8080/api/device/0x5000cca264ec3183/smart
+curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-nvme.json localhost:8080/api/device/0x5002538e40a22954/smart
+curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-scsi.json localhost:8080/api/device/0x5000cca252c859cc/smart
+curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-scsi2.json localhost:8080/api/device/0x5000cca264ebc248/smart
+
+curl localhost:8080/api/summary
+
+```
+
 ### Collector
 ```
 brew install smartmontools

docker/Dockerfile

@@ -30,11 +30,16 @@ FROM ubuntu:bionic as runtime
 EXPOSE 8080
 WORKDIR /scrutiny
 ENV PATH="/scrutiny/bin:${PATH}"
+ENV INFLUXD_CONFIG_PATH=/scrutiny/influxdb
 
 RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates curl && update-ca-certificates
 
 ADD https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-amd64.tar.gz /tmp/
 RUN tar xzf /tmp/s6-overlay-amd64.tar.gz -C /
+
+ADD https://dl.influxdata.com/influxdb/releases/influxdb2-2.0.4-amd64.deb /tmp/
+RUN dpkg -i /tmp/influxdb2-2.0.4-amd64.deb && rm -rf /tmp/influxdb2-2.0.4-amd64.deb
+
 COPY /rootfs /
 
 COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny

docker/entrypoint-collector.sh

@@ -7,4 +7,4 @@ printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
 
 # now that we have the env start cron in the foreground
 echo "starting cron"
-cron -f
+su -c "cron -l 8 -f" root

docs/dbdiagram.io.txt

@@ -0,0 +1,62 @@
+
+// SQLite Table(s)
+Table device {
+  created_at timestamp
+
+  wwn varchar [pk]
+
+  //user provided
+  label varchar
+  host_id varchar
+
+  // smartctl provided
+  device_name varchar
+  manufacturer varchar
+  model_name varchar
+  interface_type varchar
+  interface_speed varchar
+  serial_number varchar
+  firmware varchar
+  rotational_speed varchar
+  capacity varchar
+  form_factor varchar
+  smart_support varchar
+  device_protocol varchar
+  device_type varchar
+
+}
+
+
+// InfluxDB Tables
+Table device_temperature {
+  //timestamp
+  created_at timestamp
+
+  //tags (indexed & queryable)
+  device_wwn varchar [pk]
+
+  //fields
+  temp bigint
+ }
+
+
+Table smart_ata_results {
+  //timestamp
+  created_at timestamp
+
+  //tags (indexed & queryable)
+  device_wwn varchar [pk]
+  smart_status varchar
+  scrutiny_status varchar
+
+
+
+  //fields
+  temp bigint
+  power_on_hours bigint
+  power_cycle_count bigint
+
+
+ }
+
+Ref: device.wwn < smart_ata_results.device_wwn

example.scrutiny.yaml

@@ -26,7 +26,12 @@ web:
   src:
     frontend:
       path: /scrutiny/web
-
+  influxdb:
+    host: 0.0.0.0
+    port: 8086
+#    token: 'my-token'
+#    org: 'my-org'
+#    bucket: 'bucket'
 
 log:
   file: '' #absolute or relative paths allowed, eg. web.log

go.mod

@@ -9,6 +9,8 @@ require (
 	github.com/gin-gonic/gin v1.6.3
 	github.com/golang/mock v1.4.3
 	github.com/google/uuid v1.2.0 // indirect
+	github.com/hashicorp/serf v0.8.2
+	github.com/influxdata/influxdb-client-go/v2 v2.2.3
 	github.com/jaypipes/ghw v0.6.1
 	github.com/klauspost/compress v1.12.1 // indirect
 	github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect

go.sum

@@ -24,6 +24,7 @@ github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14 h1:wsrSjiqQtseStRI
 github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14/go.mod h1:lJQVqFKMV5/oDGYR2bra2OljcF3CvolAoyDRyOA4k4E=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -53,9 +54,12 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSY
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deepmap/oapi-codegen v1.3.13 h1:9HKGCsdJqE4dnrQ8VerFS0/1ZOJPmAhN+g8xgp8y3K4=
+github.com/deepmap/oapi-codegen v1.3.13/go.mod h1:WAmG5dWY8/PYHt4vKxlt90NsbHMAOCiteYKZMiIRfOo=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
@@ -69,12 +73,14 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/getkin/kin-openapi v0.13.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-interpreter/wagon v0.5.1-0.20190713202023-55a163980b6c/go.mod h1:5+b/MBYkclRZngKF5s6qrgWxSLgE9F5dFdO1hAueZLc=
 github.com/go-interpreter/wagon v0.6.0/go.mod h1:5+b/MBYkclRZngKF5s6qrgWxSLgE9F5dFdO1hAueZLc=
@@ -119,7 +125,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
@@ -149,28 +157,42 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/influxdb-client-go v1.4.0 h1:+KavOkwhLClHFfYcJMHHnTL5CZQhXJzOm5IKHI9BqJk=
+github.com/influxdata/influxdb-client-go/v2 v2.2.3 h1:082jdJ5t1CFeo0rpGQvKAK1mONVSbFhL4finWA5bRM8=
+github.com/influxdata/influxdb-client-go/v2 v2.2.3/go.mod h1:fa/d1lAdUHxuc1jedx30ZfNG573oQTQmUni3N6pcW+0=
+github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
+github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
 github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
 github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jaypipes/ghw v0.6.1 h1:Ewt3mdpiyhWotGyzg1ursV/6SnToGcG4215X6rR2af8=
@@ -210,6 +232,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 h1:3tLzEnUizyN9YLWFTT9loC30lSBvh2y70LTDcZOTs1s=
 github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0/go.mod h1:8/LTPeDLaklcUjgSQBHbhBF1ibKAFxzS5o+H7USfMSA=
+github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
+github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -220,6 +244,7 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
@@ -229,6 +254,7 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -237,6 +263,7 @@ github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGw
 github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI=
 github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -284,6 +311,8 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -301,6 +330,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -357,6 +387,9 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -373,7 +406,9 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
@@ -414,9 +449,11 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -454,7 +491,9 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -498,6 +537,7 @@ golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

rootfs/etc/services.d/influxdb/run

@@ -0,0 +1,4 @@
+#!/usr/bin/with-contenv bash
+
+echo "starting influxdb"
+influxd run

rootfs/scrutiny/influxdb/config.yaml

@@ -0,0 +1,4 @@
+bolt-path: /scrutiny/influxdb/influxd.bolt
+engine-path: /scrutiny/influxdb/engine
+http-bind-address: ":8086"
+reporting-disabled: true

webapp/backend/pkg/config/config.go

@@ -37,6 +37,13 @@ func (c *configuration) Init() error {
 
 	c.SetDefault("notify.urls", []string{})
 
+	c.SetDefault("web.influxdb.host", "0.0.0.0")
+	c.SetDefault("web.influxdb.port", "8086")
+	c.SetDefault("web.influxdb.org", "scrutiny")
+	c.SetDefault("web.influxdb.bucket", "metrics")
+	c.SetDefault("web.influxdb.init_username", "admin")
+	c.SetDefault("web.influxdb.init_password", "password12345")
+
 	//c.SetDefault("disks.include", []string{})
 	//c.SetDefault("disks.exclude", []string{})
 

webapp/backend/pkg/constants.go

@@ -0,0 +1,28 @@
+package pkg
+
+const DeviceProtocolAta = "ATA"
+const DeviceProtocolScsi = "SCSI"
+const DeviceProtocolNvme = "NVMe"
+
+const SmartAttributeStatusPassed = "passed"
+const SmartAttributeStatusFailed = "failed"
+const SmartAttributeStatusWarning = "warn"
+
+const SmartWhenFailedFailingNow = "FAILING_NOW"
+const SmartWhenFailedInThePast = "IN_THE_PAST"
+
+//const SmartStatusPassed = "passed"
+//const SmartStatusFailed = "failed"
+
+type DeviceStatus int
+
+const (
+	DeviceStatusPassed         DeviceStatus = 0
+	DeviceStatusFailedSmart    DeviceStatus = iota
+	DeviceStatusFailedScrutiny DeviceStatus = iota
+)
+
+func Set(b, flag DeviceStatus) DeviceStatus    { return b | flag }
+func Clear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
+func Toggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
+func Has(b, flag DeviceStatus) bool            { return b&flag != 0 }

webapp/backend/pkg/database/interface.go

@@ -0,0 +1,27 @@
+package database
+
+import (
+	"context"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+)
+
+type DeviceRepo interface {
+	Close() error
+
+	//GetSettings()
+	//SaveSetting()
+
+	RegisterDevice(ctx context.Context, dev models.Device) error
+	GetDevices(ctx context.Context) ([]models.Device, error)
+	UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error)
+	GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error)
+
+	SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error)
+	GetSmartAttributeHistory(ctx context.Context, wwn string, startAt string, attributes []string) ([]measurements.Smart, error)
+
+	SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error
+
+	GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
+}

webapp/backend/pkg/database/scrutiny_repository.go

@@ -0,0 +1,436 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api"
+	"github.com/sirupsen/logrus"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+	"time"
+)
+
+//// GormLogger is a custom logger for Gorm, making it use logrus.
+//type GormLogger struct{ Logger logrus.FieldLogger }
+//
+//// Print handles log events from Gorm for the custom logger.
+//func (gl *GormLogger) Print(v ...interface{}) {
+//	switch v[0] {
+//	case "sql":
+//		gl.Logger.WithFields(
+//			logrus.Fields{
+//				"module":  "gorm",
+//				"type":    "sql",
+//				"rows":    v[5],
+//				"src_ref": v[1],
+//				"values":  v[4],
+//			},
+//		).Debug(v[3])
+//	case "log":
+//		gl.Logger.WithFields(logrus.Fields{"module": "gorm", "type": "log"}).Print(v[2])
+//	}
+//}
+
+func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.FieldLogger) (DeviceRepo, error) {
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// Gorm/SQLite setup
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location"))
+	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
+		//TODO: figure out how to log database queries again.
+		//Logger: logger
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Failed to connect to database!")
+	}
+
+	//database.SetLogger()
+	database.AutoMigrate(&models.Device{})
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// InfluxDB setup
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+	// Create a new client using an InfluxDB server base URL and an authentication token
+	influxdbUrl := fmt.Sprintf("http://%s:%s", appConfig.GetString("web.influxdb.host"), appConfig.GetString("web.influxdb.port"))
+	globalLogger.Debugf("InfluxDB url: %s", influxdbUrl)
+
+	client := influxdb2.NewClient(influxdbUrl, appConfig.GetString("web.influxdb.token"))
+
+	if !appConfig.IsSet("web.influxdb.token") {
+		globalLogger.Debugf("No influxdb token found, running first-time setup...")
+
+		// if no token is provided, but we have a valid server, we're going to assume this is the first setup of our server.
+		// we will initialize with a predetermined username & password, that you should change.
+		onboardingResponse, err := client.Setup(
+			context.Background(),
+			appConfig.GetString("web.influxdb.init_username"),
+			appConfig.GetString("web.influxdb.init_password"),
+			appConfig.GetString("web.influxdb.org"),
+			appConfig.GetString("web.influxdb.bucket"),
+			0)
+		if err != nil {
+			return nil, err
+		}
+
+		appConfig.Set("web.influxdb.token", *onboardingResponse.Auth.Token)
+		//todo: determine if we should write the config file out here.
+	}
+
+	// Use blocking write client for writes to desired bucket
+	writeAPI := client.WriteAPIBlocking(appConfig.GetString("web.influxdb.org"), appConfig.GetString("web.influxdb.bucket"))
+
+	// Get query client
+	queryAPI := client.QueryAPI(appConfig.GetString("web.influxdb.org"))
+
+	if writeAPI == nil || queryAPI == nil {
+		return nil, fmt.Errorf("Failed to connect to influxdb!")
+	}
+
+	deviceRepo := scrutinyRepository{
+		appConfig:      appConfig,
+		logger:         globalLogger,
+		influxClient:   client,
+		influxWriteApi: writeAPI,
+		influxQueryApi: queryAPI,
+		gormClient:     database,
+	}
+
+	return &deviceRepo, nil
+}
+
+type scrutinyRepository struct {
+	appConfig config.Interface
+	logger    logrus.FieldLogger
+
+	influxWriteApi api.WriteAPIBlocking
+	influxQueryApi api.QueryAPI
+	influxClient   influxdb2.Client
+
+	gormClient *gorm.DB
+}
+
+func (sr *scrutinyRepository) Close() error {
+	sr.influxClient.Close()
+	return nil
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Device
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+//insert device into DB (and update specified columns if device is already registered)
+// update device fields that may change: (DeviceType, HostID)
+func (sr *scrutinyRepository) RegisterDevice(ctx context.Context, dev models.Device) error {
+	if err := sr.gormClient.WithContext(ctx).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "wwn"}},
+		DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}),
+	}).Create(&dev).Error; err != nil {
+		return err
+	}
+	return nil
+}
+
+// get a list of all devices (only device metadata, no SMART data)
+func (sr *scrutinyRepository) GetDevices(ctx context.Context) ([]models.Device, error) {
+	//Get a list of all the active devices.
+	devices := []models.Device{}
+	if err := sr.gormClient.WithContext(ctx).Find(&devices).Error; err != nil {
+		return nil, fmt.Errorf("Could not get device summary from DB", err)
+	}
+	return devices, nil
+}
+
+// update device (only metadata) from collector
+func (sr *scrutinyRepository) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) {
+	var device models.Device
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return device, fmt.Errorf("Could not get device from DB", err)
+	}
+
+	//TODO catch GormClient err
+	err := device.UpdateFromCollectorSmartInfo(collectorSmartData)
+	if err != nil {
+		return device, err
+	}
+	return device, sr.gormClient.Model(&device).Updates(device).Error
+}
+
+func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) {
+	var device models.Device
+
+	fmt.Println("GetDeviceDetails from GORM")
+
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return models.Device{}, err
+	}
+
+	return device, nil
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// SMART
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
+	deviceSmartData := measurements.Smart{}
+	err := deviceSmartData.FromCollectorSmartInfo(wwn, collectorSmartData)
+	if err != nil {
+		sr.logger.Errorln("Could not process SMART metrics", err)
+		return measurements.Smart{}, err
+	}
+
+	tags, fields := deviceSmartData.Flatten()
+	p := influxdb2.NewPoint("smart",
+		tags,
+		fields,
+		deviceSmartData.Date)
+
+	// write point immediately
+	return deviceSmartData, sr.influxWriteApi.WritePoint(ctx, p)
+}
+
+func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, startAt string, attributes []string) ([]measurements.Smart, error) {
+	// Get SMartResults from InfluxDB
+
+	fmt.Println("GetDeviceDetails from INFLUXDB")
+
+	//TODO: change the filter startrange to a real number.
+
+	// Get parser flux query result
+	//appConfig.GetString("web.influxdb.bucket")
+	queryStr := fmt.Sprintf(`
+  import "influxdata/influxdb/schema"
+  from(bucket: "%s")
+  |> range(start: -2y, stop: now())
+  |> filter(fn: (r) => r["_measurement"] == "smart" )
+  |> filter(fn: (r) => r["device_wwn"] == "%s" )
+  |> schema.fieldsAsCols()
+  |> group(columns: ["device_wwn"])
+  |> yield(name: "last")
+		`,
+		sr.appConfig.GetString("web.influxdb.bucket"),
+		wwn,
+	)
+
+	smartResults := []measurements.Smart{}
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		fmt.Println("GetDeviceDetails NO EROR")
+
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+			fmt.Println("GetDeviceDetails NEXT")
+
+			// Observe when there is new grouping key producing new table
+			if result.TableChanged() {
+				//fmt.Printf("table: %s\n", result.TableMetadata().String())
+			}
+
+			fmt.Printf("DECODINIG TABLE VALUES: %v", result.Record().Values())
+			smartData, err := measurements.NewSmartFromInfluxDB(result.Record().Values())
+			if err != nil {
+				return nil, err
+			}
+			smartResults = append(smartResults, *smartData)
+
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+
+	return smartResults, nil
+
+	//if err := device.SquashHistory(); err != nil {
+	//	logger.Errorln("An error occurred while squashing device history", err)
+	//	c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+	//	return
+	//}
+	//
+	//if err := device.ApplyMetadataRules(); err != nil {
+	//	logger.Errorln("An error occurred while applying scrutiny thresholds & rules", err)
+	//	c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+	//	return
+	//}
+
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Temperature Data
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error {
+	if len(collectorSmartData.AtaSctTemperatureHistory.Table) > 0 {
+
+		for ndx, temp := range collectorSmartData.AtaSctTemperatureHistory.Table {
+
+			minutesOffset := collectorSmartData.AtaSctTemperatureHistory.LoggingIntervalMinutes * int64(ndx) * 60
+			smartTemp := measurements.SmartTemperature{
+				Date: time.Unix(collectorSmartData.LocalTime.TimeT-minutesOffset, 0),
+				Temp: temp,
+			}
+
+			tags, fields := smartTemp.Flatten()
+			tags["device_wwn"] = wwn
+			p := influxdb2.NewPoint("temp",
+				tags,
+				fields,
+				smartTemp.Date)
+			err := sr.influxWriteApi.WritePoint(ctx, p)
+			if err != nil {
+				return err
+			}
+		}
+		// also add the current temperature.
+	} else {
+
+		smartTemp := measurements.SmartTemperature{
+			Date: time.Unix(collectorSmartData.LocalTime.TimeT, 0),
+			Temp: collectorSmartData.Temperature.Current,
+		}
+
+		tags, fields := smartTemp.Flatten()
+		tags["device_wwn"] = wwn
+		p := influxdb2.NewPoint("temp",
+			tags,
+			fields,
+			smartTemp.Date)
+		return sr.influxWriteApi.WritePoint(ctx, p)
+	}
+	return nil
+}
+
+func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context) (map[string][]measurements.SmartTemperature, error) {
+
+	deviceTempHistory := map[string][]measurements.SmartTemperature{}
+
+	//TODO: change the query range to a variable.
+	queryStr := fmt.Sprintf(`
+  import "influxdata/influxdb/schema"
+  from(bucket: "%s")
+  |> range(start: -3y, stop: now())
+  |> filter(fn: (r) => r["_measurement"] == "temp" )
+  |> filter(fn: (r) => r["_field"] == "temp")
+  |> schema.fieldsAsCols()
+  |> group(columns: ["device_wwn"])
+  |> yield(name: "last")
+		`,
+		sr.appConfig.GetString("web.influxdb.bucket"),
+	)
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+
+			if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
+
+				//check if deviceWWN has been seen and initialized already
+				if _, ok := deviceTempHistory[deviceWWN.(string)]; !ok {
+					deviceTempHistory[deviceWWN.(string)] = []measurements.SmartTemperature{}
+				}
+
+				currentTempHistory := deviceTempHistory[deviceWWN.(string)]
+				smartTemp := measurements.SmartTemperature{}
+
+				for key, val := range result.Record().Values() {
+					smartTemp.Inflate(key, val)
+				}
+				smartTemp.Date = result.Record().Values()["_time"].(time.Time)
+				currentTempHistory = append(currentTempHistory, smartTemp)
+				deviceTempHistory[deviceWWN.(string)] = currentTempHistory
+			}
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+	return deviceTempHistory, nil
+
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// DeviceSummary
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// get a map of all devices and associated SMART data
+func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) {
+	devices, err := sr.GetDevices(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	summaries := map[string]*models.DeviceSummary{}
+
+	for _, device := range devices {
+		summaries[device.WWN] = &models.DeviceSummary{Device: device}
+	}
+
+	// Get parser flux query result
+	//appConfig.GetString("web.influxdb.bucket")
+	queryStr := fmt.Sprintf(`
+  import "influxdata/influxdb/schema"
+  from(bucket: "%s")
+  |> range(start: -1y, stop: now())
+  |> filter(fn: (r) => r["_measurement"] == "smart" )
+  |> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+  |> schema.fieldsAsCols()
+  |> group(columns: ["device_wwn"])
+  |> yield(name: "last")
+		`,
+		sr.appConfig.GetString("web.influxdb.bucket"),
+	)
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+			// Observe when there is new grouping key producing new table
+			if result.TableChanged() {
+				//fmt.Printf("table: %s\n", result.TableMetadata().String())
+			}
+			// read result
+
+			//get summary data from Influxdb.
+			//result.Record().Values()
+			if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
+				summaries[deviceWWN.(string)].SmartResults = &models.SmartSummary{
+					Temp:          result.Record().Values()["temp"].(int64),
+					PowerOnHours:  result.Record().Values()["power_on_hours"].(int64),
+					CollectorDate: result.Record().Values()["_time"].(time.Time),
+				}
+			}
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+
+	deviceTempHistory, err := sr.GetSmartTemperatureHistory(ctx)
+	if err != nil {
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("Error: %v", err)
+	}
+	for wwn, tempHistory := range deviceTempHistory {
+		summaries[wwn].TempHistory = tempHistory
+	}
+
+	return summaries, nil
+}

webapp/backend/pkg/metadata/ata_attribute_metadata.go

@@ -11,10 +11,10 @@ type AtaAttributeMetadata struct {
 	Critical    bool   `json:"critical"`
 	Description string `json:"description"`
 
-	Transform          func(int, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
+	Transform          func(int64, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
-	TransformValueUnit string                         `json:"transform_value_unit,omitempty"`
+	TransformValueUnit string                           `json:"transform_value_unit,omitempty"`
-	ObservedThresholds []ObservedThreshold            `json:"observed_thresholds,omitempty"` //these thresholds must match the DisplayType
+	ObservedThresholds []ObservedThreshold              `json:"observed_thresholds,omitempty"` //these thresholds must match the DisplayType
-	DisplayType        string                         `json:"display_type"`                  //"raw" "normalized" or "transformed"
+	DisplayType        string                           `json:"display_type"`                  //"raw" "normalized" or "transformed"
 }
 
 const ObservedThresholdIdealLow = "low"
@@ -1014,7 +1014,7 @@ var AtaMetadata = map[int]AtaAttributeMetadata{
 		Ideal:       ObservedThresholdIdealLow,
 		Critical:    false,
 		Description: "Indicates the device temperature, if the appropriate sensor is fitted. Lowest byte of the raw value contains the exact temperature value (Celsius degrees).",
-		Transform: func(normValue int, rawValue int64, rawString string) int64 {
+		Transform: func(normValue int64, rawValue int64, rawString string) int64 {
 			return rawValue & 0b11111111
 		},
 		TransformValueUnit: "°C",

webapp/backend/pkg/metadata/nvme_attribute_metadata.go

@@ -11,9 +11,9 @@ type NvmeAttributeMetadata struct {
 	Critical    bool   `json:"critical"`
 	Description string `json:"description"`
 
-	Transform          func(int, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
+	Transform          func(int64, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
-	TransformValueUnit string                         `json:"transform_value_unit,omitempty"`
+	TransformValueUnit string                           `json:"transform_value_unit,omitempty"`
-	DisplayType        string                         `json:"display_type"` //"raw" "normalized" or "transformed"
+	DisplayType        string                           `json:"display_type"` //"raw" "normalized" or "transformed"
 }
 
 var NmveMetadata = map[string]NvmeAttributeMetadata{

webapp/backend/pkg/metadata/scsi_attribute_metadata.go

@@ -7,9 +7,9 @@ type ScsiAttributeMetadata struct {
 	Critical    bool   `json:"critical"`
 	Description string `json:"description"`
 
-	Transform          func(int, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
+	Transform          func(int64, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
-	TransformValueUnit string                         `json:"transform_value_unit,omitempty"`
+	TransformValueUnit string                           `json:"transform_value_unit,omitempty"`
-	DisplayType        string                         `json:"display_type"` //"raw" "normalized" or "transformed"
+	DisplayType        string                           `json:"display_type"` //"raw" "normalized" or "transformed"
 }
 
 var ScsiMetadata = map[string]ScsiAttributeMetadata{
@@ -21,96 +21,96 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		Critical:    true,
 		Description: "",
 	},
-	"read.errors_corrected_by_eccfast": {
+	"read_errors_corrected_by_eccfast": {
-		ID:          "read.errors_corrected_by_eccfast",
+		ID:          "read_errors_corrected_by_eccfast",
 		DisplayName: "Read Errors Corrected by ECC Fast",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"read.errors_corrected_by_eccdelayed": {
+	"read_errors_corrected_by_eccdelayed": {
-		ID:          "read.errors_corrected_by_eccdelayed",
+		ID:          "read_errors_corrected_by_eccdelayed",
 		DisplayName: "Read Errors Corrected by ECC Delayed",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"read.errors_corrected_by_rereads_rewrites": {
+	"read_errors_corrected_by_rereads_rewrites": {
-		ID:          "read.errors_corrected_by_rereads_rewrites",
+		ID:          "read_errors_corrected_by_rereads_rewrites",
 		DisplayName: "Read Errors Corrected by ReReads/ReWrites",
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
 		Description: "",
 	},
-	"read.total_errors_corrected": {
+	"read_total_errors_corrected": {
-		ID:          "read.total_errors_corrected",
+		ID:          "read_total_errors_corrected",
 		DisplayName: "Read Total Errors Corrected",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"read.correction_algorithm_invocations": {
+	"read_correction_algorithm_invocations": {
-		ID:          "read.correction_algorithm_invocations",
+		ID:          "read_correction_algorithm_invocations",
 		DisplayName: "Read Correction Algorithm Invocations",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"read.total_uncorrected_errors": {
+	"read_total_uncorrected_errors": {
-		ID:          "read.total_uncorrected_errors",
+		ID:          "read_total_uncorrected_errors",
 		DisplayName: "Read Total Uncorrected Errors",
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
 		Description: "",
 	},
-	"write.errors_corrected_by_eccfast": {
+	"write_errors_corrected_by_eccfast": {
-		ID:          "write.errors_corrected_by_eccfast",
+		ID:          "write_errors_corrected_by_eccfast",
 		DisplayName: "Write Errors Corrected by ECC Fast",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"write.errors_corrected_by_eccdelayed": {
+	"write_errors_corrected_by_eccdelayed": {
-		ID:          "write.errors_corrected_by_eccdelayed",
+		ID:          "write_errors_corrected_by_eccdelayed",
 		DisplayName: "Write Errors Corrected by ECC Delayed",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"write.errors_corrected_by_rereads_rewrites": {
+	"write_errors_corrected_by_rereads_rewrites": {
-		ID:          "write.errors_corrected_by_rereads_rewrites",
+		ID:          "write_errors_corrected_by_rereads_rewrites",
 		DisplayName: "Write Errors Corrected by ReReads/ReWrites",
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
 		Description: "",
 	},
-	"write.total_errors_corrected": {
+	"write_total_errors_corrected": {
-		ID:          "write.total_errors_corrected",
+		ID:          "write_total_errors_corrected",
 		DisplayName: "Write Total Errors Corrected",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"write.correction_algorithm_invocations": {
+	"write_correction_algorithm_invocations": {
-		ID:          "write.correction_algorithm_invocations",
+		ID:          "write_correction_algorithm_invocations",
 		DisplayName: "Write Correction Algorithm Invocations",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"write.total_uncorrected_errors": {
+	"write_total_uncorrected_errors": {
-		ID:          "write.total_uncorrected_errors",
+		ID:          "write_total_uncorrected_errors",
 		DisplayName: "Write Total Uncorrected Errors",
 		DisplayType: "",
 		Ideal:       "low",

webapp/backend/pkg/models/collector/smart.go

@@ -119,14 +119,28 @@ type SmartInfo struct {
 		FeatureControlSupported       bool `json:"feature_control_supported"`
 		DataTableSupported            bool `json:"data_table_supported"`
 	} `json:"ata_sct_capabilities"`
+	AtaSctTemperatureHistory struct {
+		Version                int   `json:"version"`
+		SamplingPeriodMinutes  int64 `json:"sampling_period_minutes"`
+		LoggingIntervalMinutes int64 `json:"logging_interval_minutes"`
+		Temperature            struct {
+			OpLimitMin int `json:"op_limit_min"`
+			OpLimitMax int `json:"op_limit_max"`
+			LimitMin   int `json:"limit_min"`
+			LimitMax   int `json:"limit_max"`
+		} `json:"temperature"`
+		Size  int     `json:"size"`
+		Index int     `json:"index"`
+		Table []int64 `json:"table"`
+	} `json:"ata_sct_temperature_history"`
 	AtaSmartAttributes struct {
 		Revision int `json:"revision"`
 		Table    []struct {
 			ID         int    `json:"id"`
 			Name       string `json:"name"`
-			Value      int    `json:"value"`
+			Value      int64  `json:"value"`
-			Worst      int    `json:"worst"`
+			Worst      int64  `json:"worst"`
-			Thresh     int    `json:"thresh"`
+			Thresh     int64  `json:"thresh"`
 			WhenFailed string `json:"when_failed"`
 			Flags      struct {
 				Value         int    `json:"value"`
@@ -237,48 +251,48 @@ type SmartInfo struct {
 		FormattedLbaSize int `json:"formatted_lba_size"`
 	} `json:"nvme_namespaces"`
 	NvmeSmartHealthInformationLog struct {
-		CriticalWarning         int `json:"critical_warning"`
+		CriticalWarning         int64 `json:"critical_warning"`
-		Temperature             int `json:"temperature"`
+		Temperature             int64 `json:"temperature"`
-		AvailableSpare          int `json:"available_spare"`
+		AvailableSpare          int64 `json:"available_spare"`
-		AvailableSpareThreshold int `json:"available_spare_threshold"`
+		AvailableSpareThreshold int64 `json:"available_spare_threshold"`
-		PercentageUsed          int `json:"percentage_used"`
+		PercentageUsed          int64 `json:"percentage_used"`
-		DataUnitsRead           int `json:"data_units_read"`
+		DataUnitsRead           int64 `json:"data_units_read"`
-		DataUnitsWritten        int `json:"data_units_written"`
+		DataUnitsWritten        int64 `json:"data_units_written"`
-		HostReads               int `json:"host_reads"`
+		HostReads               int64 `json:"host_reads"`
-		HostWrites              int `json:"host_writes"`
+		HostWrites              int64 `json:"host_writes"`
-		ControllerBusyTime      int `json:"controller_busy_time"`
+		ControllerBusyTime      int64 `json:"controller_busy_time"`
-		PowerCycles             int `json:"power_cycles"`
+		PowerCycles             int64 `json:"power_cycles"`
-		PowerOnHours            int `json:"power_on_hours"`
+		PowerOnHours            int64 `json:"power_on_hours"`
-		UnsafeShutdowns         int `json:"unsafe_shutdowns"`
+		UnsafeShutdowns         int64 `json:"unsafe_shutdowns"`
-		MediaErrors             int `json:"media_errors"`
+		MediaErrors             int64 `json:"media_errors"`
-		NumErrLogEntries        int `json:"num_err_log_entries"`
+		NumErrLogEntries        int64 `json:"num_err_log_entries"`
-		WarningTempTime         int `json:"warning_temp_time"`
+		WarningTempTime         int64 `json:"warning_temp_time"`
-		CriticalCompTime        int `json:"critical_comp_time"`
+		CriticalCompTime        int64 `json:"critical_comp_time"`
 	} `json:"nvme_smart_health_information_log"`
 
 	// SCSI Protocol Specific Fields
 	Vendor              string `json:"vendor"`
 	Product             string `json:"product"`
 	ScsiVersion         string `json:"scsi_version"`
-	ScsiGrownDefectList int    `json:"scsi_grown_defect_list"`
+	ScsiGrownDefectList int64  `json:"scsi_grown_defect_list"`
 	ScsiErrorCounterLog struct {
 		Read struct {
-			ErrorsCorrectedByEccfast         int    `json:"errors_corrected_by_eccfast"`
+			ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
-			ErrorsCorrectedByEccdelayed      int    `json:"errors_corrected_by_eccdelayed"`
+			ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
-			ErrorsCorrectedByRereadsRewrites int    `json:"errors_corrected_by_rereads_rewrites"`
+			ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
-			TotalErrorsCorrected             int    `json:"total_errors_corrected"`
+			TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
-			CorrectionAlgorithmInvocations   int    `json:"correction_algorithm_invocations"`
+			CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
 			GigabytesProcessed               string `json:"gigabytes_processed"`
-			TotalUncorrectedErrors           int    `json:"total_uncorrected_errors"`
+			TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
 		} `json:"read"`
 		Write struct {
-			ErrorsCorrectedByEccfast         int    `json:"errors_corrected_by_eccfast"`
+			ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
-			ErrorsCorrectedByEccdelayed      int    `json:"errors_corrected_by_eccdelayed"`
+			ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
-			ErrorsCorrectedByRereadsRewrites int    `json:"errors_corrected_by_rereads_rewrites"`
+			ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
-			TotalErrorsCorrected             int    `json:"total_errors_corrected"`
+			TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
-			CorrectionAlgorithmInvocations   int    `json:"correction_algorithm_invocations"`
+			CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
 			GigabytesProcessed               string `json:"gigabytes_processed"`
-			TotalUncorrectedErrors           int    `json:"total_uncorrected_errors"`
+			TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
 		} `json:"write"`
 	} `json:"scsi_error_counter_log"`
 }

webapp/backend/pkg/models/db/device.go

@@ -1,160 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"time"
-)
-
-type DeviceWrapper struct {
-	Success bool     `json:"success"`
-	Errors  []error  `json:"errors"`
-	Data    []Device `json:"data"`
-}
-
-const DeviceProtocolAta = "ATA"
-const DeviceProtocolScsi = "SCSI"
-const DeviceProtocolNvme = "NVMe"
-
-type Device struct {
-	//GORM attributes, see: http://gorm.io/docs/conventions.html
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt *time.Time
-
-	WWN    string `json:"wwn" gorm:"primary_key"`
-	HostId string `json:"host_id"`
-
-	DeviceName     string  `json:"device_name"`
-	Manufacturer   string  `json:"manufacturer"`
-	ModelName      string  `json:"model_name"`
-	InterfaceType  string  `json:"interface_type"`
-	InterfaceSpeed string  `json:"interface_speed"`
-	SerialNumber   string  `json:"serial_number"`
-	Firmware       string  `json:"firmware"`
-	RotationSpeed  int     `json:"rotational_speed"`
-	Capacity       int64   `json:"capacity"`
-	FormFactor     string  `json:"form_factor"`
-	SmartSupport   bool    `json:"smart_support"`
-	DeviceProtocol string  `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
-	DeviceType     string  `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
-	SmartResults   []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
-}
-
-func (dv *Device) IsAta() bool {
-	return dv.DeviceProtocol == DeviceProtocolAta
-}
-
-func (dv *Device) IsScsi() bool {
-	return dv.DeviceProtocol == DeviceProtocolScsi
-}
-
-func (dv *Device) IsNvme() bool {
-	return dv.DeviceProtocol == DeviceProtocolNvme
-}
-
-//This method requires a device with an array of SmartResults.
-//It will remove all SmartResults other than the first (the latest one)
-//All removed SmartResults, will be processed, grouping SmartAtaAttribute by attribute_id
-// and adding theme to an array called History.
-func (dv *Device) SquashHistory() error {
-	if len(dv.SmartResults) <= 1 {
-		return nil //no ataHistory found. ignore
-	}
-
-	latestSmartResultSlice := dv.SmartResults[0:1]
-	historicalSmartResultSlice := dv.SmartResults[1:]
-
-	//re-assign the latest slice to the SmartResults field
-	dv.SmartResults = latestSmartResultSlice
-
-	//process the historical slice for ATA data
-	if len(dv.SmartResults[0].AtaAttributes) > 0 {
-		ataHistory := map[int][]SmartAtaAttribute{}
-		for _, smartResult := range historicalSmartResultSlice {
-			for _, smartAttribute := range smartResult.AtaAttributes {
-				if _, ok := ataHistory[smartAttribute.AttributeId]; !ok {
-					ataHistory[smartAttribute.AttributeId] = []SmartAtaAttribute{}
-				}
-				ataHistory[smartAttribute.AttributeId] = append(ataHistory[smartAttribute.AttributeId], smartAttribute)
-			}
-		}
-
-		//now assign the historical slices to the AtaAttributes in the latest SmartResults
-		for sandx, smartAttribute := range dv.SmartResults[0].AtaAttributes {
-			if attributeHistory, ok := ataHistory[smartAttribute.AttributeId]; ok {
-				dv.SmartResults[0].AtaAttributes[sandx].History = attributeHistory
-			}
-		}
-	}
-
-	//process the historical slice for Nvme data
-	if len(dv.SmartResults[0].NvmeAttributes) > 0 {
-		nvmeHistory := map[string][]SmartNvmeAttribute{}
-		for _, smartResult := range historicalSmartResultSlice {
-			for _, smartAttribute := range smartResult.NvmeAttributes {
-				if _, ok := nvmeHistory[smartAttribute.AttributeId]; !ok {
-					nvmeHistory[smartAttribute.AttributeId] = []SmartNvmeAttribute{}
-				}
-				nvmeHistory[smartAttribute.AttributeId] = append(nvmeHistory[smartAttribute.AttributeId], smartAttribute)
-			}
-		}
-
-		//now assign the historical slices to the AtaAttributes in the latest SmartResults
-		for sandx, smartAttribute := range dv.SmartResults[0].NvmeAttributes {
-			if attributeHistory, ok := nvmeHistory[smartAttribute.AttributeId]; ok {
-				dv.SmartResults[0].NvmeAttributes[sandx].History = attributeHistory
-			}
-		}
-	}
-	//process the historical slice for Scsi data
-	if len(dv.SmartResults[0].ScsiAttributes) > 0 {
-		scsiHistory := map[string][]SmartScsiAttribute{}
-		for _, smartResult := range historicalSmartResultSlice {
-			for _, smartAttribute := range smartResult.ScsiAttributes {
-				if _, ok := scsiHistory[smartAttribute.AttributeId]; !ok {
-					scsiHistory[smartAttribute.AttributeId] = []SmartScsiAttribute{}
-				}
-				scsiHistory[smartAttribute.AttributeId] = append(scsiHistory[smartAttribute.AttributeId], smartAttribute)
-			}
-		}
-
-		//now assign the historical slices to the AtaAttributes in the latest SmartResults
-		for sandx, smartAttribute := range dv.SmartResults[0].ScsiAttributes {
-			if attributeHistory, ok := scsiHistory[smartAttribute.AttributeId]; ok {
-				dv.SmartResults[0].ScsiAttributes[sandx].History = attributeHistory
-			}
-		}
-	}
-	return nil
-}
-
-func (dv *Device) ApplyMetadataRules() error {
-
-	//embed metadata in the latest smart attributes object
-	if len(dv.SmartResults) > 0 {
-		for ndx, attr := range dv.SmartResults[0].AtaAttributes {
-			attr.PopulateAttributeStatus()
-			dv.SmartResults[0].AtaAttributes[ndx] = attr
-		}
-
-		for ndx, attr := range dv.SmartResults[0].NvmeAttributes {
-			attr.PopulateAttributeStatus()
-			dv.SmartResults[0].NvmeAttributes[ndx] = attr
-
-		}
-
-		for ndx, attr := range dv.SmartResults[0].ScsiAttributes {
-			attr.PopulateAttributeStatus()
-			dv.SmartResults[0].ScsiAttributes[ndx] = attr
-
-		}
-	}
-	return nil
-}
-
-// This function is called every time the collector sends SMART data to the API.
-// It can be used to update device data that can change over time.
-func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
-	dv.Firmware = info.FirmwareVersion
-	return nil
-}

webapp/backend/pkg/models/db/selftest.go

@@ -1,15 +0,0 @@
-package db
-
-import "time"
-
-type SelfTest struct {
-	//GORM attributes, see: http://gorm.io/docs/conventions.html
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt *time.Time
-
-	DeviceWWN string
-	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
-
-	Date time.Time
-}

webapp/backend/pkg/models/db/smart.go

@@ -1,127 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"gorm.io/gorm"
-	"time"
-)
-
-const SmartWhenFailedFailingNow = "FAILING_NOW"
-const SmartWhenFailedInThePast = "IN_THE_PAST"
-
-const SmartStatusPassed = "passed"
-const SmartStatusFailed = "failed"
-
-type Smart struct {
-	gorm.Model
-
-	DeviceWWN string `json:"device_wwn"`
-	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
-
-	TestDate    time.Time `json:"date"`
-	SmartStatus string    `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed
-
-	//Metrics
-	Temp            int64 `json:"temp"`
-	PowerOnHours    int64 `json:"power_on_hours"`
-	PowerCycleCount int64 `json:"power_cycle_count"`
-
-	AtaAttributes  []SmartAtaAttribute  `json:"ata_attributes" gorm:"foreignkey:SmartId"`
-	NvmeAttributes []SmartNvmeAttribute `json:"nvme_attributes" gorm:"foreignkey:SmartId"`
-	ScsiAttributes []SmartScsiAttribute `json:"scsi_attributes" gorm:"foreignkey:SmartId"`
-}
-
-//Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries)
-func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error {
-	sm.DeviceWWN = wwn
-	sm.TestDate = time.Unix(info.LocalTime.TimeT, 0)
-
-	//smart metrics
-	sm.Temp = info.Temperature.Current
-	sm.PowerCycleCount = info.PowerCycleCount
-	sm.PowerOnHours = info.PowerOnTime.Hours
-
-	// process ATA/NVME/SCSI protocol data
-	if info.Device.Protocol == DeviceProtocolAta {
-		sm.ProcessAtaSmartInfo(info)
-	} else if info.Device.Protocol == DeviceProtocolNvme {
-		sm.ProcessNvmeSmartInfo(info)
-	} else if info.Device.Protocol == DeviceProtocolScsi {
-		sm.ProcessScsiSmartInfo(info)
-	}
-
-	if info.SmartStatus.Passed {
-		sm.SmartStatus = SmartStatusPassed
-	} else {
-		sm.SmartStatus = SmartStatusFailed
-	}
-	return nil
-}
-
-//generate SmartAtaAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) {
-	sm.AtaAttributes = []SmartAtaAttribute{}
-	for _, collectorAttr := range info.AtaSmartAttributes.Table {
-		attrModel := SmartAtaAttribute{
-			AttributeId: collectorAttr.ID,
-			Name:        collectorAttr.Name,
-			Value:       collectorAttr.Value,
-			Worst:       collectorAttr.Worst,
-			Threshold:   collectorAttr.Thresh,
-			RawValue:    collectorAttr.Raw.Value,
-			RawString:   collectorAttr.Raw.String,
-			WhenFailed:  collectorAttr.WhenFailed,
-		}
-
-		//now that we've parsed the data from the smartctl response, lets match it against our metadata rules and add additional Scrutiny specific data.
-		if smartMetadata, ok := metadata.AtaMetadata[collectorAttr.ID]; ok {
-			attrModel.Name = smartMetadata.DisplayName
-			if smartMetadata.Transform != nil {
-				attrModel.TransformedValue = smartMetadata.Transform(attrModel.Value, attrModel.RawValue, attrModel.RawString)
-			}
-		}
-		sm.AtaAttributes = append(sm.AtaAttributes, attrModel)
-	}
-}
-
-//generate SmartNvmeAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) {
-	sm.NvmeAttributes = []SmartNvmeAttribute{
-		{AttributeId: "critical_warning", Name: "Critical Warning", Value: info.NvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0},
-		{AttributeId: "temperature", Name: "Temperature", Value: info.NvmeSmartHealthInformationLog.Temperature, Threshold: -1},
-		{AttributeId: "available_spare", Name: "Available Spare", Value: info.NvmeSmartHealthInformationLog.AvailableSpare, Threshold: info.NvmeSmartHealthInformationLog.AvailableSpareThreshold},
-		{AttributeId: "percentage_used", Name: "Percentage Used", Value: info.NvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100},
-		{AttributeId: "data_units_read", Name: "Data Units Read", Value: info.NvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1},
-		{AttributeId: "data_units_written", Name: "Data Units Written", Value: info.NvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1},
-		{AttributeId: "host_reads", Name: "Host Reads", Value: info.NvmeSmartHealthInformationLog.HostReads, Threshold: -1},
-		{AttributeId: "host_writes", Name: "Host Writes", Value: info.NvmeSmartHealthInformationLog.HostWrites, Threshold: -1},
-		{AttributeId: "controller_busy_time", Name: "Controller Busy Time", Value: info.NvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1},
-		{AttributeId: "power_cycles", Name: "Power Cycles", Value: info.NvmeSmartHealthInformationLog.PowerCycles, Threshold: -1},
-		{AttributeId: "power_on_hours", Name: "Power on Hours", Value: info.NvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1},
-		{AttributeId: "unsafe_shutdowns", Name: "Unsafe Shutdowns", Value: info.NvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1},
-		{AttributeId: "media_errors", Name: "Media Errors", Value: info.NvmeSmartHealthInformationLog.MediaErrors, Threshold: 0},
-		{AttributeId: "num_err_log_entries", Name: "Numb Err Log Entries", Value: info.NvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0},
-		{AttributeId: "warning_temp_time", Name: "Warning Temp Time", Value: info.NvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1},
-		{AttributeId: "critical_comp_time", Name: "Critical CompTime", Value: info.NvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1},
-	}
-}
-
-//generate SmartScsiAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessScsiSmartInfo(info collector.SmartInfo) {
-	sm.ScsiAttributes = []SmartScsiAttribute{
-		{AttributeId: "scsi_grown_defect_list", Name: "Grown Defect List", Value: info.ScsiGrownDefectList, Threshold: 0},
-		{AttributeId: "read.errors_corrected_by_eccfast", Name: "Read Errors Corrected by ECC Fast", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1},
-		{AttributeId: "read.errors_corrected_by_eccdelayed", Name: "Read Errors Corrected by ECC Delayed", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1},
-		{AttributeId: "read.errors_corrected_by_rereads_rewrites", Name: "Read Errors Corrected by ReReads/ReWrites", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0},
-		{AttributeId: "read.total_errors_corrected", Name: "Read Total Errors Corrected", Value: info.ScsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1},
-		{AttributeId: "read.correction_algorithm_invocations", Name: "Read Correction Algorithm Invocations", Value: info.ScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1},
-		{AttributeId: "read.total_uncorrected_errors", Name: "Read Total Uncorrected Errors", Value: info.ScsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0},
-		{AttributeId: "write.errors_corrected_by_eccfast", Name: "Write Errors Corrected by ECC Fast", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1},
-		{AttributeId: "write.errors_corrected_by_eccdelayed", Name: "Write Errors Corrected by ECC Delayed", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1},
-		{AttributeId: "write.errors_corrected_by_rereads_rewrites", Name: "Write Errors Corrected by ReReads/ReWrites", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0},
-		{AttributeId: "write.total_errors_corrected", Name: "Write Total Errors Corrected", Value: info.ScsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1},
-		{AttributeId: "write.correction_algorithm_invocations", Name: "Write Correction Algorithm Invocations", Value: info.ScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1},
-		{AttributeId: "write.total_uncorrected_errors", Name: "Write Total Uncorrected Errors", Value: info.ScsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0},
-	}
-}

webapp/backend/pkg/models/db/smart_ata_attribute.go

@@ -1,111 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"gorm.io/gorm"
-	"strings"
-)
-
-const SmartAttributeStatusPassed = "passed"
-const SmartAttributeStatusFailed = "failed"
-const SmartAttributeStatusWarning = "warn"
-
-type SmartAtaAttribute struct {
-	gorm.Model
-
-	SmartId int    `json:"smart_id"`
-	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
-
-	AttributeId int    `json:"attribute_id"`
-	Name        string `json:"name"`
-	Value       int    `json:"value"`
-	Worst       int    `json:"worst"`
-	Threshold   int    `json:"thresh"`
-	RawValue    int64  `json:"raw_value"`
-	RawString   string `json:"raw_string"`
-	WhenFailed  string `json:"when_failed"`
-
-	TransformedValue int64               `json:"transformed_value"`
-	Status           string              `gorm:"-" json:"status,omitempty"`
-	StatusReason     string              `gorm:"-" json:"status_reason,omitempty"`
-	FailureRate      float64             `gorm:"-" json:"failure_rate,omitempty"`
-	History          []SmartAtaAttribute `gorm:"-" json:"history,omitempty"`
-}
-
-//populate attribute status, using SMART Thresholds & Observed Metadata
-func (sa *SmartAtaAttribute) PopulateAttributeStatus() {
-	if strings.ToUpper(sa.WhenFailed) == SmartWhenFailedFailingNow {
-		//this attribute has previously failed
-		sa.Status = SmartAttributeStatusFailed
-		sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
-
-	} else if strings.ToUpper(sa.WhenFailed) == SmartWhenFailedInThePast {
-		sa.Status = SmartAttributeStatusWarning
-		sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
-	}
-
-	if smartMetadata, ok := metadata.AtaMetadata[sa.AttributeId]; ok {
-		sa.MetadataObservedThresholdStatus(smartMetadata)
-	}
-
-	//check if status is blank, set to "passed"
-	if len(sa.Status) == 0 {
-		sa.Status = SmartAttributeStatusPassed
-	}
-}
-
-// compare the attribute (raw, normalized, transformed) value to observed thresholds, and update status if necessary
-func (sa *SmartAtaAttribute) MetadataObservedThresholdStatus(smartMetadata metadata.AtaAttributeMetadata) {
-	//TODO: multiple rules
-	// try to predict the failure rates for observed thresholds that have 0 failure rate and error bars.
-	// - if the attribute is critical
-	//		- the failure rate is over 10 - set to failed
-	//		- the attribute does not match any threshold, set to warn
-	// - if the attribute is not critical
-	//		- if failure rate is above 20 - set to failed
-	// 		- if failure rate is above 10 but below 20 - set to warn
-
-	//update the smart attribute status based on Observed thresholds.
-	var value int64
-	if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeNormalized {
-		value = int64(sa.Value)
-	} else if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeTransformed {
-		value = sa.TransformedValue
-	} else {
-		value = sa.RawValue
-	}
-
-	for _, obsThresh := range smartMetadata.ObservedThresholds {
-
-		//check if "value" is in this bucket
-		if ((obsThresh.Low == obsThresh.High) && value == obsThresh.Low) ||
-			(obsThresh.Low < value && value <= obsThresh.High) {
-			sa.FailureRate = obsThresh.AnnualFailureRate
-
-			if smartMetadata.Critical {
-				if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = SmartAttributeStatusFailed
-					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
-				}
-			} else {
-				if obsThresh.AnnualFailureRate >= 0.20 {
-					sa.Status = SmartAttributeStatusFailed
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
-				} else if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = SmartAttributeStatusWarning
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
-				}
-			}
-
-			//we've found the correct bucket, we can drop out of this loop
-			return
-		}
-	}
-	// no bucket found
-	if smartMetadata.Critical {
-		sa.Status = SmartAttributeStatusWarning
-		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
-	}
-
-	return
-}

webapp/backend/pkg/models/db/smart_nvme_attribute.go

@@ -1,46 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"gorm.io/gorm"
-)
-
-type SmartNvmeAttribute struct {
-	gorm.Model
-
-	SmartId int    `json:"smart_id"`
-	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
-
-	AttributeId string `json:"attribute_id"` //json string from smartctl
-	Name        string `json:"name"`
-	Value       int    `json:"value"`
-	Threshold   int    `json:"thresh"`
-
-	TransformedValue int64                `json:"transformed_value"`
-	Status           string               `gorm:"-" json:"status,omitempty"`
-	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
-	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
-	History          []SmartNvmeAttribute `gorm:"-" json:"history,omitempty"`
-}
-
-//populate attribute status, using SMART Thresholds & Observed Metadata
-func (sa *SmartNvmeAttribute) PopulateAttributeStatus() {
-
-	//-1 is a special number meaning no threshold.
-	if sa.Threshold != -1 {
-		if smartMetadata, ok := metadata.NmveMetadata[sa.AttributeId]; ok {
-			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
-			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
-				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = SmartAttributeStatusFailed
-				sa.StatusReason = "Attribute is failing recommended SMART threshold"
-			}
-		}
-	}
-	//TODO: eventually figure out the critical_warning bits and determine correct error messages here.
-
-	//check if status is blank, set to "passed"
-	if len(sa.Status) == 0 {
-		sa.Status = SmartAttributeStatusPassed
-	}
-}

webapp/backend/pkg/models/db/smart_scsci_attribute.go

@@ -1,45 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"gorm.io/gorm"
-)
-
-type SmartScsiAttribute struct {
-	gorm.Model
-
-	SmartId int    `json:"smart_id"`
-	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
-
-	AttributeId string `json:"attribute_id"` //json string from smartctl
-	Name        string `json:"name"`
-	Value       int    `json:"value"`
-	Threshold   int    `json:"thresh"`
-
-	TransformedValue int64                `json:"transformed_value"`
-	Status           string               `gorm:"-" json:"status,omitempty"`
-	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
-	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
-	History          []SmartScsiAttribute `gorm:"-" json:"history,omitempty"`
-}
-
-//populate attribute status, using SMART Thresholds & Observed Metadata
-func (sa *SmartScsiAttribute) PopulateAttributeStatus() {
-
-	//-1 is a special number meaning no threshold.
-	if sa.Threshold != -1 {
-		if smartMetadata, ok := metadata.NmveMetadata[sa.AttributeId]; ok {
-			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
-			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
-				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = SmartAttributeStatusFailed
-				sa.StatusReason = "Attribute is failing recommended SMART threshold"
-			}
-		}
-	}
-
-	//check if status is blank, set to "passed"
-	if len(sa.Status) == 0 {
-		sa.Status = SmartAttributeStatusPassed
-	}
-}

webapp/backend/pkg/models/db/smart_test.go

@@ -1,155 +0,0 @@
-package db_test
-
-import (
-	"encoding/json"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
-	"github.com/stretchr/testify/require"
-	"io/ioutil"
-	"os"
-	"testing"
-)
-
-func TestFromCollectorSmartInfo(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-ata.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "passed", smartMdl.SmartStatus)
-	require.Equal(t, 18, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-
-	//check that temperature was correctly parsed
-	for _, attr := range smartMdl.AtaAttributes {
-		if attr.AttributeId == 194 {
-			require.Equal(t, int64(163210330144), attr.RawValue)
-			require.Equal(t, int64(32), attr.TransformedValue)
-		}
-	}
-}
-
-func TestFromCollectorSmartInfo_Fail(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-fail.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "failed", smartMdl.SmartStatus)
-	require.Equal(t, 0, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-}
-
-func TestFromCollectorSmartInfo_Fail2(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-fail2.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "failed", smartMdl.SmartStatus)
-	require.Equal(t, 17, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-}
-
-func TestFromCollectorSmartInfo_Nvme(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-nvme.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "passed", smartMdl.SmartStatus)
-	require.Equal(t, 0, len(smartMdl.AtaAttributes))
-	require.Equal(t, 16, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-
-	require.Equal(t, 111303174, smartMdl.NvmeAttributes[6].Value)
-	require.Equal(t, 83170961, smartMdl.NvmeAttributes[7].Value)
-}
-
-func TestFromCollectorSmartInfo_Scsi(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-scsi.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "passed", smartMdl.SmartStatus)
-	require.Equal(t, 0, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 13, len(smartMdl.ScsiAttributes))
-
-	require.Equal(t, 56, smartMdl.ScsiAttributes[0].Value)
-	require.Equal(t, 300357663, smartMdl.ScsiAttributes[4].Value) //total_errors_corrected
-}

webapp/backend/pkg/models/device.go

@@ -0,0 +1,169 @@
+package models
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"time"
+)
+
+type DeviceWrapper struct {
+	Success bool     `json:"success"`
+	Errors  []error  `json:"errors"`
+	Data    []Device `json:"data"`
+}
+
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN string `json:"wwn" gorm:"primary_key"`
+
+	DeviceName     string `json:"device_name"`
+	Manufacturer   string `json:"manufacturer"`
+	ModelName      string `json:"model_name"`
+	InterfaceType  string `json:"interface_type"`
+	InterfaceSpeed string `json:"interface_speed"`
+	SerialNumber   string `json:"serial_number"`
+	Firmware       string `json:"firmware"`
+	RotationSpeed  int    `json:"rotational_speed"`
+	Capacity       int64  `json:"capacity"`
+	FormFactor     string `json:"form_factor"`
+	SmartSupport   bool   `json:"smart_support"`
+	DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string `json:"label"`
+	HostId string `json:"host_id"`
+
+	// Data set by Scrutiny
+	DeviceStatus pkg.DeviceStatus `json:"device_status"`
+}
+
+func (dv *Device) IsAta() bool {
+	return dv.DeviceProtocol == pkg.DeviceProtocolAta
+}
+
+func (dv *Device) IsScsi() bool {
+	return dv.DeviceProtocol == pkg.DeviceProtocolScsi
+}
+
+func (dv *Device) IsNvme() bool {
+	return dv.DeviceProtocol == pkg.DeviceProtocolNvme
+}
+
+//
+////This method requires a device with an array of SmartResults.
+////It will remove all SmartResults other than the first (the latest one)
+////All removed SmartResults, will be processed, grouping SmartAtaAttribute by attribute_id
+//// and adding theme to an array called History.
+//func (dv *Device) SquashHistory() error {
+//	if len(dv.SmartResults) <= 1 {
+//		return nil //no ataHistory found. ignore
+//	}
+//
+//	latestSmartResultSlice := dv.SmartResults[0:1]
+//	historicalSmartResultSlice := dv.SmartResults[1:]
+//
+//	//re-assign the latest slice to the SmartResults field
+//	dv.SmartResults = latestSmartResultSlice
+//
+//	//process the historical slice for ATA data
+//	if len(dv.SmartResults[0].AtaAttributes) > 0 {
+//		ataHistory := map[int][]SmartAtaAttribute{}
+//		for _, smartResult := range historicalSmartResultSlice {
+//			for _, smartAttribute := range smartResult.AtaAttributes {
+//				if _, ok := ataHistory[smartAttribute.AttributeId]; !ok {
+//					ataHistory[smartAttribute.AttributeId] = []SmartAtaAttribute{}
+//				}
+//				ataHistory[smartAttribute.AttributeId] = append(ataHistory[smartAttribute.AttributeId], smartAttribute)
+//			}
+//		}
+//
+//		//now assign the historical slices to the AtaAttributes in the latest SmartResults
+//		for sandx, smartAttribute := range dv.SmartResults[0].AtaAttributes {
+//			if attributeHistory, ok := ataHistory[smartAttribute.AttributeId]; ok {
+//				dv.SmartResults[0].AtaAttributes[sandx].History = attributeHistory
+//			}
+//		}
+//	}
+//
+//	//process the historical slice for Nvme data
+//	if len(dv.SmartResults[0].NvmeAttributes) > 0 {
+//		nvmeHistory := map[string][]SmartNvmeAttribute{}
+//		for _, smartResult := range historicalSmartResultSlice {
+//			for _, smartAttribute := range smartResult.NvmeAttributes {
+//				if _, ok := nvmeHistory[smartAttribute.AttributeId]; !ok {
+//					nvmeHistory[smartAttribute.AttributeId] = []SmartNvmeAttribute{}
+//				}
+//				nvmeHistory[smartAttribute.AttributeId] = append(nvmeHistory[smartAttribute.AttributeId], smartAttribute)
+//			}
+//		}
+//
+//		//now assign the historical slices to the AtaAttributes in the latest SmartResults
+//		for sandx, smartAttribute := range dv.SmartResults[0].NvmeAttributes {
+//			if attributeHistory, ok := nvmeHistory[smartAttribute.AttributeId]; ok {
+//				dv.SmartResults[0].NvmeAttributes[sandx].History = attributeHistory
+//			}
+//		}
+//	}
+//	//process the historical slice for Scsi data
+//	if len(dv.SmartResults[0].ScsiAttributes) > 0 {
+//		scsiHistory := map[string][]SmartScsiAttribute{}
+//		for _, smartResult := range historicalSmartResultSlice {
+//			for _, smartAttribute := range smartResult.ScsiAttributes {
+//				if _, ok := scsiHistory[smartAttribute.AttributeId]; !ok {
+//					scsiHistory[smartAttribute.AttributeId] = []SmartScsiAttribute{}
+//				}
+//				scsiHistory[smartAttribute.AttributeId] = append(scsiHistory[smartAttribute.AttributeId], smartAttribute)
+//			}
+//		}
+//
+//		//now assign the historical slices to the AtaAttributes in the latest SmartResults
+//		for sandx, smartAttribute := range dv.SmartResults[0].ScsiAttributes {
+//			if attributeHistory, ok := scsiHistory[smartAttribute.AttributeId]; ok {
+//				dv.SmartResults[0].ScsiAttributes[sandx].History = attributeHistory
+//			}
+//		}
+//	}
+//	return nil
+//}
+//
+//func (dv *Device) ApplyMetadataRules() error {
+//
+//	//embed metadata in the latest smart attributes object
+//	if len(dv.SmartResults) > 0 {
+//		for ndx, attr := range dv.SmartResults[0].AtaAttributes {
+//			attr.PopulateAttributeStatus()
+//			dv.SmartResults[0].AtaAttributes[ndx] = attr
+//		}
+//
+//		for ndx, attr := range dv.SmartResults[0].NvmeAttributes {
+//			attr.PopulateAttributeStatus()
+//			dv.SmartResults[0].NvmeAttributes[ndx] = attr
+//
+//		}
+//
+//		for ndx, attr := range dv.SmartResults[0].ScsiAttributes {
+//			attr.PopulateAttributeStatus()
+//			dv.SmartResults[0].ScsiAttributes[ndx] = attr
+//
+//		}
+//	}
+//	return nil
+//}
+
+// This function is called every time the collector sends SMART data to the API.
+// It can be used to update device data that can change over time.
+func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
+	dv.Firmware = info.FirmwareVersion
+	dv.DeviceProtocol = info.Device.Protocol
+
+	if !info.SmartStatus.Passed {
+		dv.DeviceStatus = pkg.Set(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
+	}
+
+	return nil
+}

webapp/backend/pkg/models/device_summary.go

@@ -0,0 +1,19 @@
+package models
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"time"
+)
+
+type DeviceSummary struct {
+	Device Device `json:"device"`
+
+	SmartResults *SmartSummary                   `json:"smart,omitempty"`
+	TempHistory  []measurements.SmartTemperature `json:"temp_history,omitempty"`
+}
+type SmartSummary struct {
+	// Collector Summary Data
+	CollectorDate time.Time `json:"collector_date,omitempty"`
+	Temp          int64     `json:"temp,omitempty"`
+	PowerOnHours  int64     `json:"power_on_hours,omitempty"`
+}

webapp/backend/pkg/models/measurements/smart.go

@@ -0,0 +1,198 @@
+package measurements
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"log"
+	"strings"
+	"time"
+)
+
+type Smart struct {
+	Date           time.Time `json:"date"`
+	DeviceWWN      string    `json:"device_wwn"` //(tag)
+	DeviceProtocol string    `json:"device_protocol"`
+
+	//Metrics (fields)
+	Temp            int64 `json:"temp"`
+	PowerOnHours    int64 `json:"power_on_hours"`
+	PowerCycleCount int64 `json:"power_cycle_count"`
+
+	//Attributes (fields)
+	Attributes map[string]SmartAttribute `json:"attrs"`
+}
+
+func (sm *Smart) Flatten() (tags map[string]string, fields map[string]interface{}) {
+	tags = map[string]string{
+		"device_wwn":      sm.DeviceWWN,
+		"device_protocol": sm.DeviceProtocol,
+	}
+
+	fields = map[string]interface{}{
+		"temp":              sm.Temp,
+		"power_on_hours":    sm.PowerOnHours,
+		"power_cycle_count": sm.PowerCycleCount,
+	}
+
+	for _, attr := range sm.Attributes {
+		for attrKey, attrVal := range attr.Flatten() {
+			fields[attrKey] = attrVal
+		}
+	}
+
+	return tags, fields
+}
+
+func NewSmartFromInfluxDB(attrs map[string]interface{}) (*Smart, error) {
+	//go though the massive map returned from influxdb. If a key is associated with the Smart struct, assign it. If it starts with "attr.*" group it by attributeId, and pass to attribute inflate.
+
+	sm := Smart{
+		//required fields
+		Date:           attrs["_time"].(time.Time),
+		DeviceWWN:      attrs["device_wwn"].(string),
+		DeviceProtocol: attrs["device_protocol"].(string),
+
+		Attributes: map[string]SmartAttribute{},
+	}
+
+	log.Printf("Prefetched Smart: %v\n", sm)
+
+	//two steps (because we dont know the
+	for key, val := range attrs {
+		log.Printf("Found Attribute (%s = %v)\n", key, val)
+
+		switch key {
+		case "temp":
+			sm.Temp = val.(int64)
+		case "power_on_hours":
+			sm.PowerOnHours = val.(int64)
+		case "power_cycle_count":
+			sm.PowerCycleCount = val.(int64)
+		default:
+			// this key is unknown.
+			if !strings.HasPrefix(key, "attr.") {
+				continue
+			}
+			//this is a attribute, lets group it with its related "siblings", populating a SmartAttribute object
+			keyParts := strings.Split(key, ".")
+			attributeId := keyParts[1]
+			if _, ok := sm.Attributes[attributeId]; !ok {
+				// init the attribute group
+				if sm.DeviceProtocol == pkg.DeviceProtocolAta {
+					sm.Attributes[attributeId] = &SmartAtaAttribute{}
+				} else if sm.DeviceProtocol == pkg.DeviceProtocolNvme {
+					sm.Attributes[attributeId] = &SmartNvmeAttribute{}
+				} else if sm.DeviceProtocol == pkg.DeviceProtocolScsi {
+					sm.Attributes[attributeId] = &SmartScsiAttribute{}
+				} else {
+					return nil, fmt.Errorf("Unknown Device Protocol: %s", sm.DeviceProtocol)
+				}
+			}
+
+			sm.Attributes[attributeId].Inflate(key, val)
+		}
+
+	}
+
+	log.Printf("########NUMBER OF ATTRIBUTES: %v", len(sm.Attributes))
+	log.Printf("########SMART: %v", sm)
+
+	//panic("ERROR HERE.")
+
+	//log.Printf("Sm.Attributes: %v", sm.Attributes)
+	//log.Printf("sm.Attributes[attributeId]: %v", sm.Attributes[attributeId])
+
+	return &sm, nil
+}
+
+//Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries)
+func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error {
+	sm.DeviceWWN = wwn
+	sm.Date = time.Unix(info.LocalTime.TimeT, 0)
+
+	//smart metrics
+	sm.Temp = info.Temperature.Current
+	sm.PowerCycleCount = info.PowerCycleCount
+	sm.PowerOnHours = info.PowerOnTime.Hours
+
+	sm.DeviceProtocol = info.Device.Protocol
+	// process ATA/NVME/SCSI protocol data
+	sm.Attributes = map[string]SmartAttribute{}
+	if sm.DeviceProtocol == pkg.DeviceProtocolAta {
+		sm.ProcessAtaSmartInfo(info)
+	} else if sm.DeviceProtocol == pkg.DeviceProtocolNvme {
+		sm.ProcessNvmeSmartInfo(info)
+	} else if sm.DeviceProtocol == pkg.DeviceProtocolScsi {
+		sm.ProcessScsiSmartInfo(info)
+	}
+
+	return nil
+}
+
+//generate SmartAtaAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) {
+	for _, collectorAttr := range info.AtaSmartAttributes.Table {
+		attrModel := SmartAtaAttribute{
+			AttributeId: collectorAttr.ID,
+			Name:        collectorAttr.Name,
+			Value:       collectorAttr.Value,
+			Worst:       collectorAttr.Worst,
+			Threshold:   collectorAttr.Thresh,
+			RawValue:    collectorAttr.Raw.Value,
+			RawString:   collectorAttr.Raw.String,
+			WhenFailed:  collectorAttr.WhenFailed,
+		}
+
+		//now that we've parsed the data from the smartctl response, lets match it against our metadata rules and add additional Scrutiny specific data.
+		if smartMetadata, ok := metadata.AtaMetadata[collectorAttr.ID]; ok {
+			attrModel.Name = smartMetadata.DisplayName
+			if smartMetadata.Transform != nil {
+				attrModel.TransformedValue = smartMetadata.Transform(attrModel.Value, attrModel.RawValue, attrModel.RawString)
+			}
+		}
+		sm.Attributes[string(collectorAttr.ID)] = &attrModel
+	}
+}
+
+//generate SmartNvmeAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) {
+	sm.Attributes = map[string]SmartAttribute{
+		"critical_warning":     &SmartNvmeAttribute{AttributeId: "critical_warning", Name: "Critical Warning", Value: info.NvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0},
+		"temperature":          &SmartNvmeAttribute{AttributeId: "temperature", Name: "Temperature", Value: info.NvmeSmartHealthInformationLog.Temperature, Threshold: -1},
+		"available_spare":      &SmartNvmeAttribute{AttributeId: "available_spare", Name: "Available Spare", Value: info.NvmeSmartHealthInformationLog.AvailableSpare, Threshold: info.NvmeSmartHealthInformationLog.AvailableSpareThreshold},
+		"percentage_used":      &SmartNvmeAttribute{AttributeId: "percentage_used", Name: "Percentage Used", Value: info.NvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100},
+		"data_units_read":      &SmartNvmeAttribute{AttributeId: "data_units_read", Name: "Data Units Read", Value: info.NvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1},
+		"data_units_written":   &SmartNvmeAttribute{AttributeId: "data_units_written", Name: "Data Units Written", Value: info.NvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1},
+		"host_reads":           &SmartNvmeAttribute{AttributeId: "host_reads", Name: "Host Reads", Value: info.NvmeSmartHealthInformationLog.HostReads, Threshold: -1},
+		"host_writes":          &SmartNvmeAttribute{AttributeId: "host_writes", Name: "Host Writes", Value: info.NvmeSmartHealthInformationLog.HostWrites, Threshold: -1},
+		"controller_busy_time": &SmartNvmeAttribute{AttributeId: "controller_busy_time", Name: "Controller Busy Time", Value: info.NvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1},
+		"power_cycles":         &SmartNvmeAttribute{AttributeId: "power_cycles", Name: "Power Cycles", Value: info.NvmeSmartHealthInformationLog.PowerCycles, Threshold: -1},
+		"power_on_hours":       &SmartNvmeAttribute{AttributeId: "power_on_hours", Name: "Power on Hours", Value: info.NvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1},
+		"unsafe_shutdowns":     &SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Name: "Unsafe Shutdowns", Value: info.NvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1},
+		"media_errors":         &SmartNvmeAttribute{AttributeId: "media_errors", Name: "Media Errors", Value: info.NvmeSmartHealthInformationLog.MediaErrors, Threshold: 0},
+		"num_err_log_entries":  &SmartNvmeAttribute{AttributeId: "num_err_log_entries", Name: "Numb Err Log Entries", Value: info.NvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0},
+		"warning_temp_time":    &SmartNvmeAttribute{AttributeId: "warning_temp_time", Name: "Warning Temp Time", Value: info.NvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1},
+		"critical_comp_time":   &SmartNvmeAttribute{AttributeId: "critical_comp_time", Name: "Critical CompTime", Value: info.NvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1},
+	}
+}
+
+//generate SmartScsiAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessScsiSmartInfo(info collector.SmartInfo) {
+	sm.Attributes = map[string]SmartAttribute{
+		"scsi_grown_defect_list":                     &SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Name: "Grown Defect List", Value: info.ScsiGrownDefectList, Threshold: 0},
+		"read_errors_corrected_by_eccfast":           &SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Name: "Read Errors Corrected by ECC Fast", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1},
+		"read_errors_corrected_by_eccdelayed":        &SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Name: "Read Errors Corrected by ECC Delayed", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1},
+		"read_errors_corrected_by_rereads_rewrites":  &SmartScsiAttribute{AttributeId: "read_errors_corrected_by_rereads_rewrites", Name: "Read Errors Corrected by ReReads/ReWrites", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0},
+		"read_total_errors_corrected":                &SmartScsiAttribute{AttributeId: "read_total_errors_corrected", Name: "Read Total Errors Corrected", Value: info.ScsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1},
+		"read_correction_algorithm_invocations":      &SmartScsiAttribute{AttributeId: "read_correction_algorithm_invocations", Name: "Read Correction Algorithm Invocations", Value: info.ScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1},
+		"read_total_uncorrected_errors":              &SmartScsiAttribute{AttributeId: "read_total_uncorrected_errors", Name: "Read Total Uncorrected Errors", Value: info.ScsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0},
+		"write_errors_corrected_by_eccfast":          &SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccfast", Name: "Write Errors Corrected by ECC Fast", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1},
+		"write_errors_corrected_by_eccdelayed":       &SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccdelayed", Name: "Write Errors Corrected by ECC Delayed", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1},
+		"write_errors_corrected_by_rereads_rewrites": &SmartScsiAttribute{AttributeId: "write_errors_corrected_by_rereads_rewrites", Name: "Write Errors Corrected by ReReads/ReWrites", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0},
+		"write_total_errors_corrected":               &SmartScsiAttribute{AttributeId: "write_total_errors_corrected", Name: "Write Total Errors Corrected", Value: info.ScsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1},
+		"write_correction_algorithm_invocations":     &SmartScsiAttribute{AttributeId: "write_correction_algorithm_invocations", Name: "Write Correction Algorithm Invocations", Value: info.ScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1},
+		"write_total_uncorrected_errors":             &SmartScsiAttribute{AttributeId: "write_total_uncorrected_errors", Name: "Write Total Uncorrected Errors", Value: info.ScsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0},
+	}
+}

webapp/backend/pkg/models/measurements/smart_ata_attribute.go

@@ -0,0 +1,151 @@
+package measurements
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+const SmartAttributeStatusPassed = "passed"
+const SmartAttributeStatusFailed = "failed"
+const SmartAttributeStatusWarning = "warn"
+
+type SmartAtaAttribute struct {
+	AttributeId int    `json:"attribute_id"`
+	Name        string `json:"name"`
+	Value       int64  `json:"value"`
+	Threshold   int64  `json:"thresh"`
+	Worst       int64  `json:"worst"`
+	RawValue    int64  `json:"raw_value"`
+	RawString   string `json:"raw_string"`
+	WhenFailed  string `json:"when_failed"`
+
+	//Generated data
+	TransformedValue int64   `json:"transformed_value"`
+	Status           string  `json:"status,omitempty"`
+	StatusReason     string  `json:"status_reason,omitempty"`
+	FailureRate      float64 `json:"failure_rate,omitempty"`
+}
+
+func (sa *SmartAtaAttribute) Flatten() map[string]interface{} {
+
+	idString := strconv.Itoa(sa.AttributeId)
+
+	return map[string]interface{}{
+		fmt.Sprintf("attr.%s.attribute_id", idString): idString,
+		fmt.Sprintf("attr.%s.name", idString):         sa.Name,
+		fmt.Sprintf("attr.%s.value", idString):        sa.Value,
+		fmt.Sprintf("attr.%s.worst", idString):        sa.Worst,
+		fmt.Sprintf("attr.%s.thresh", idString):       sa.Threshold,
+		fmt.Sprintf("attr.%s.raw_value", idString):    sa.RawValue,
+		fmt.Sprintf("attr.%s.raw_string", idString):   sa.RawString,
+		fmt.Sprintf("attr.%s.when_failed", idString):  sa.WhenFailed,
+	}
+}
+func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+	keyParts := strings.Split(key, ".")
+
+	switch keyParts[2] {
+	case "attribute_id":
+		attrId, err := strconv.Atoi(val.(string))
+		if err == nil {
+			sa.AttributeId = attrId
+		}
+	case "name":
+		sa.Name = val.(string)
+	case "value":
+		sa.Value = val.(int64)
+	case "worst":
+		sa.Worst = val.(int64)
+	case "thresh":
+		sa.Threshold = val.(int64)
+	case "raw_value":
+		sa.RawValue = val.(int64)
+	case "raw_string":
+		sa.RawString = val.(string)
+	case "when_failed":
+		sa.WhenFailed = val.(string)
+	}
+}
+
+//
+////populate attribute status, using SMART Thresholds & Observed Metadata
+//func (sa *SmartAtaAttribute) PopulateAttributeStatus() {
+//	if strings.ToUpper(sa.WhenFailed) == SmartWhenFailedFailingNow {
+//		//this attribute has previously failed
+//		sa.Status = SmartAttributeStatusFailed
+//		sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
+//
+//	} else if strings.ToUpper(sa.WhenFailed) == SmartWhenFailedInThePast {
+//		sa.Status = SmartAttributeStatusWarning
+//		sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
+//	}
+//
+//	if smartMetadata, ok := metadata.AtaMetadata[sa.AttributeId]; ok {
+//		sa.MetadataObservedThresholdStatus(smartMetadata)
+//	}
+//
+//	//check if status is blank, set to "passed"
+//	if len(sa.Status) == 0 {
+//		sa.Status = SmartAttributeStatusPassed
+//	}
+//}
+//
+//// compare the attribute (raw, normalized, transformed) value to observed thresholds, and update status if necessary
+//func (sa *SmartAtaAttribute) MetadataObservedThresholdStatus(smartMetadata metadata.AtaAttributeMetadata) {
+//	//TODO: multiple rules
+//	// try to predict the failure rates for observed thresholds that have 0 failure rate and error bars.
+//	// - if the attribute is critical
+//	//		- the failure rate is over 10 - set to failed
+//	//		- the attribute does not match any threshold, set to warn
+//	// - if the attribute is not critical
+//	//		- if failure rate is above 20 - set to failed
+//	// 		- if failure rate is above 10 but below 20 - set to warn
+//
+//	//update the smart attribute status based on Observed thresholds.
+//	var value int64
+//	if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeNormalized {
+//		value = int64(sa.Value)
+//	} else if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeTransformed {
+//		value = sa.TransformedValue
+//	} else {
+//		value = sa.RawValue
+//	}
+//
+//	for _, obsThresh := range smartMetadata.ObservedThresholds {
+//
+//		//check if "value" is in this bucket
+//		if ((obsThresh.Low == obsThresh.High) && value == obsThresh.Low) ||
+//			(obsThresh.Low < value && value <= obsThresh.High) {
+//			sa.FailureRate = obsThresh.AnnualFailureRate
+//
+//			if smartMetadata.Critical {
+//				if obsThresh.AnnualFailureRate >= 0.10 {
+//					sa.Status = SmartAttributeStatusFailed
+//					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
+//				}
+//			} else {
+//				if obsThresh.AnnualFailureRate >= 0.20 {
+//					sa.Status = SmartAttributeStatusFailed
+//					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
+//				} else if obsThresh.AnnualFailureRate >= 0.10 {
+//					sa.Status = SmartAttributeStatusWarning
+//					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
+//				}
+//			}
+//
+//			//we've found the correct bucket, we can drop out of this loop
+//			return
+//		}
+//	}
+//	// no bucket found
+//	if smartMetadata.Critical {
+//		sa.Status = SmartAttributeStatusWarning
+//		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
+//	}
+//
+//	return
+//}

webapp/backend/pkg/models/measurements/smart_attribute.go

@@ -0,0 +1,6 @@
+package measurements
+
+type SmartAttribute interface {
+	Flatten() (fields map[string]interface{})
+	Inflate(key string, val interface{})
+}

webapp/backend/pkg/models/measurements/smart_nvme_attribute.go

@@ -0,0 +1,68 @@
+package measurements
+
+import (
+	"fmt"
+	"strings"
+)
+
+type SmartNvmeAttribute struct {
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int64  `json:"value"`
+	Threshold   int64  `json:"thresh"`
+
+	TransformedValue int64   `json:"transformed_value"`
+	Status           string  `json:"status,omitempty"`
+	StatusReason     string  `json:"status_reason,omitempty"`
+	FailureRate      float64 `json:"failure_rate,omitempty"`
+}
+
+func (sa *SmartNvmeAttribute) Flatten() map[string]interface{} {
+	return map[string]interface{}{
+		fmt.Sprintf("attr.%s.attribute_id", sa.AttributeId): sa.AttributeId,
+		fmt.Sprintf("attr.%s.name", sa.AttributeId):         sa.Name,
+		fmt.Sprintf("attr.%s.value", sa.AttributeId):        sa.Value,
+		fmt.Sprintf("attr.%s.thresh", sa.AttributeId):       sa.Threshold,
+	}
+}
+func (sa *SmartNvmeAttribute) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+
+	keyParts := strings.Split(key, ".")
+
+	switch keyParts[2] {
+	case "attribute_id":
+		sa.AttributeId = val.(string)
+	case "name":
+		sa.Name = val.(string)
+	case "value":
+		sa.Value = val.(int64)
+	case "thresh":
+		sa.Threshold = val.(int64)
+	}
+}
+
+//
+////populate attribute status, using SMART Thresholds & Observed Metadata
+//func (sa *SmartNvmeAttribute) PopulateAttributeStatus() {
+//
+//	//-1 is a special number meaning no threshold.
+//	if sa.Threshold != -1 {
+//		if smartMetadata, ok := metadata.NmveMetadata[sa.AttributeId]; ok {
+//			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
+//			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
+//				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
+//				sa.Status = SmartAttributeStatusFailed
+//				sa.StatusReason = "Attribute is failing recommended SMART threshold"
+//			}
+//		}
+//	}
+//	//TODO: eventually figure out the critical_warning bits and determine correct error messages here.
+//
+//	//check if status is blank, set to "passed"
+//	if len(sa.Status) == 0 {
+//		sa.Status = SmartAttributeStatusPassed
+//	}
+//}

webapp/backend/pkg/models/measurements/smart_scsci_attribute.go

@@ -0,0 +1,67 @@
+package measurements
+
+import (
+	"fmt"
+	"strings"
+)
+
+type SmartScsiAttribute struct {
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int64  `json:"value"`
+	Threshold   int64  `json:"thresh"`
+
+	TransformedValue int64   `json:"transformed_value"`
+	Status           string  `json:"status,omitempty"`
+	StatusReason     string  `json:"status_reason,omitempty"`
+	FailureRate      float64 `json:"failure_rate,omitempty"`
+}
+
+func (sa *SmartScsiAttribute) Flatten() map[string]interface{} {
+	return map[string]interface{}{
+		fmt.Sprintf("attr.%s.attribute_id", sa.AttributeId): sa.AttributeId,
+		fmt.Sprintf("attr.%s.name", sa.AttributeId):         sa.Name,
+		fmt.Sprintf("attr.%s.value", sa.AttributeId):        sa.Value,
+		fmt.Sprintf("attr.%s.thresh", sa.AttributeId):       sa.Threshold,
+	}
+}
+func (sa *SmartScsiAttribute) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+
+	keyParts := strings.Split(key, ".")
+
+	switch keyParts[2] {
+	case "attribute_id":
+		sa.AttributeId = val.(string)
+	case "name":
+		sa.Name = val.(string)
+	case "value":
+		sa.Value = val.(int64)
+	case "thresh":
+		sa.Threshold = val.(int64)
+	}
+}
+
+//
+////populate attribute status, using SMART Thresholds & Observed Metadata
+//func (sa *SmartScsiAttribute) PopulateAttributeStatus() {
+//
+//	//-1 is a special number meaning no threshold.
+//	if sa.Threshold != -1 {
+//		if smartMetadata, ok := metadata.NmveMetadata[sa.AttributeId]; ok {
+//			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
+//			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
+//				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
+//				sa.Status = SmartAttributeStatusFailed
+//				sa.StatusReason = "Attribute is failing recommended SMART threshold"
+//			}
+//		}
+//	}
+//
+//	//check if status is blank, set to "passed"
+//	if len(sa.Status) == 0 {
+//		sa.Status = SmartAttributeStatusPassed
+//	}
+//}

webapp/backend/pkg/models/measurements/smart_temperature.go

@@ -0,0 +1,29 @@
+package measurements
+
+import (
+	"time"
+)
+
+type SmartTemperature struct {
+	Date time.Time `json:"date"`
+	Temp int64     `json:"temp"`
+}
+
+func (st *SmartTemperature) Flatten() (tags map[string]string, fields map[string]interface{}) {
+	fields = map[string]interface{}{
+		"temp": st.Temp,
+	}
+	tags = map[string]string{}
+
+	return tags, fields
+}
+
+func (st *SmartTemperature) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+
+	if key == "temp" {
+		st.Temp = val.(int64)
+	}
+}

webapp/backend/pkg/models/measurements/smart_test.go

@@ -0,0 +1,141 @@
+package measurements_test
+
+//func TestFromCollectorSmartInfo(t *testing.T) {
+//	//setup
+//	smartDataFile, err := os.Open("../testdata/smart-ata.json")
+//	require.NoError(t, err)
+//	defer smartDataFile.Close()
+//
+//	var smartJson collector.SmartInfo
+//
+//	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+//	require.NoError(t, err)
+//	err = json.Unmarshal(smartDataBytes, &smartJson)
+//	require.NoError(t, err)
+//
+//	//test
+//	smartMdl := db.Smart{}
+//	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+//
+//	//assert
+//	require.NoError(t, err)
+//	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+//	require.Equal(t, "passed", smartMdl.SmartStatus)
+//	require.Equal(t, 18, len(smartMdl.Attributes))
+//
+//	//check that temperature was correctly parsed
+//	for _, attr := range smartMdl.Attributes {
+//		if attr.AttributeId == 194 {
+//			require.Equal(t, int64(163210330144), attr.RawValue)
+//			require.Equal(t, int64(32), attr.TransformedValue)
+//		}
+//	}
+//}
+//
+//func TestFromCollectorSmartInfo_Fail(t *testing.T) {
+//	//setup
+//	smartDataFile, err := os.Open("../testdata/smart-fail.json")
+//	require.NoError(t, err)
+//	defer smartDataFile.Close()
+//
+//	var smartJson collector.SmartInfo
+//
+//	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+//	require.NoError(t, err)
+//	err = json.Unmarshal(smartDataBytes, &smartJson)
+//	require.NoError(t, err)
+//
+//	//test
+//	smartMdl := db.Smart{}
+//	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+//
+//	//assert
+//	require.NoError(t, err)
+//	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+//	require.Equal(t, "failed", smartMdl.SmartStatus)
+//	require.Equal(t, 0, len(smartMdl.AtaAttributes))
+//	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
+//	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
+//}
+//
+//func TestFromCollectorSmartInfo_Fail2(t *testing.T) {
+//	//setup
+//	smartDataFile, err := os.Open("../testdata/smart-fail2.json")
+//	require.NoError(t, err)
+//	defer smartDataFile.Close()
+//
+//	var smartJson collector.SmartInfo
+//
+//	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+//	require.NoError(t, err)
+//	err = json.Unmarshal(smartDataBytes, &smartJson)
+//	require.NoError(t, err)
+//
+//	//test
+//	smartMdl := db.Smart{}
+//	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+//
+//	//assert
+//	require.NoError(t, err)
+//	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+//	require.Equal(t, "failed", smartMdl.SmartStatus)
+//	require.Equal(t, 17, len(smartMdl.Attributes))
+//}
+//
+//func TestFromCollectorSmartInfo_Nvme(t *testing.T) {
+//	//setup
+//	smartDataFile, err := os.Open("../testdata/smart-nvme.json")
+//	require.NoError(t, err)
+//	defer smartDataFile.Close()
+//
+//	var smartJson collector.SmartInfo
+//
+//	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+//	require.NoError(t, err)
+//	err = json.Unmarshal(smartDataBytes, &smartJson)
+//	require.NoError(t, err)
+//
+//	//test
+//	smartMdl := db.Smart{}
+//	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+//
+//	//assert
+//	require.NoError(t, err)
+//	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+//	require.Equal(t, "passed", smartMdl.SmartStatus)
+//	require.Equal(t, 0, len(smartMdl.AtaAttributes))
+//	require.Equal(t, 16, len(smartMdl.NvmeAttributes))
+//	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
+//
+//	require.Equal(t, 111303174, smartMdl.NvmeAttributes[6].Value)
+//	require.Equal(t, 83170961, smartMdl.NvmeAttributes[7].Value)
+//}
+//
+//func TestFromCollectorSmartInfo_Scsi(t *testing.T) {
+//	//setup
+//	smartDataFile, err := os.Open("../testdata/smart-scsi.json")
+//	require.NoError(t, err)
+//	defer smartDataFile.Close()
+//
+//	var smartJson collector.SmartInfo
+//
+//	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+//	require.NoError(t, err)
+//	err = json.Unmarshal(smartDataBytes, &smartJson)
+//	require.NoError(t, err)
+//
+//	//test
+//	smartMdl := db.Smart{}
+//	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+//
+//	//assert
+//	require.NoError(t, err)
+//	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+//	require.Equal(t, "passed", smartMdl.SmartStatus)
+//	require.Equal(t, 0, len(smartMdl.AtaAttributes))
+//	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
+//	require.Equal(t, 13, len(smartMdl.ScsiAttributes))
+//
+//	require.Equal(t, 56, smartMdl.ScsiAttributes[0].Value)
+//	require.Equal(t, 300357663, smartMdl.ScsiAttributes[4].Value) //total_errors_corrected
+//}

webapp/backend/pkg/models/setting.go

@@ -0,0 +1,5 @@
+package models
+
+// Temperature Format
+// Date Format
+// Device History window

webapp/backend/pkg/models/testdata/smart-ata-date.json

@@ -0,0 +1,846 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-4.19.128-flatcar",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "-a",
+      "/dev/sdb"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/sdb",
+    "info_name": "/dev/sdb [SAT]",
+    "type": "sat",
+    "protocol": "ATA"
+  },
+  "model_name": "WDC WD140EDFZ-11A0VA0",
+  "serial_number": "9RK1XXXX",
+  "wwn": {
+    "naa": 5,
+    "oui": 3274,
+    "id": 10283057623
+  },
+  "firmware_version": "81.00A81",
+  "user_capacity": {
+    "blocks": 27344764928,
+    "bytes": 14000519643136
+  },
+  "logical_block_size": 512,
+  "physical_block_size": 4096,
+  "rotation_rate": 5400,
+  "form_factor": {
+    "ata_value": 2,
+    "name": "3.5 inches"
+  },
+  "in_smartctl_database": false,
+  "ata_version": {
+    "string": "ACS-2, ATA8-ACS T13/1699-D revision 4",
+    "major_value": 1020,
+    "minor_value": 41
+  },
+  "sata_version": {
+    "string": "SATA 3.2",
+    "value": 255
+  },
+  "interface_speed": {
+    "max": {
+      "sata_value": 14,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    },
+    "current": {
+      "sata_value": 3,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    }
+  },
+  "local_time": {
+    "time_t": 1611419146,
+    "asctime": "Sun Jun 30 00:03:30 2021 UTC"
+  },
+  "smart_status": {
+    "passed": true
+  },
+  "ata_smart_data": {
+    "offline_data_collection": {
+      "status": {
+        "value": 130,
+        "string": "was completed without error",
+        "passed": true
+      },
+      "completion_seconds": 101
+    },
+    "self_test": {
+      "status": {
+        "value": 241,
+        "string": "in progress, 10% remaining",
+        "remaining_percent": 10
+      },
+      "polling_minutes": {
+        "short": 2,
+        "extended": 1479
+      }
+    },
+    "capabilities": {
+      "values": [
+        91,
+        3
+      ],
+      "exec_offline_immediate_supported": true,
+      "offline_is_aborted_upon_new_cmd": false,
+      "offline_surface_scan_supported": true,
+      "self_tests_supported": true,
+      "conveyance_self_test_supported": false,
+      "selective_self_test_supported": true,
+      "attribute_autosave_enabled": true,
+      "error_logging_supported": true,
+      "gp_logging_supported": true
+    }
+  },
+  "ata_sct_capabilities": {
+    "value": 61,
+    "error_recovery_control_supported": true,
+    "feature_control_supported": true,
+    "data_table_supported": true
+  },
+  "ata_smart_attributes": {
+    "revision": 16,
+    "table": [
+      {
+        "id": 1,
+        "name": "Raw_Read_Error_Rate",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 11,
+          "string": "PO-R-- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 2,
+        "name": "Throughput_Performance",
+        "value": 135,
+        "worst": 135,
+        "thresh": 54,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 108,
+          "string": "108"
+        }
+      },
+      {
+        "id": 3,
+        "name": "Spin_Up_Time",
+        "value": 81,
+        "worst": 81,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 7,
+          "string": "POS--- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 30089675132,
+          "string": "380 (Average 380)"
+        }
+      },
+      {
+        "id": 4,
+        "name": "Start_Stop_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 9,
+          "string": "9"
+        }
+      },
+      {
+        "id": 5,
+        "name": "Reallocated_Sector_Ct",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 51,
+          "string": "PO--CK ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 7,
+        "name": "Seek_Error_Rate",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 8,
+        "name": "Seek_Time_Performance",
+        "value": 133,
+        "worst": 133,
+        "thresh": 20,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 18,
+          "string": "18"
+        }
+      },
+      {
+        "id": 9,
+        "name": "Power_On_Hours",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 1730,
+          "string": "1730"
+        }
+      },
+      {
+        "id": 10,
+        "name": "Spin_Retry_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 12,
+        "name": "Power_Cycle_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 9,
+          "string": "9"
+        }
+      },
+      {
+        "id": 22,
+        "name": "Unknown_Attribute",
+        "value": 100,
+        "worst": 100,
+        "thresh": 25,
+        "when_failed": "",
+        "flags": {
+          "value": 35,
+          "string": "PO---K ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 100,
+          "string": "100"
+        }
+      },
+      {
+        "id": 192,
+        "name": "Power-Off_Retract_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 329,
+          "string": "329"
+        }
+      },
+      {
+        "id": 193,
+        "name": "Load_Cycle_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 329,
+          "string": "329"
+        }
+      },
+      {
+        "id": 194,
+        "name": "Temperature_Celsius",
+        "value": 51,
+        "worst": 51,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 2,
+          "string": "-O---- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 163210330144,
+          "string": "32 (Min/Max 24/38)"
+        }
+      },
+      {
+        "id": 196,
+        "name": "Reallocated_Event_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 197,
+        "name": "Current_Pending_Sector",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 34,
+          "string": "-O---K ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 198,
+        "name": "Offline_Uncorrectable",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 8,
+          "string": "---R-- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 199,
+        "name": "UDMA_CRC_Error_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      }
+    ]
+  },
+  "power_on_time": {
+    "hours": 1730
+  },
+  "power_cycle_count": 9,
+  "temperature": {
+    "current": 32
+  },
+  "ata_smart_error_log": {
+    "summary": {
+      "revision": 1,
+      "count": 0
+    }
+  },
+  "ata_smart_self_test_log": {
+    "standard": {
+      "revision": 1,
+      "table": [
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1708
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1684
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1661
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1636
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1624
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1541
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1517
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1493
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1469
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1445
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1439
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1373
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1349
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1325
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1301
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1277
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1253
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1252
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1205
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1181
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1157
+        }
+      ],
+      "count": 21,
+      "error_count_total": 0,
+      "error_count_outdated": 0
+    }
+  },
+  "ata_smart_selective_self_test_log": {
+    "revision": 1,
+    "table": [
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      }
+    ],
+    "flags": {
+      "value": 0,
+      "remainder_scan_enabled": false
+    },
+    "power_up_scan_resume_minutes": 0
+  }
+}

webapp/backend/pkg/web/handler/get_device_details.go

@@ -1,44 +1,25 @@
 package handler
 
 import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"gorm.io/gorm"
 	"net/http"
 )
 
 func GetDeviceDetails(c *gin.Context) {
-	db := c.MustGet("DB").(*gorm.DB)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
-	device := dbModels.Device{}
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
-
-	if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
-		return db.Order("smarts.created_at DESC").Limit(40)
-	}).
-		Preload("SmartResults.AtaAttributes").
-		Preload("SmartResults.NvmeAttributes").
-		Preload("SmartResults.ScsiAttributes").
-		Where("wwn = ?", c.Param("wwn")).
-		First(&device).Error; err != nil {
 
+	device, err := deviceRepo.GetDeviceDetails(c, c.Param("wwn"))
+	if err != nil {
 		logger.Errorln("An error occurred while retrieving device details", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
 
-	if err := device.SquashHistory(); err != nil {
+	smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), "", nil)
-		logger.Errorln("An error occurred while squashing device history", err)
-		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
-		return
-	}
-
-	if err := device.ApplyMetadataRules(); err != nil {
-		logger.Errorln("An error occurred while applying scrutiny thresholds & rules", err)
-		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
-		return
-	}
 
 	var deviceMetadata interface{}
 	if device.IsAta() {
@@ -49,5 +30,5 @@ func GetDeviceDetails(c *gin.Context) {
 		deviceMetadata = metadata.ScsiMetadata
 	}
 
-	c.JSON(http.StatusOK, gin.H{"success": true, "data": device, "metadata": deviceMetadata})
+	c.JSON(http.StatusOK, gin.H{"success": true, "data": map[string]interface{}{"device": device, "smart_results": smartResults}, "metadata": deviceMetadata})
 }

webapp/backend/pkg/web/handler/get_devices_summary.go

@@ -1,31 +1,28 @@
 package handler
 
 import (
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"gorm.io/gorm"
 	"net/http"
 )
 
 func GetDevicesSummary(c *gin.Context) {
-	db := c.MustGet("DB").(*gorm.DB)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
-	devices := []dbModels.Device{}
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 
-	//We need the last x (for now all) Smart objects for each Device, so that we can graph Temperature
+	summary, err := deviceRepo.GetSummary(c)
-	//We also need the last
+	if err != nil {
-	if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
+		logger.Errorln("An error occurred while retrieving device summary", err)
-		return db.Order("smarts.created_at DESC") //OLD: .Limit(devicesCount)
-	}).
-		Find(&devices).Error; err != nil {
-		logger.Errorln("Could not get device summary from DB", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
 
 	c.JSON(http.StatusOK, gin.H{
 		"success": true,
-		"data":    devices,
+		"data": map[string]interface{}{
+			"summary": summary,
+			//"temperature": tem
+		},
 	})
 }

webapp/backend/pkg/web/handler/register_devices.go

@@ -1,22 +1,20 @@
 package handler
 
 import (
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/gin-gonic/gin"
-	"gorm.io/gorm"
-	"gorm.io/gorm/clause"
-
 	"github.com/sirupsen/logrus"
 	"net/http"
 )
 
 // register devices that are detected by various collectors.
-// This function is run everytime a collector is about to start a run. It can be used to update device data.
+// This function is run everytime a collector is about to start a run. It can be used to update device metadata.
 func RegisterDevices(c *gin.Context) {
-	db := c.MustGet("DB").(*gorm.DB)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 
-	var collectorDeviceWrapper dbModels.DeviceWrapper
+	var collectorDeviceWrapper models.DeviceWrapper
 	err := c.BindJSON(&collectorDeviceWrapper)
 	if err != nil {
 		logger.Errorln("Cannot parse detected devices", err)
@@ -28,11 +26,7 @@ func RegisterDevices(c *gin.Context) {
 	for _, dev := range collectorDeviceWrapper.Data {
 		//insert devices into DB (and update specified columns if device is already registered)
 		// update device fields that may change: (DeviceType, HostID)
-		if err := db.Clauses(clause.OnConflict{
+		if err := deviceRepo.RegisterDevice(c, dev); err != nil {
-			Columns:   []clause.Column{{Name: "wwn"}},
-			DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}),
-		}).Create(&dev).Error; err != nil {
-
 			errs = append(errs, err)
 		}
 	}
@@ -44,7 +38,7 @@ func RegisterDevices(c *gin.Context) {
 		})
 		return
 	} else {
-		c.JSON(http.StatusOK, dbModels.DeviceWrapper{
+		c.JSON(http.StatusOK, models.DeviceWrapper{
 			Success: true,
 			Data:    collectorDeviceWrapper.Data,
 		})

webapp/backend/pkg/web/handler/send_test_notification.go

@@ -1,8 +1,9 @@
 package handler
 
 import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
@@ -20,7 +21,7 @@ func SendTestNotification(c *gin.Context) {
 		Payload: notify.Payload{
 			FailureType:  "EmailTest",
 			DeviceSerial: "FAKEWDDJ324KSO",
-			DeviceType:   dbModels.DeviceProtocolAta,
+			DeviceType:   pkg.DeviceProtocolAta,
 			DeviceName:   "/dev/sda",
 			Test:         true,
 		},
@@ -33,7 +34,7 @@ func SendTestNotification(c *gin.Context) {
 			"errors":  []string{err.Error()},
 		})
 	} else {
-		c.JSON(http.StatusOK, dbModels.DeviceWrapper{
+		c.JSON(http.StatusOK, models.DeviceWrapper{
 			Success: true,
 		})
 	}

webapp/backend/pkg/web/handler/upload_device_metrics.go

@@ -1,20 +1,24 @@
 package handler
 
 import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"gorm.io/gorm"
 	"net/http"
 )
 
 func UploadDeviceMetrics(c *gin.Context) {
-	db := c.MustGet("DB").(*gorm.DB)
+	//db := c.MustGet("DB").(*gorm.DB)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 	appConfig := c.MustGet("CONFIG").(config.Interface)
+	//influxWriteDb := c.MustGet("INFLUXDB_WRITE").(*api.WriteAPIBlocking)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	//appConfig := c.MustGet("CONFIG").(config.Interface)
 
 	var collectorSmartData collector.SmartInfo
 	err := c.BindJSON(&collectorSmartData)
@@ -25,39 +29,39 @@ func UploadDeviceMetrics(c *gin.Context) {
 	}
 
 	//update the device information if necessary
-	var device dbModels.Device
+	updatedDevice, err := deviceRepo.UpdateDevice(c, c.Param("wwn"), collectorSmartData)
-	db.Where("wwn = ?", c.Param("wwn")).First(&device)
+	if err != nil {
-	device.UpdateFromCollectorSmartInfo(collectorSmartData)
-	if err := db.Model(&device).Updates(device).Error; err != nil {
 		logger.Errorln("An error occurred while updating device data from smartctl metrics", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
 
 	// insert smart info
-	deviceSmartData := dbModels.Smart{}
+	_, err = deviceRepo.SaveSmartAttributes(c, c.Param("wwn"), collectorSmartData)
-	err = deviceSmartData.FromCollectorSmartInfo(c.Param("wwn"), collectorSmartData)
 	if err != nil {
-		logger.Errorln("Could not process SMART metrics", err)
-		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
-		return
-	}
-	if err := db.Create(&deviceSmartData).Error; err != nil {
 		logger.Errorln("An error occurred while saving smartctl metrics", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
 
+	// save smart temperature data (ignore failures)
+	err = deviceRepo.SaveSmartTemperature(c, c.Param("wwn"), updatedDevice.DeviceProtocol, collectorSmartData)
+	if err != nil {
+		logger.Errorln("An error occurred while saving smartctl temp data", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
 	//check for error
-	if deviceSmartData.SmartStatus == dbModels.SmartStatusFailed {
+	if updatedDevice.DeviceStatus != pkg.DeviceStatusPassed {
 		//send notifications
 		testNotify := notify.Notify{
 			Config: appConfig,
 			Payload: notify.Payload{
 				FailureType:  notify.NotifyFailureTypeSmartFailure,
-				DeviceName:   device.DeviceName,
+				DeviceName:   updatedDevice.DeviceName,
-				DeviceType:   device.DeviceProtocol,
+				DeviceType:   updatedDevice.DeviceProtocol,
-				DeviceSerial: device.SerialNumber,
+				DeviceSerial: updatedDevice.SerialNumber,
 				Test:         false,
 			},
 			Logger: logger,

webapp/backend/pkg/web/middleware/repository.go

@@ -0,0 +1,22 @@
+package middleware
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+)
+
+func RepositoryMiddleware(appConfig config.Interface, globalLogger logrus.FieldLogger) gin.HandlerFunc {
+
+	deviceRepo, err := database.NewScrutinyRepository(appConfig, globalLogger)
+	if err != nil {
+		panic(err)
+	}
+
+	//TODO: determine where we can call defer deviceRepo.Close()
+	return func(c *gin.Context) {
+		c.Set("DEVICE_REPOSITORY", deviceRepo)
+		c.Next()
+	}
+}

webapp/backend/pkg/web/middleware/sqlite3.go

@@ -1,59 +0,0 @@
-package middleware
-
-import (
-	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
-	"github.com/gin-gonic/gin"
-	"github.com/sirupsen/logrus"
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-)
-
-func DatabaseMiddleware(appConfig config.Interface, globalLogger logrus.FieldLogger) gin.HandlerFunc {
-
-	//var database *gorm.DB
-	fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location"))
-	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
-		//TODO: figure out how to log database queries again.
-		//Logger: logger
-	})
-	if err != nil {
-		panic("Failed to connect to database!")
-	}
-
-	//database.SetLogger()
-	database.AutoMigrate(&db.Device{})
-	database.AutoMigrate(&db.SelfTest{})
-	database.AutoMigrate(&db.Smart{})
-	database.AutoMigrate(&db.SmartAtaAttribute{})
-	database.AutoMigrate(&db.SmartNvmeAttribute{})
-	database.AutoMigrate(&db.SmartScsiAttribute{})
-
-	//TODO: detrmine where we can call defer database.Close()
-	return func(c *gin.Context) {
-		c.Set("DB", database)
-		c.Next()
-	}
-}
-
-// GormLogger is a custom logger for Gorm, making it use logrus.
-type GormLogger struct{ Logger logrus.FieldLogger }
-
-// Print handles log events from Gorm for the custom logger.
-func (gl *GormLogger) Print(v ...interface{}) {
-	switch v[0] {
-	case "sql":
-		gl.Logger.WithFields(
-			logrus.Fields{
-				"module":  "gorm",
-				"type":    "sql",
-				"rows":    v[5],
-				"src_ref": v[1],
-				"values":  v[4],
-			},
-		).Debug(v[3])
-	case "log":
-		gl.Logger.WithFields(logrus.Fields{"module": "gorm", "type": "log"}).Print(v[2])
-	}
-}

webapp/backend/pkg/web/server.go

@@ -23,7 +23,7 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 	r := gin.New()
 
 	r.Use(middleware.LoggerMiddleware(logger))
-	r.Use(middleware.DatabaseMiddleware(ae.Config, logger))
+	r.Use(middleware.RepositoryMiddleware(ae.Config, logger))
 	r.Use(middleware.ConfigMiddleware(ae.Config))
 	r.Use(gin.Recovery())
 

webapp/backend/pkg/web/server_test.go

@@ -3,7 +3,7 @@ package web_test
 import (
 	"encoding/json"
 	mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/web"
 	"github.com/golang/mock/gomock"
 	"github.com/sirupsen/logrus"
@@ -319,7 +319,7 @@ func TestGetDevicesSummaryRoute_Nvme(t *testing.T) {
 	req, _ = http.NewRequest("GET", "/api/summary", nil)
 	router.ServeHTTP(sr, req)
 	require.Equal(t, 200, sr.Code)
-	var device dbModels.DeviceWrapper
+	var device models.DeviceWrapper
 	json.Unmarshal(sr.Body.Bytes(), &device)
 
 	//assert

webapp/frontend/src/app/modules/dashboard/dashboard.component.html

@@ -1,4 +1,5 @@
-<div *ngIf="data && data.data && data.data.length > 0; else emptyDashboard">
+
+<div *ngIf="data && data.data && data.data.summary; else emptyDashboard">
     <div class="flex flex-col flex-auto w-full p-8 xs:p-2">
 
         <div class="flex flex-wrap w-full">
@@ -47,38 +48,37 @@
             </div>
 
             <div class="flex flex-wrap w-full">
-
+                <div *ngFor="let summary of data.data.summary | keyvalue" class="flex gt-sm:w-1/2 min-w-80 p-4">
-                <div *ngFor="let disk of data.data | deviceSort" class="flex gt-sm:w-1/2 min-w-80 p-4">
+                    <div [ngClass]="{ 'border-green': summary.value.device.device_status == 0 && summary.value.smart,
-                    <div [ngClass]="{'border-green': disk.smart_results[0]?.smart_status == 'passed',
+                                    'border-red': summary.value.device.device_status != 0 }"
-                                    'border-red': disk.smart_results[0]?.smart_status == 'failed' }"
                          class="relative flex flex-col flex-auto p-6 pr-3 pb-3 bg-card rounded border-l-4 shadow-md overflow-hidden">
                         <div class="absolute bottom-0 right-0 w-24 h-24 -m-6">
                             <mat-icon class="icon-size-96 opacity-12 text-green"
-                                      *ngIf="disk.smart_results[0]?.smart_status == 'passed'"
+                                      *ngIf="summary.value.device.device_status == 0 && summary.value.smart"
                                       [svgIcon]="'heroicons_outline:check-circle'"></mat-icon>
                             <mat-icon class="icon-size-96 opacity-12 text-red"
-                                      *ngIf="disk.smart_results[0]?.smart_status == 'failed'"
+                                      *ngIf="summary.value.device.device_status != 0"
                                       [svgIcon]="'heroicons_outline:exclamation-circle'"></mat-icon>
                             <mat-icon class="icon-size-96 opacity-12 text-yellow"
-                                      *ngIf="!disk.smart_results[0]"
+                                      *ngIf="!summary.value.smart"
                                       [svgIcon]="'heroicons_outline:question-mark-circle'"></mat-icon>
                         </div>
                         <div class="flex items-center">
                             <div class="flex flex-col">
-                                <a [routerLink]="'/device/'+ disk.wwn"
+                                <a [routerLink]="'/device/'+ summary.value.device.wwn"
-                                   class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceTitle(disk)}}</a>
+                                   class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceTitle(summary.value.device)}}</a>
-                                <div [ngClass]="{'text-green': disk.smart_results[0]?.smart_status == 'passed',
+                                <div [ngClass]="{'text-green': summary.value.device.device_status == 0 && summary.value.smart,
-                                    'text-red': disk.smart_results[0]?.smart_status == 'failed' }" class="font-medium text-sm" *ngIf="disk.smart_results[0]">
+                                    'text-red': summary.value.device.device_status != 0 }" class="font-medium text-sm" *ngIf="summary.value.smart">
-                                    Last Updated on {{disk.smart_results[0]?.date | date:'MMMM dd, yyyy - HH:mm' }}
+                                    Last Updated on {{summary.value.smart.collector_date | date:'MMMM dd, yyyy - HH:mm' }}
                                 </div>
                             </div>
-                            <div class="ml-auto" *ngIf="disk.smart_results">
+                            <div class="ml-auto" *ngIf="summary.value.device">
                                 <button mat-icon-button
                                         [matMenuTriggerFor]="previousStatementMenu">
                                     <mat-icon [svgIcon]="'more_vert'"></mat-icon>
                                 </button>
                                 <mat-menu #previousStatementMenu="matMenu">
-                                    <a mat-menu-item [routerLink]="'/device/'+ disk.wwn">
+                                    <a mat-menu-item [routerLink]="'/device/'+ summary.value.device.wwn">
                                     <span class="flex items-center">
                                         <mat-icon class="icon-size-20 mr-3"
                                                   [svgIcon]="'payment'"></mat-icon>
@@ -90,22 +90,22 @@
                         </div>
                         <div class="flex flex-row flex-wrap mt-4 -mx-6">
                             <div class="flex flex-col mx-6 my-3 xs:w-full">
-                                <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">S.M.A.R.T</div>
+                                <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Status</div>
-                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="disk.smart_results[0]; else unknownStatus">{{ disk.smart_results[0]?.smart_status | titlecase}}</div>
+                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="summary.value.smart?.collector_date; else unknownStatus">{{ deviceStatusString(summary.value.device.device_status) | titlecase}}</div>
                                 <ng-template #unknownStatus><div class="mt-2 font-medium text-3xl leading-none">No Data</div></ng-template>
                             </div>
                             <div class="flex flex-col mx-6 my-3 xs:w-full">
                                 <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Temperature</div>
-                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="disk.smart_results[0]; else unknownTemp">{{ disk.smart_results[0]?.temp }}°C</div>
+                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="summary.value.smart?.collector_date; else unknownTemp">{{ summary.value.smart?.temp }}°C</div>
                                 <ng-template #unknownTemp><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
                             </div>
                             <div class="flex flex-col mx-6 my-3 xs:w-full">
                                 <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Capacity</div>
-                                <div class="mt-2 font-medium text-3xl leading-none">{{ disk.capacity | fileSize}}</div>
+                                <div class="mt-2 font-medium text-3xl leading-none">{{ summary.value.device.capacity | fileSize}}</div>
                             </div>
                             <div class="flex flex-col mx-6 my-3 xs:w-full">
                                 <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Powered On</div>
-                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="disk.smart_results[0]?.power_on_hours; else unknownPoweredOn">{{ humanizeDuration(disk.smart_results[0]?.power_on_hours * 60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] }) }}</div>
+                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="summary.value.smart?.power_on_hours; else unknownPoweredOn">{{ humanizeDuration(summary.value.smart?.power_on_hours * 60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] }) }}</div>
                                 <ng-template #unknownPoweredOn><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
                             </div>
                         </div>
@@ -140,7 +140,7 @@
 
                     </div>
                     <div class="flex flex-col flex-auto">
-                        <apx-chart class="flex-auto w-full h-full"
+                        <apx-chart *ngIf="temperatureOptions" class="flex-auto w-full h-full"
                                    [chart]="temperatureOptions.chart"
                                    [colors]="temperatureOptions.colors"
                                    [fill]="temperatureOptions.fill"

webapp/frontend/src/app/modules/dashboard/dashboard.component.ts

@@ -84,16 +84,23 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
     private _deviceDataTemperatureSeries() {
         var deviceTemperatureSeries = []
 
-        for(let device of this.data.data){
+        console.log("DEVICE DATA SUMMARY", this.data)
+
+        for(const wwn in this.data.data.summary){
+            var deviceSummary = this.data.data.summary[wwn]
+            if (!deviceSummary.temp_history){
+                continue
+            }
             var deviceSeriesMetadata = {
-                name: `/dev/${device.device_name}`,
+                name: `/dev/${deviceSummary.device.device_name}`,
                 data: []
             }
-            for(let smartResults of device.smart_results){
+
-                let newDate = new Date(smartResults.CreatedAt);
+            for(let tempHistory of deviceSummary.temp_history){
+                let newDate = new Date(tempHistory.date);
                 deviceSeriesMetadata.data.push({
                     x: newDate,
-                    y: smartResults.temp
+                    y: tempHistory.temp
                 })
             }
             deviceTemperatureSeries.push(deviceSeriesMetadata)
@@ -181,6 +188,14 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
         return title.join(' - ')
     }
 
+    deviceStatusString(deviceStatus){
+        if(deviceStatus == 0){
+            return "passed"
+        } else {
+            return "failed"
+        }
+    }
+
     /**
      * Track by function for ngFor loops
      *

webapp/frontend/src/app/modules/detail/detail.component.html

@@ -53,59 +53,59 @@
         <div class="flex flex-auto w-1/4 p-4 lt-md:w-full">
             <treo-card class="flex flex-auto p-4 pt-6 flex-col flex-auto filter-list">
                 <div class="flex items-center justify-between">
-                    <div class="text-2xl font-semibold leading-tight">/dev/{{data.data.device_name}}</div>
+                    <div class="text-2xl font-semibold leading-tight">/dev/{{device?.device_name}}</div>
                 </div>
                 <div class="flex flex-col my-2 grid grid-cols-2">
-                    <div *ngIf="data.data.host_id" class="my-2 col-span-2 lt-md:col-span-1">
+                    <div *ngIf="device?.host_id" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.host_id}}</div>
+                        <div>{{device?.host_id}}</div>
                         <div class="text-secondary text-md">Host ID</div>
                     </div>
-                    <div *ngIf="data.data.device_type && data.data.device_type != 'ata' && data.data.device_type != 'scsi'" class="my-2 col-span-2 lt-md:col-span-1">
+                    <div *ngIf="device?.device_type && device?.device_type != 'ata' && device?.device_type != 'scsi'" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.device_type | uppercase}}</div>
+                        <div>{{device?.device_type | uppercase}}</div>
                         <div class="text-secondary text-md">Device Type</div>
                     </div>
-                    <div *ngIf="data.data.manufacturer" class="my-2 col-span-2 lt-md:col-span-1">
+                    <div *ngIf="device?.manufacturer" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.manufacturer}}</div>
+                        <div>{{device?.manufacturer}}</div>
                         <div class="text-secondary text-md">Model Family</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.model_name}}</div>
+                        <div>{{device?.model_name}}</div>
                         <div class="text-secondary text-md">Device Model</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.serial_number}}</div>
+                        <div>{{device?.serial_number}}</div>
                         <div class="text-secondary text-md">Serial Number</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.wwn}}</div>
+                        <div>{{device?.wwn}}</div>
                         <div class="text-secondary text-md">LU WWN Device Id</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.firmware}}</div>
+                        <div>{{device?.firmware}}</div>
                         <div class="text-secondary text-md">Firmware Version</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.capacity | fileSize}}</div>
+                        <div>{{device?.capacity | fileSize}}</div>
                         <div class="text-secondary text-md">Capacity</div>
                     </div>
-                    <div *ngIf="data.data.rotational_speed" class="my-2 col-span-2 lt-md:col-span-1">
+                    <div *ngIf="device?.rotational_speed" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.rotational_speed}} RPM</div>
+                        <div>{{device?.rotational_speed}} RPM</div>
                         <div class="text-secondary text-md">Rotation Rate</div>
                     </div>
-                    <div *ngIf="data.data.device_protocol" class="my-2 col-span-2 lt-md:col-span-1">
+                    <div *ngIf="device?.device_protocol" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.device_protocol}}</div>
+                        <div>{{device?.device_protocol}}</div>
                         <div class="text-secondary text-md">Protocol</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.smart_results[0]?.power_cycle_count}}</div>
+                        <div>{{smart_results[0]?.power_cycle_count}}</div>
                         <div class="text-secondary text-md">Power Cycle Count</div>
                     </div>
-                    <div *ngIf="data.data.smart_results[0]?.power_on_hours"  class="my-2 col-span-2 lt-md:col-span-1">
+                    <div *ngIf="smart_results[0]?.power_on_hours"  class="my-2 col-span-2 lt-md:col-span-1">
-                        <div matTooltip="{{humanizeDuration(data.data.smart_results[0]?.power_on_hours * 60 * 60 * 1000, { conjunction: ' and ', serialComma: false })}}">{{humanizeDuration(data.data.smart_results[0]?.power_on_hours *60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] })}}</div>
+                        <div matTooltip="{{humanizeDuration(smart_results[0]?.power_on_hours * 60 * 60 * 1000, { conjunction: ' and ', serialComma: false })}}">{{humanizeDuration(smart_results[0]?.power_on_hours *60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] })}}</div>
                         <div class="text-secondary text-md">Powered On</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.smart_results[0]?.temp}}°C</div>
+                        <div>{{smart_results[0]?.temp}}°C</div>
                         <div class="text-secondary text-md">Temperature</div>
                     </div>
                 </div>
@@ -115,7 +115,7 @@
         <div class="flex flex-auto w-3/4 p-4 lt-md:w-full">
             <div class="flex flex-col flex-auto w-full bg-card shadow-md rounded ">
                 <div class="p-6">
-                    <div class="font-bold text-md text-secondary uppercase tracking-wider">S.M.A.R.T {{data.data.device_protocol}} Attributes</div>
+                    <div class="font-bold text-md text-secondary uppercase tracking-wider">S.M.A.R.T {{device?.device_protocol}} Attributes</div>
                     <div class="text-sm text-hint font-medium">{{this.smartAttributeDataSource.data.length}} visible, {{getHiddenAttributes()}} hidden</div>
                 </div>
                 <div class="overflow-auto">

webapp/frontend/src/app/modules/detail/detail.component.ts

@@ -19,7 +19,12 @@ import humanizeDuration from 'humanize-duration';
 export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     onlyCritical: boolean = true;
-    data: any;
+    // data: any;
+
+    metadata: any;
+    device: any;
+    smart_results: any[];
+
     commonSparklineOptions: Partial<ApexOptions>;
     smartAttributeDataSource: MatTableDataSource<any>;
     smartAttributeTableColumns: string[];
@@ -66,10 +71,14 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
             .subscribe((data) => {
 
                 // Store the data
-                this.data = data;
+                // this.data = data;
+                this.device = data.data.device;
+                this.smart_results = data.data.smart_results
+                this.metadata = data.metadata;
+
 
                 // Store the table data
-                this.smartAttributeDataSource.data = this._generateSmartAttributeTableDataSource(data.data.smart_results);
+                this.smartAttributeDataSource.data = this._generateSmartAttributeTableDataSource(this.smart_results);
 
                 // Prepare the chart data
                 this._prepareChartData();
@@ -99,7 +108,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
     // @ Private methods
     // -----------------------------------------------------------------------------------------------------
     getAttributeDescription(attribute_data){
-        let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+        let attribute_metadata = this.metadata[attribute_data.attribute_id]
         if(!attribute_metadata){
             return 'Unknown'
         } else {
@@ -110,7 +119,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     getAttributeValue(attribute_data){
         if(this.isAta()) {
-            let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+            let attribute_metadata = this.metadata[attribute_data.attribute_id]
             if(!attribute_metadata){
                 return attribute_data.value
             } else if (attribute_metadata.display_type == "raw") {
@@ -128,7 +137,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     getAttributeValueType(attribute_data){
         if(this.isAta()) {
-            let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+            let attribute_metadata = this.metadata[attribute_data.attribute_id]
             if(!attribute_metadata){
                 return ''
             } else {
@@ -141,14 +150,14 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     getAttributeIdeal(attribute_data){
         if(this.isAta()){
-            return this.data.metadata[attribute_data.attribute_id]?.display_type == "raw" ? this.data.metadata[attribute_data.attribute_id]?.ideal : ''
+            return this.metadata[attribute_data.attribute_id]?.display_type == "raw" ? this.metadata[attribute_data.attribute_id]?.ideal : ''
         } else {
-            return this.data.metadata[attribute_data.attribute_id]?.ideal
+            return this.metadata[attribute_data.attribute_id]?.ideal
         }
     }
 
     getAttributeWorst(attribute_data){
-        let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+        let attribute_metadata = this.metadata[attribute_data.attribute_id]
         if(!attribute_metadata){
             return attribute_data.worst
         } else {
@@ -158,7 +167,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     getAttributeThreshold(attribute_data){
         if(this.isAta()){
-            let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+            let attribute_metadata = this.metadata[attribute_data.attribute_id]
             if(!attribute_metadata || attribute_metadata.display_type == "normalized"){
                 return attribute_data.thresh
             } else {
@@ -175,29 +184,30 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
     }
 
     getAttributeCritical(attribute_data){
-        return this.data.metadata[attribute_data.attribute_id]?.critical
+        return this.metadata[attribute_data.attribute_id]?.critical
     }
     getHiddenAttributes(){
-        let attributes_list
+        if (!this.smart_results || this.smart_results.length == 0) {
-        if(this.isAta()){
+            return 0
-            attributes_list = this.data.data.smart_results[0]?.ata_attributes
-        } else if(this.isNvme()){
-            attributes_list = this.data.data.smart_results[0]?.nvme_attributes
-        } else {
-            attributes_list = this.data.data.smart_results[0]?.scsi_attributes
         }
 
-        return attributes_list.length - this.smartAttributeDataSource.data.length
+        let attributes_length = 0
+        let attributes = this.smart_results[0]?.attrs
+        if (attributes) {
+            attributes_length = Object.keys(attributes).length
+        }
+
+        return attributes_length - this.smartAttributeDataSource.data.length
     }
 
     isAta(): boolean {
-        return this.data.data.device_protocol == 'ATA'
+        return this.device.device_protocol == 'ATA'
     }
     isScsi(): boolean {
-        return this.data.data.device_protocol == 'SCSI'
+        return this.device.device_protocol == 'SCSI'
     }
     isNvme(): boolean {
-        return this.data.data.device_protocol == 'NVMe'
+        return this.device.device_protocol == 'NVMe'
     }
 
     private _generateSmartAttributeTableDataSource(smart_results){
@@ -207,21 +217,22 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
             return smartAttributeDataSource
         }
         var latest_smart_result = smart_results[0];
-        let attributes_list = []
+        let attributes = {}
         if(this.isScsi()) {
             this.smartAttributeTableColumns = ['status', 'name', 'value', 'thresh', 'history'];
-            attributes_list = latest_smart_result.scsi_attributes
+            attributes = latest_smart_result.attrs
         } else if(this.isNvme()){
             this.smartAttributeTableColumns = ['status', 'name', 'value', 'thresh', 'ideal', 'history'];
-            attributes_list = latest_smart_result.nvme_attributes
+            attributes = latest_smart_result.attrs
         } else {
             //ATA
-            attributes_list = latest_smart_result.ata_attributes
+            attributes = latest_smart_result.attrs
             this.smartAttributeTableColumns = ['status', 'id', 'name', 'value', 'worst', 'thresh','ideal', 'failure', 'history'];
         }
 
+        for(const attrId in attributes){
+            var attr = attributes[attrId]
 
-        for(let attr of attributes_list){
             //chart history data
             if (!attr.chartData) {
                 var rawHistory = (attr.history || []).map(hist_attr => this.getAttributeValue(hist_attr)).reverse()
@@ -235,7 +246,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
             }
             //determine when to include the attributes in table.
 
-            if(!this.onlyCritical || this.onlyCritical && this.data.metadata[attr.attribute_id]?.critical || attr.value < attr.thresh){
+            if(!this.onlyCritical || this.onlyCritical && this.metadata[attr.attribute_id]?.critical || attr.value < attr.thresh){
                 smartAttributeDataSource.push(attr)
             }
         }
@@ -297,7 +308,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
     }
     toggleOnlyCritical(){
         this.onlyCritical = !this.onlyCritical
-        this.smartAttributeDataSource.data = this._generateSmartAttributeTableDataSource(this.data.data.smart_results);
+        this.smartAttributeDataSource.data = this._generateSmartAttributeTableDataSource(this.smart_results);
 
     }